Skip to contentSkip to conversation

Retrieve zPCI hardware information from VFIO

PATCH Superseded Matthew Rosato 4 participants 20 messages Oct 2, 2020 – Oct 7, 2020

Subsystems (2)

v2 of 3 Superseded Latest is v3 →

lore
Share & tools
lore
#0 lore raw

v2 cover Retrieve zPCI hardware information from VFIO

This patchset exploits the VFIO ZPCI CLP region, which provides hardware information about passed-through s390 PCI devices that can be shared with the guest.

The retrieval of this information is done once per function (and for a subset of data, once per function group) and is performed at time of device plug. Some elements provided in the CLP region must still be forced to default values for now to reflect what QEMU actually provides support for.

The original work for this feature was done by Pierre Morel.

Associated kernel patchset:
https://lkml.org/lkml/2020/10/2/981

Changes from v1:
- Added 2 patches to the front of this set that move the s390-pci-bus.h and s390-pci-inst.h files to include + associated MAINTAINERS hit. These can be applied separately, but are included here for the sake of simplicity.
- Patch 4: header update placeholder refreshed to rc7
- Patch 5: Move new s390-pci-clp.h to include folder
- Patch 6+: s/grp/group/ and fallout from this
- Patch 9: Move new s390-pci-vfio.h to include folder

Matthew Rosato (6):
s390x/pci: Move header files to include/hw/s390x
MAINTAINERS: Update s390 PCI entry to include headers
update-linux-headers: Add vfio_zdev.h
linux-headers: update against 5.9-rc7 s390x/pci: clean up s390 PCI groups s390x/pci: get zPCI function info from host

Pierre Morel (3):
s390x/pci: create a header dedicated to PCI CLP
s390x/pci: use a PCI Group structure
s390x/pci: use a PCI Function structure

 MAINTAINERS                                        |   1 +
 hw/s390x/meson.build                               |   1 +
 hw/s390x/s390-pci-bus.c                            |  86 ++++-
 hw/s390x/s390-pci-bus.h                            | 372 --------------------
 hw/s390x/s390-pci-inst.c                           |  33 +-
 hw/s390x/s390-pci-inst.h                           | 312 -----------------
 hw/s390x/s390-pci-vfio.c                           | 235 +++++++++++++
 hw/s390x/s390-virtio-ccw.c                         |   2 +-
 include/hw/s390x/s390-pci-bus.h                    | 385 +++++++++++++++++++++
 include/hw/s390x/s390-pci-clp.h                    | 215 ++++++++++++
 include/hw/s390x/s390-pci-inst.h                   | 116 +++++++
 include/hw/s390x/s390-pci-vfio.h                   |  19 +
 .../drivers/infiniband/hw/vmw_pvrdma/pvrdma_ring.h |  14 +-
 linux-headers/linux/kvm.h                          |   6 +-
 linux-headers/linux/vfio.h                         |   5 +
 scripts/update-linux-headers.sh                    |   2 +-
 16 files changed, 1085 insertions(+), 719 deletions(-)
 delete mode 100644 hw/s390x/s390-pci-bus.h
 delete mode 100644 hw/s390x/s390-pci-inst.h
 create mode 100644 hw/s390x/s390-pci-vfio.c
 create mode 100644 include/hw/s390x/s390-pci-bus.h
 create mode 100644 include/hw/s390x/s390-pci-clp.h
 create mode 100644 include/hw/s390x/s390-pci-inst.h
 create mode 100644 include/hw/s390x/s390-pci-vfio.h
Signature · 1 line
1.8.3.1
#1 lore raw

PATCH v2 1/9 s390x/pci: Move header files to include/hw/s390x

Seems a more appropriate location for them.

---
 hw/s390x/s390-pci-bus.c          |   4 +-
 hw/s390x/s390-pci-bus.h          | 372 ---------------------------------------
 hw/s390x/s390-pci-inst.c         |   4 +-
 hw/s390x/s390-pci-inst.h         | 312 --------------------------------
 hw/s390x/s390-virtio-ccw.c       |   2 +-
 include/hw/s390x/s390-pci-bus.h  | 372 +++++++++++++++++++++++++++++++++++++++
 include/hw/s390x/s390-pci-inst.h | 312 ++++++++++++++++++++++++++++++++
 7 files changed, 689 insertions(+), 689 deletions(-)
 delete mode 100644 hw/s390x/s390-pci-bus.h
 delete mode 100644 hw/s390x/s390-pci-inst.h
 create mode 100644 include/hw/s390x/s390-pci-bus.h
 create mode 100644 include/hw/s390x/s390-pci-inst.h
hw/s390x/s390-pci-bus.c +2 -2
index fb4cee8..a929340 100644
@@ -15,8 +15,8 @@
 #include "qapi/error.h"
 #include "qapi/visitor.h"
 #include "cpu.h"
-#include "s390-pci-bus.h"
-#include "s390-pci-inst.h"
+#include "hw/s390x/s390-pci-bus.h"
+#include "hw/s390x/s390-pci-inst.h"
 #include "hw/pci/pci_bus.h"
 #include "hw/qdev-properties.h"
 #include "hw/pci/pci_bridge.h"
hw/s390x/s390-pci-bus.h deleted +0 -372

372 lines not on this page · raw message

hw/s390x/s390-pci-inst.c +2 -2
index 2f7a7d7..639b13c 100644
@@ -13,12 +13,12 @@
 
 #include "qemu/osdep.h"
 #include "cpu.h"
-#include "s390-pci-inst.h"
-#include "s390-pci-bus.h"
 #include "exec/memop.h"
 #include "exec/memory-internal.h"
 #include "qemu/error-report.h"
 #include "sysemu/hw_accel.h"
+#include "hw/s390x/s390-pci-inst.h"
+#include "hw/s390x/s390-pci-bus.h"
 #include "hw/s390x/tod.h"
 
 #ifndef DEBUG_S390PCI_INST
hw/s390x/s390-pci-inst.h deleted +0 -312

312 lines not on this page · raw message

hw/s390x/s390-virtio-ccw.c +1 -1
index 28266a3..f31ae2e 100644
@@ -28,7 +28,7 @@
 #include "qemu/error-report.h"
 #include "qemu/option.h"
 #include "qemu/qemu-print.h"
-#include "s390-pci-bus.h"
+#include "hw/s390x/s390-pci-bus.h"
 #include "sysemu/reset.h"
 #include "hw/s390x/storage-keys.h"
 #include "hw/s390x/storage-attributes.h"
include/hw/s390x/s390-pci-bus.h new file +372 -0

372 lines not on this page · raw message

include/hw/s390x/s390-pci-inst.h new file +312 -0

312 lines not on this page · raw message

Signature · 1 line
1.8.3.1
#2 lore raw

Re: [PATCH v2 1/9] s390x/pci: Move header files to include/hw/s390x

Quoted 16 lines · On Fri, 2 Oct 2020 16:06:23 -0400 Matthew Rosato <[email protected]>…

Seems a more appropriate location for them.

---
 hw/s390x/s390-pci-bus.c          |   4 +-
 hw/s390x/s390-pci-bus.h          | 372 ---------------------------------------
 hw/s390x/s390-pci-inst.c         |   4 +-
 hw/s390x/s390-pci-inst.h         | 312 --------------------------------
 hw/s390x/s390-virtio-ccw.c       |   2 +-
 include/hw/s390x/s390-pci-bus.h  | 372 +++++++++++++++++++++++++++++++++++++++
 include/hw/s390x/s390-pci-inst.h | 312 ++++++++++++++++++++++++++++++++
 7 files changed, 689 insertions(+), 689 deletions(-)
 delete mode 100644 hw/s390x/s390-pci-bus.h
 delete mode 100644 hw/s390x/s390-pci-inst.h
 create mode 100644 include/hw/s390x/s390-pci-bus.h
 create mode 100644 include/hw/s390x/s390-pci-inst.h

Looks good, but...

<meta>Is there any way to coax out a more reviewable version of this via git mv?</meta>

#3 lore raw

Re: [PATCH v2 1/9] s390x/pci: Move header files to include/hw/s390x

Quoted 20 lines · Cornelia Huck
Quoted 12 lines · On Fri, 2 Oct 2020 16:06:23 -0400 Matthew Rosato <[email protected]>…

Seems a more appropriate location for them.

---
  hw/s390x/s390-pci-bus.c          |   4 +-
  hw/s390x/s390-pci-bus.h          | 372 ---------------------------------------
  hw/s390x/s390-pci-inst.c         |   4 +-
  hw/s390x/s390-pci-inst.h         | 312 --------------------------------
  hw/s390x/s390-virtio-ccw.c       |   2 +-
  include/hw/s390x/s390-pci-bus.h  | 372 +++++++++++++++++++++++++++++++++++++++
  include/hw/s390x/s390-pci-inst.h | 312 ++++++++++++++++++++++++++++++++
  7 files changed, 689 insertions(+), 689 deletions(-)
Quoted 4 lines

delete mode 100644 hw/s390x/s390-pci-bus.h delete mode 100644 hw/s390x/s390-pci-inst.h create mode 100644 include/hw/s390x/s390-pci-bus.h create mode 100644 include/hw/s390x/s390-pci-inst.h

Quoted 5 lines

Looks good, but...

<meta>Is there any way to coax out a more reviewable version of this via git mv?</meta>

I tried git mv, but a diff between the old patch and the new patch looks the same (other than the fact that I squashed the MAINTAINERS hit in)

#4 lore raw

Re: [PATCH v2 1/9] s390x/pci: Move header files to include/hw/s390x

Quoted 8 lines · Matthew Rosato
Quoted 5 lines

Looks good, but...

<meta>Is there any way to coax out a more reviewable version of this via git mv?</meta>

I tried git mv, but a diff between the old patch and the new patch looks the same (other than the fact that I squashed the MAINTAINERS hit in)

git format-patch --find-renames[=<pct>]

Though I'm surprised it's not doing that by default.

r~

#5 lore raw

Re: [PATCH v2 1/9] s390x/pci: Move header files to include/hw/s390x

Quoted 7 lines · Richard Henderson
Quoted 6 lines · Matthew Rosato
Quoted 5 lines

Looks good, but...

<meta>Is there any way to coax out a more reviewable version of this via git mv?</meta>

Quoted 9 lines
Quoted 2 lines

I tried git mv, but a diff between the old patch and the new patch looks the same (other than the fact that I squashed the MAINTAINERS hit in)

git format-patch --find-renames[=<pct>]

Though I'm surprised it's not doing that by default.

r~

Huh, neat. That looks alot better, gives something that looks like:

include/hw/s390x/s390-pci-bus.h renamed from hw/s390x/s390-pci-bus.h +0 -0
similarity index 100%
rename from hw/s390x/s390-pci-bus.h
rename to include/hw/s390x/s390-pci-bus.h
include/hw/s390x/s390-pci-inst.h renamed from hw/s390x/s390-pci-inst.h +0 -0
similarity index 100%
rename from hw/s390x/s390-pci-inst.h
rename to include/hw/s390x/s390-pci-inst.h

Thanks!

maintainer
#6 lore raw

Re: [PATCH v2 1/9] s390x/pci: Move header files to include/hw/s390x

In reply to Richard Henderson
Quoted 7 lines · Richard Henderson
Quoted 6 lines · Matthew Rosato
Quoted 5 lines

Looks good, but...

<meta>Is there any way to coax out a more reviewable version of this via git mv?</meta>

Quoted 6 lines
Quoted 2 lines

I tried git mv, but a diff between the old patch and the new patch looks the same (other than the fact that I squashed the MAINTAINERS hit in)

git format-patch --find-renames[=<pct>]

Though I'm surprised it's not doing that by default.

Yeah, diff.renames should default to true. But you can try

git config --global diff.renames true

and if it fails

git config diff.renames true

Paolo

#7 lore raw

PATCH v2 2/9 MAINTAINERS: Update s390 PCI entry to include headers

In reply to Matthew Rosato

Accomodate changes to file locations.

---
 MAINTAINERS | 1 +
 1 file changed, 1 insertion(+)
MAINTAINERS +1 -0
index b76fb31..dd4e0ea 100644
@@ -1442,6 +1442,7 @@ S390 PCI
 M: Matthew Rosato <[email protected]>
 S: Supported
 F: hw/s390x/s390-pci*
+F: include/hw/s390x/s390-pci*
 L: [email protected]
 
 UniCore32 Machines
Signature · 1 line
1.8.3.1
#8 lore raw

Re: [PATCH v2 2/9] MAINTAINERS: Update s390 PCI entry to include headers

Quoted 12 lines · On Fri, 2 Oct 2020 16:06:24 -0400 Matthew Rosato <[email protected]>…

Accomodate changes to file locations.

---
 MAINTAINERS | 1 +
 1 file changed, 1 insertion(+)
MAINTAINERS +0 -0
index b76fb31..dd4e0ea 100644

MAINTAINERS S390 PCI · line 1448

Quoted 7 lines
MAINTAINERS +1 -0
@@ -1442,6 +1442,7 @@ S390 PCI
 M: Matthew Rosato <[email protected]>
 S: Supported
 F: hw/s390x/s390-pci*
+F: include/hw/s390x/s390-pci*
 L: [email protected]
 
 UniCore32 Machines

I'd probably merge this into the previous patch.

#9 lore raw

PATCH v2 3/9 update-linux-headers: Add vfio_zdev.h

In reply to Matthew Rosato

vfio_zdev.h is used by s390x zPCI support to pass device-specific CLP information between host and userspace.

---
 scripts/update-linux-headers.sh | 2 +-
 1 file changed, 1 insertion(+), 1 deletion(-)
scripts/update-linux-headers.sh +1 -1
index 29c27f4..9efbaf2 100755
@@ -141,7 +141,7 @@ done
 
 rm -rf "$output/linux-headers/linux"
 mkdir -p "$output/linux-headers/linux"
-for header in kvm.h vfio.h vfio_ccw.h vhost.h \
+for header in kvm.h vfio.h vfio_ccw.h vfio_zdev.h vhost.h \
               psci.h psp-sev.h userfaultfd.h mman.h; do
     cp "$tmpdir/include/linux/$header" "$output/linux-headers/linux"
 done
Signature · 1 line
1.8.3.1
Acked-by on v2 3/9
#10 lore raw

Re: [PATCH v2 3/9] update-linux-headers: Add vfio_zdev.h

Quoted 13 lines · On Fri, 2 Oct 2020 16:06:25 -0400 Matthew Rosato <[email protected]>…

vfio_zdev.h is used by s390x zPCI support to pass device-specific CLP information between host and userspace.

---
 scripts/update-linux-headers.sh | 2 +-
 1 file changed, 1 insertion(+), 1 deletion(-)
scripts/update-linux-headers.sh +0 -0
index 29c27f4..9efbaf2 100755

scripts/update-linux-headers.sh done · line 147

Quoted 8 lines
scripts/update-linux-headers.sh +1 -1
@@ -141,7 +141,7 @@ done
 
 rm -rf "$output/linux-headers/linux"
 mkdir -p "$output/linux-headers/linux"
-for header in kvm.h vfio.h vfio_ccw.h vhost.h \
+for header in kvm.h vfio.h vfio_ccw.h vfio_zdev.h vhost.h \
               psci.h psp-sev.h userfaultfd.h mman.h; do
     cp "$tmpdir/include/linux/$header" "$output/linux-headers/linux"
 done

Obviously requires the kernel part to be merged first, but

#11 lore raw

PATCH v2 4/9 linux-headers: update against 5.9-rc7

In reply to Matthew Rosato

PLACEHOLDER as the kernel patch driving the need for this ("vfio-pci/zdev:
define the vfio_zdev header") isn't merged yet.

---
 .../drivers/infiniband/hw/vmw_pvrdma/pvrdma_ring.h         | 14 +++++++-------
 linux-headers/linux/kvm.h                                  |  6 ++++--
 linux-headers/linux/vfio.h                                 |  5 +++++
 3 files changed, 16 insertions(+), 9 deletions(-)
include/standard-headers/drivers/infiniband/hw/vmw_pvrdma/pvrdma_ring.h +7 -7
index 7b4062a..acd4c83 100644
@@ -68,7 +68,7 @@ static inline int pvrdma_idx_valid(uint32_t idx, uint32_t max_elems)
 
 static inline int32_t pvrdma_idx(int *var, uint32_t max_elems)
 {
-	const unsigned int idx = qatomic_read(var);
+	const unsigned int idx = atomic_read(var);
 
 	if (pvrdma_idx_valid(idx, max_elems))
 		return idx & (max_elems - 1);
@@ -77,17 +77,17 @@ static inline int32_t pvrdma_idx(int *var, uint32_t max_elems)
 
 static inline void pvrdma_idx_ring_inc(int *var, uint32_t max_elems)
 {
-	uint32_t idx = qatomic_read(var) + 1;	/* Increment. */
+	uint32_t idx = atomic_read(var) + 1;	/* Increment. */
 
 	idx &= (max_elems << 1) - 1;		/* Modulo size, flip gen. */
-	qatomic_set(var, idx);
+	atomic_set(var, idx);
 }
 
 static inline int32_t pvrdma_idx_ring_has_space(const struct pvrdma_ring *r,
 					      uint32_t max_elems, uint32_t *out_tail)
 {
-	const uint32_t tail = qatomic_read(&r->prod_tail);
-	const uint32_t head = qatomic_read(&r->cons_head);
+	const uint32_t tail = atomic_read(&r->prod_tail);
+	const uint32_t head = atomic_read(&r->cons_head);
 
 	if (pvrdma_idx_valid(tail, max_elems) &&
 	    pvrdma_idx_valid(head, max_elems)) {
@@ -100,8 +100,8 @@ static inline int32_t pvrdma_idx_ring_has_space(const struct pvrdma_ring *r,
 static inline int32_t pvrdma_idx_ring_has_data(const struct pvrdma_ring *r,
 					     uint32_t max_elems, uint32_t *out_head)
 {
-	const uint32_t tail = qatomic_read(&r->prod_tail);
-	const uint32_t head = qatomic_read(&r->cons_head);
+	const uint32_t tail = atomic_read(&r->prod_tail);
+	const uint32_t head = atomic_read(&r->cons_head);
 
 	if (pvrdma_idx_valid(tail, max_elems) &&
 	    pvrdma_idx_valid(head, max_elems)) {
linux-headers/linux/kvm.h +4 -2
index 6683e2e..43580c7 100644
@@ -790,9 +790,10 @@ struct kvm_ppc_resize_hpt {
 #define KVM_VM_PPC_HV 1
 #define KVM_VM_PPC_PR 2
 
-/* on MIPS, 0 forces trap & emulate, 1 forces VZ ASE */
-#define KVM_VM_MIPS_TE		0
+/* on MIPS, 0 indicates auto, 1 forces VZ ASE, 2 forces trap & emulate */
+#define KVM_VM_MIPS_AUTO	0
 #define KVM_VM_MIPS_VZ		1
+#define KVM_VM_MIPS_TE		2
 
 #define KVM_S390_SIE_PAGE_OFFSET 1
 
@@ -1035,6 +1036,7 @@ struct kvm_ppc_resize_hpt {
 #define KVM_CAP_LAST_CPU 184
 #define KVM_CAP_SMALLER_MAXPHYADDR 185
 #define KVM_CAP_S390_DIAG318 186
+#define KVM_CAP_STEAL_TIME 187
 
 #ifdef KVM_CAP_IRQ_ROUTING
 
linux-headers/linux/vfio.h +5 -0
index a906724..68fd67a 100644
@@ -326,6 +326,11 @@ struct vfio_region_info_cap_type {
  * to do TLB invalidation on a GPU.
  */
 #define VFIO_REGION_SUBTYPE_IBM_NVLINK2_ATSD	(1)
+/*
+ * IBM zPCI specific hardware feature information for a devcie.  The contents
+ * of this region are mapped by struct vfio_region_zpci_info.
+ */
+#define VFIO_REGION_SUBTYPE_IBM_ZPCI_CLP	(2)
 
 /* sub-types for VFIO_REGION_TYPE_GFX */
 #define VFIO_REGION_SUBTYPE_GFX_EDID            (1)
Signature · 1 line
1.8.3.1
#12 lore raw

Re: [PATCH v2 4/9] linux-headers: update against 5.9-rc7

Quoted 15 lines · On Fri, 2 Oct 2020 16:06:26 -0400 Matthew Rosato <[email protected]>…

PLACEHOLDER as the kernel patch driving the need for this ("vfio-pci/zdev:
define the vfio_zdev header") isn't merged yet.

---
 .../drivers/infiniband/hw/vmw_pvrdma/pvrdma_ring.h         | 14 +++++++-------
 linux-headers/linux/kvm.h                                  |  6 ++++--
 linux-headers/linux/vfio.h                                 |  5 +++++
 3 files changed, 16 insertions(+), 9 deletions(-)
include/standard-headers/drivers/infiniband/hw/vmw_pvrdma/pvrdma_ring.h +0 -0
index 7b4062a..acd4c83 100644

include/standard-headers/drivers/infiniband/hw/vmw_pvrdma/pvrdma_ring.h static inline int pvrdma_idx_valid(uint32_t idx, uint32_t max_elems) · line 71

Quoted 5 lines
include/standard-headers/drivers/infiniband/hw/vmw_pvrdma/pvrdma_ring.h +1 -1
@@ -68,7 +68,7 @@ static inline int pvrdma_idx_valid(uint32_t idx, uint32_t max_elems)
 
 static inline int32_t pvrdma_idx(int *var, uint32_t max_elems)
 {
-	const unsigned int idx = qatomic_read(var);
+	const unsigned int idx = atomic_read(var);

Hm... either this shouldn't have been renamed to qatomic_read() in the first place, or we need to add some post-processing to the update script.

Quoted 3 lines
include/standard-headers/drivers/infiniband/hw/vmw_pvrdma/pvrdma_ring.h +0 -0
 	if (pvrdma_idx_valid(idx, max_elems))
 		return idx & (max_elems - 1);
#13 lore raw

Re: [PATCH v2 4/9] linux-headers: update against 5.9-rc7

Quoted 19 lines · Cornelia Huck
Quoted 16 lines · On Fri, 2 Oct 2020 16:06:26 -0400 Matthew Rosato <[email protected]>…

PLACEHOLDER as the kernel patch driving the need for this ("vfio-pci/zdev:
define the vfio_zdev header") isn't merged yet.

---
  .../drivers/infiniband/hw/vmw_pvrdma/pvrdma_ring.h         | 14 +++++++-------
  linux-headers/linux/kvm.h                                  |  6 ++++--
  linux-headers/linux/vfio.h                                 |  5 +++++
  3 files changed, 16 insertions(+), 9 deletions(-)
include/standard-headers/drivers/infiniband/hw/vmw_pvrdma/pvrdma_ring.h +0 -0
index 7b4062a..acd4c83 100644
@@ -68,7 +68,7 @@ static inline int pvrdma_idx_valid(uint32_t idx, uint32_t max_elems)
 
Quoted 9 lines

include/standard-headers/drivers/infiniband/hw/vmw_pvrdma/pvrdma_ring.h static inline int pvrdma_idx_valid(uint32_t idx, uint32_t max_elems) · line 71

Quoted 4 lines
include/standard-headers/drivers/infiniband/hw/vmw_pvrdma/pvrdma_ring.h +1 -1
  static inline int32_t pvrdma_idx(int *var, uint32_t max_elems)
  {
-	const unsigned int idx = qatomic_read(var);
+	const unsigned int idx = atomic_read(var);

Hm... either this shouldn't have been renamed to qatomic_read() in the first place, or we need to add some post-processing to the update script.

Before I posted this set, I mentioned this in a reply to Stefan's atomic->qatomic patchset that introduced the change... Paolo replied and said the code shouldn't be getting imported during header updates at all:

https://lists.gnu.org/archive/html/qemu-devel/2020-10/msg00734.html

Copying the maintainers of the pvrdma stuff for their awareness in case they missed the first exchange.

#14 lore raw

PATCH v2 5/9 s390x/pci: create a header dedicated to PCI CLP

In reply to Matthew Rosato

From: Pierre Morel <[email protected]>

To have a clean separation between s390-pci-bus.h and s390-pci-inst.h headers we export the PCI CLP instructions in a dedicated header.

---
 include/hw/s390x/s390-pci-bus.h  |   1 +
 include/hw/s390x/s390-pci-clp.h  | 211 +++++++++++++++++++++++++++++++++++++++
 include/hw/s390x/s390-pci-inst.h | 196 ------------------------------------
 3 files changed, 212 insertions(+), 196 deletions(-)
 create mode 100644 include/hw/s390x/s390-pci-clp.h
include/hw/s390x/s390-pci-bus.h +1 -0
index 97464d0..04e7cb6 100644
@@ -19,6 +19,7 @@
 #include "hw/s390x/sclp.h"
 #include "hw/s390x/s390_flic.h"
 #include "hw/s390x/css.h"
+#include "hw/s390x/s390-pci-clp.h"
 #include "qom/object.h"
 
 #define TYPE_S390_PCI_HOST_BRIDGE "s390-pcihost"
include/hw/s390x/s390-pci-clp.h new file +211 -0

211 lines not on this page · raw message

include/hw/s390x/s390-pci-inst.h +0 -196

202 lines not on this page · raw message

Signature · 1 line
1.8.3.1
Reviewed-by on v2 5/9
#15 lore raw

Re: [PATCH v2 5/9] s390x/pci: create a header dedicated to PCI CLP

Quoted 13 lines · On Fri, 2 Oct 2020 16:06:27 -0400 Matthew Rosato <[email protected]>…

From: Pierre Morel <[email protected]>

To have a clean separation between s390-pci-bus.h and s390-pci-inst.h headers we export the PCI CLP instructions in a dedicated header.

---
 include/hw/s390x/s390-pci-bus.h  |   1 +
 include/hw/s390x/s390-pci-clp.h  | 211 +++++++++++++++++++++++++++++++++++++++
 include/hw/s390x/s390-pci-inst.h | 196 ------------------------------------
 3 files changed, 212 insertions(+), 196 deletions(-)
 create mode 100644 include/hw/s390x/s390-pci-clp.h 

(...)

include/hw/s390x/s390-pci-clp.h line 2

Quoted 8 lines
include/hw/s390x/s390-pci-clp.h new file +2 -0
new file mode 100644
index 0000000..e442307
@@ -0,0 +1,211 @@
+/*
+ * s390 CLPinstruction definitions

s/CLPinstruction/CLP instruction/ ?

Quoted 8 lines
include/hw/s390x/s390-pci-clp.h new file +8 -0
+ *
+ * Copyright 2019 IBM Corp.
+ * Author(s): Pierre Morel <[email protected]>
+ *
+ * This work is licensed under the terms of the GNU GPL, version 2 or (at
+ * your option) any later version. See the COPYING file in the top-level
+ * directory.
+ */

(...)

#16 lore raw

PATCH v2 6/9 s390x/pci: use a PCI Group structure

In reply to Matthew Rosato

From: Pierre Morel <[email protected]>

We use a S390PCIGroup structure to hold the information related to a zPCI Function group.

This allows us to be ready to support multiple groups and to retrieve the group information from the host.

---
 hw/s390x/s390-pci-bus.c         | 42 +++++++++++++++++++++++++++++++++++++++++
 hw/s390x/s390-pci-inst.c        | 23 +++++++++++++---------
 include/hw/s390x/s390-pci-bus.h | 10 ++++++++++
 3 files changed, 66 insertions(+), 9 deletions(-)
hw/s390x/s390-pci-bus.c +42 -0
index a929340..c99f2f0 100644
@@ -737,6 +737,46 @@ static void s390_pci_iommu_free(S390pciState *s, PCIBus *bus, int32_t devfn)
     object_unref(OBJECT(iommu));
 }
 
+static S390PCIGroup *s390_group_create(int id)
+{
+    S390PCIGroup *group;
+    S390pciState *s = s390_get_phb();
+
+    group = g_new0(S390PCIGroup, 1);
+    group->id = id;
+    QTAILQ_INSERT_TAIL(&s->zpci_groups, group, link);
+    return group;
+}
+
+S390PCIGroup *s390_group_find(int id)
+{
+    S390PCIGroup *group;
+    S390pciState *s = s390_get_phb();
+
+    QTAILQ_FOREACH(group, &s->zpci_groups, link) {
+        if (group->id == id) {
+            return group;
+        }
+    }
+    return NULL;
+}
+
+static void s390_pci_init_default_group(void)
+{
+    S390PCIGroup *group;
+    ClpRspQueryPciGrp *resgrp;
+
+    group = s390_group_create(ZPCI_DEFAULT_FN_GRP);
+    resgrp = &group->zpci_group;
+    resgrp->fr = 1;
+    stq_p(&resgrp->dasm, 0);
+    stq_p(&resgrp->msia, ZPCI_MSI_ADDR);
+    stw_p(&resgrp->mui, DEFAULT_MUI);
+    stw_p(&resgrp->i, 128);
+    stw_p(&resgrp->maxstbl, 128);
+    resgrp->version = 0;
+}
+
 static void s390_pcihost_realize(DeviceState *dev, Error **errp)
 {
     PCIBus *b;
@@ -764,7 +804,9 @@ static void s390_pcihost_realize(DeviceState *dev, Error **errp)
     s->bus_no = 0;
     QTAILQ_INIT(&s->pending_sei);
     QTAILQ_INIT(&s->zpci_devs);
+    QTAILQ_INIT(&s->zpci_groups);
 
+    s390_pci_init_default_group();
     css_register_io_adapters(CSS_IO_ADAPTER_PCI, true, false,
                              S390_ADAPTER_SUPPRESSIBLE, errp);
 }
hw/s390x/s390-pci-inst.c +14 -9
index 639b13c..aeb8b5f 100644
@@ -284,21 +284,25 @@ int clp_service_call(S390CPU *cpu, uint8_t r2, uintptr_t ra)
         stq_p(&resquery->edma, ZPCI_EDMA_ADDR);
         stl_p(&resquery->fid, pbdev->fid);
         stw_p(&resquery->pchid, 0);
-        stw_p(&resquery->ug, 1);
+        stw_p(&resquery->ug, ZPCI_DEFAULT_FN_GRP);
         stl_p(&resquery->uid, pbdev->uid);
         stw_p(&resquery->hdr.rsp, CLP_RC_OK);
         break;
     }
     case CLP_QUERY_PCI_FNGRP: {
         ClpRspQueryPciGrp *resgrp = (ClpRspQueryPciGrp *)resh;
-        resgrp->fr = 1;
-        stq_p(&resgrp->dasm, 0);
-        stq_p(&resgrp->msia, ZPCI_MSI_ADDR);
-        stw_p(&resgrp->mui, DEFAULT_MUI);
-        stw_p(&resgrp->i, 128);
-        stw_p(&resgrp->maxstbl, 128);
-        resgrp->version = 0;
 
+        ClpReqQueryPciGrp *reqgrp = (ClpReqQueryPciGrp *)reqh;
+        S390PCIGroup *group;
+
+        group = s390_group_find(reqgrp->g);
+        if (!group) {
+            /* We do not allow access to unknown groups */
+            /* The group must have been obtained with a vfio device */
+            stw_p(&resgrp->hdr.rsp, CLP_RC_QUERYPCIFG_PFGID);
+            goto out;
+        }
+        memcpy(resgrp, &group->zpci_group, sizeof(ClpRspQueryPciGrp));
         stw_p(&resgrp->hdr.rsp, CLP_RC_OK);
         break;
     }
@@ -754,7 +758,8 @@ int pcistb_service_call(S390CPU *cpu, uint8_t r1, uint8_t r3, uint64_t gaddr,
     }
     /* Length must be greater than 8, a multiple of 8 */
     /* and not greater than maxstbl */
-    if ((len <= 8) || (len % 8) || (len > pbdev->maxstbl)) {
+    if ((len <= 8) || (len % 8) ||
+        (len > pbdev->pci_group->zpci_group.maxstbl)) {
         goto specification_error;
     }
     /* Do not cross a 4K-byte boundary */
include/hw/s390x/s390-pci-bus.h +10 -0
index 04e7cb6..52f6e7b 100644
@@ -308,6 +308,14 @@ typedef struct ZpciFmb {
 } ZpciFmb;
 QEMU_BUILD_BUG_MSG(offsetof(ZpciFmb, fmt0) != 48, "padding in ZpciFmb");
 
+#define ZPCI_DEFAULT_FN_GRP 0x20
+typedef struct S390PCIGroup {
+    ClpRspQueryPciGrp zpci_group;
+    int id;
+    QTAILQ_ENTRY(S390PCIGroup) link;
+} S390PCIGroup;
+S390PCIGroup *s390_group_find(int id);
+
 struct S390PCIBusDevice {
     DeviceState qdev;
     PCIDevice *pdev;
@@ -325,6 +333,7 @@ struct S390PCIBusDevice {
     uint16_t noi;
     uint16_t maxstbl;
     uint8_t sum;
+    S390PCIGroup *pci_group;
     S390MsixInfo msix;
     AdapterRoutes routes;
     S390PCIIOMMU *iommu;
@@ -349,6 +358,7 @@ struct S390pciState {
     GHashTable *zpci_table;
     QTAILQ_HEAD(, SeiContainer) pending_sei;
     QTAILQ_HEAD(, S390PCIBusDevice) zpci_devs;
+    QTAILQ_HEAD(, S390PCIGroup) zpci_groups;
 };
 
 S390pciState *s390_get_phb(void);
Signature · 1 line
1.8.3.1
#17 lore raw

PATCH v2 7/9 s390x/pci: clean up s390 PCI groups

In reply to Matthew Rosato

Add a step to remove all stashed PCI groups to avoid stale data between machine resets.

---
 hw/s390x/s390-pci-bus.c | 12 ++++++++++++
 1 file changed, 12 insertions(+)
hw/s390x/s390-pci-bus.c +12 -0
index c99f2f0..c34f14a 100644
@@ -811,6 +811,17 @@ static void s390_pcihost_realize(DeviceState *dev, Error **errp)
                              S390_ADAPTER_SUPPRESSIBLE, errp);
 }
 
+static void s390_pcihost_unrealize(DeviceState *dev)
+{
+    S390PCIGroup *group;
+    S390pciState *s = S390_PCI_HOST_BRIDGE(dev);
+
+    while (!QTAILQ_EMPTY(&s->zpci_groups)) {
+        group = QTAILQ_FIRST(&s->zpci_groups);
+        QTAILQ_REMOVE(&s->zpci_groups, group, link);
+    }
+}
+
 static int s390_pci_msix_init(S390PCIBusDevice *pbdev)
 {
     char *name;
@@ -1165,6 +1176,7 @@ static void s390_pcihost_class_init(ObjectClass *klass, void *data)
 
     dc->reset = s390_pcihost_reset;
     dc->realize = s390_pcihost_realize;
+    dc->unrealize = s390_pcihost_unrealize;
     hc->pre_plug = s390_pcihost_pre_plug;
     hc->plug = s390_pcihost_plug;
     hc->unplug_request = s390_pcihost_unplug_request;
Signature · 1 line
1.8.3.1
#18 lore raw

PATCH v2 8/9 s390x/pci: use a PCI Function structure

In reply to Matthew Rosato

From: Pierre Morel <[email protected]>

We use a ClpRspQueryPci structure to hold the information related to a zPCI Function.

This allows us to be ready to support different zPCI functions and to retrieve the zPCI function information from the host.

---
 hw/s390x/s390-pci-bus.c         | 22 +++++++++++++++++-----
 hw/s390x/s390-pci-inst.c        |  8 ++------
 include/hw/s390x/s390-pci-bus.h |  1 +
 3 files changed, 20 insertions(+), 11 deletions(-)
hw/s390x/s390-pci-bus.c +17 -5
index c34f14a..3dc8a10 100644
@@ -777,6 +777,17 @@ static void s390_pci_init_default_group(void)
     resgrp->version = 0;
 }
 
+static void set_pbdev_info(S390PCIBusDevice *pbdev)
+{
+    pbdev->zpci_fn.sdma = ZPCI_SDMA_ADDR;
+    pbdev->zpci_fn.edma = ZPCI_EDMA_ADDR;
+    pbdev->zpci_fn.pchid = 0;
+    pbdev->zpci_fn.ug = ZPCI_DEFAULT_FN_GRP;
+    pbdev->zpci_fn.fid = pbdev->fid;
+    pbdev->zpci_fn.uid = pbdev->uid;
+    pbdev->pci_group = s390_group_find(ZPCI_DEFAULT_FN_GRP);
+}
+
 static void s390_pcihost_realize(DeviceState *dev, Error **errp)
 {
     PCIBus *b;
@@ -994,17 +1005,18 @@ static void s390_pcihost_plug(HotplugHandler *hotplug_dev, DeviceState *dev,
             }
         }
 
+        pbdev->pdev = pdev;
+        pbdev->iommu = s390_pci_get_iommu(s, pci_get_bus(pdev), pdev->devfn);
+        pbdev->iommu->pbdev = pbdev;
+        pbdev->state = ZPCI_FS_DISABLED;
+        set_pbdev_info(pbdev);
+
         if (object_dynamic_cast(OBJECT(dev), "vfio-pci")) {
             pbdev->fh |= FH_SHM_VFIO;
         } else {
             pbdev->fh |= FH_SHM_EMUL;
         }
 
-        pbdev->pdev = pdev;
-        pbdev->iommu = s390_pci_get_iommu(s, pci_get_bus(pdev), pdev->devfn);
-        pbdev->iommu->pbdev = pbdev;
-        pbdev->state = ZPCI_FS_DISABLED;
-
         if (s390_pci_msix_init(pbdev)) {
             error_setg(errp, "MSI-X support is mandatory "
                        "in the S390 architecture");
hw/s390x/s390-pci-inst.c +2 -6
index aeb8b5f..cc3d274 100644
@@ -267,6 +267,8 @@ int clp_service_call(S390CPU *cpu, uint8_t r2, uintptr_t ra)
             goto out;
         }
 
+        memcpy(resquery, &pbdev->zpci_fn, sizeof(*resquery));
+
         for (i = 0; i < PCI_BAR_COUNT; i++) {
             uint32_t data = pci_get_long(pbdev->pdev->config +
                 PCI_BASE_ADDRESS_0 + (i * 4));
@@ -280,12 +282,6 @@ int clp_service_call(S390CPU *cpu, uint8_t r2, uintptr_t ra)
                     resquery->bar_size[i]);
         }
 
-        stq_p(&resquery->sdma, ZPCI_SDMA_ADDR);
-        stq_p(&resquery->edma, ZPCI_EDMA_ADDR);
-        stl_p(&resquery->fid, pbdev->fid);
-        stw_p(&resquery->pchid, 0);
-        stw_p(&resquery->ug, ZPCI_DEFAULT_FN_GRP);
-        stl_p(&resquery->uid, pbdev->uid);
         stw_p(&resquery->hdr.rsp, CLP_RC_OK);
         break;
     }
include/hw/s390x/s390-pci-bus.h +1 -0
index 52f6e7b..79206be 100644
@@ -334,6 +334,7 @@ struct S390PCIBusDevice {
     uint16_t maxstbl;
     uint8_t sum;
     S390PCIGroup *pci_group;
+    ClpRspQueryPci zpci_fn;
     S390MsixInfo msix;
     AdapterRoutes routes;
     S390PCIIOMMU *iommu;
Signature · 1 line
1.8.3.1
#19 lore raw

PATCH v2 9/9 s390x/pci: get zPCI function info from host

In reply to Matthew Rosato

We use the VFIO_REGION_SUBTYPE_ZDEV_CLP subregion of PCI_VENDOR_ID_IBM to retrieve the CLP information the kernel exports.

To be compatible with previous kernel versions we fall back on previous predefined values, same as the emulation values, when the region is not found. If individual CLP feature(s) are not found in the region, we fall back on default values for only those features missing from the region.

This patch is based on work previously done by Pierre Morel.

---
 hw/s390x/meson.build             |   1 +
 hw/s390x/s390-pci-bus.c          |  10 +-
 hw/s390x/s390-pci-vfio.c         | 235 +++++++++++++++++++++++++++++++++++++++
 include/hw/s390x/s390-pci-bus.h  |   1 +
 include/hw/s390x/s390-pci-clp.h  |  12 +-
 include/hw/s390x/s390-pci-vfio.h |  19 ++++
 6 files changed, 271 insertions(+), 7 deletions(-)
 create mode 100644 hw/s390x/s390-pci-vfio.c
 create mode 100644 include/hw/s390x/s390-pci-vfio.h
hw/s390x/meson.build +1 -0
index 948ceae..3ee4594 100644
@@ -10,6 +10,7 @@ s390x_ss.add(files(
   's390-ccw.c',
   's390-pci-bus.c',
   's390-pci-inst.c',
+  's390-pci-vfio.c',
   's390-skeys.c',
   's390-stattrib.c',
   's390-virtio-hcall.c',
hw/s390x/s390-pci-bus.c +7 -3
index 3dc8a10..182e76e 100644
@@ -17,6 +17,7 @@
 #include "cpu.h"
 #include "hw/s390x/s390-pci-bus.h"
 #include "hw/s390x/s390-pci-inst.h"
+#include "hw/s390x/s390-pci-vfio.h"
 #include "hw/pci/pci_bus.h"
 #include "hw/qdev-properties.h"
 #include "hw/pci/pci_bridge.h"
@@ -737,7 +738,7 @@ static void s390_pci_iommu_free(S390pciState *s, PCIBus *bus, int32_t devfn)
     object_unref(OBJECT(iommu));
 }
 
-static S390PCIGroup *s390_group_create(int id)
+S390PCIGroup *s390_group_create(int id)
 {
     S390PCIGroup *group;
     S390pciState *s = s390_get_phb();
@@ -782,7 +783,7 @@ static void set_pbdev_info(S390PCIBusDevice *pbdev)
     pbdev->zpci_fn.sdma = ZPCI_SDMA_ADDR;
     pbdev->zpci_fn.edma = ZPCI_EDMA_ADDR;
     pbdev->zpci_fn.pchid = 0;
-    pbdev->zpci_fn.ug = ZPCI_DEFAULT_FN_GRP;
+    pbdev->zpci_fn.pfgid = ZPCI_DEFAULT_FN_GRP;
     pbdev->zpci_fn.fid = pbdev->fid;
     pbdev->zpci_fn.uid = pbdev->uid;
     pbdev->pci_group = s390_group_find(ZPCI_DEFAULT_FN_GRP);
@@ -861,7 +862,8 @@ static int s390_pci_msix_init(S390PCIBusDevice *pbdev)
     name = g_strdup_printf("msix-s390-%04x", pbdev->uid);
     memory_region_init_io(&pbdev->msix_notify_mr, OBJECT(pbdev),
                           &s390_msi_ctrl_ops, pbdev, name, PAGE_SIZE);
-    memory_region_add_subregion(&pbdev->iommu->mr, ZPCI_MSI_ADDR,
+    memory_region_add_subregion(&pbdev->iommu->mr,
+                                pbdev->pci_group->zpci_group.msia,
                                 &pbdev->msix_notify_mr);
     g_free(name);
 
@@ -1013,6 +1015,8 @@ static void s390_pcihost_plug(HotplugHandler *hotplug_dev, DeviceState *dev,
 
         if (object_dynamic_cast(OBJECT(dev), "vfio-pci")) {
             pbdev->fh |= FH_SHM_VFIO;
+            /* Fill in CLP information passed via the vfio region */
+            s390_pci_get_clp_info(pbdev);
         } else {
             pbdev->fh |= FH_SHM_EMUL;
         }
hw/s390x/s390-pci-vfio.c new file +235 -0

235 lines not on this page · raw message

include/hw/s390x/s390-pci-bus.h +1 -0
index 79206be..945d5c1 100644
@@ -314,6 +314,7 @@ typedef struct S390PCIGroup {
     int id;
     QTAILQ_ENTRY(S390PCIGroup) link;
 } S390PCIGroup;
+S390PCIGroup *s390_group_create(int id);
 S390PCIGroup *s390_group_find(int id);
 
 struct S390PCIBusDevice {
include/hw/s390x/s390-pci-clp.h +8 -4
index e442307..5dd87c8 100644
@@ -79,6 +79,7 @@ typedef struct ClpFhListEntry {
 #define CLP_SET_DISABLE_PCI_FN 1 /* Yes, 1 disables it */
 
 #define CLP_UTIL_STR_LEN 64
+#define CLP_PFIP_NR_SEGMENTS 4
 
 #define CLP_MASK_FMT 0xf0000000
 
@@ -120,14 +121,17 @@ typedef struct ClpRspQueryPci {
     uint32_t fmt;
     uint64_t reserved1;
     uint16_t vfn; /* virtual fn number */
-#define CLP_RSP_QPCI_MASK_UTIL  0x100
-#define CLP_RSP_QPCI_MASK_PFGID 0xff
-    uint16_t ug;
+#define CLP_RSP_QPCI_MASK_UTIL  0x01
+    uint8_t flags;
+    uint8_t pfgid;
     uint32_t fid; /* pci function id */
     uint8_t bar_size[PCI_BAR_COUNT];
     uint16_t pchid;
     uint32_t bar[PCI_BAR_COUNT];
-    uint64_t reserved2;
+    uint8_t pfip[CLP_PFIP_NR_SEGMENTS];
+    uint16_t reserved2;
+    uint8_t fmbl;
+    uint8_t pft;
     uint64_t sdma; /* start dma as */
     uint64_t edma; /* end dma as */
     uint32_t reserved3[11];
include/hw/s390x/s390-pci-vfio.h new file +19 -0
new file mode 100644
index 0000000..f690dae
@@ -0,0 +1,19 @@
+/*
+ * s390 vfio-pci interfaces
+ *
+ * Copyright 2020 IBM Corp.
+ * Author(s): Matthew Rosato <[email protected]>
+ *
+ * This work is licensed under the terms of the GNU GPL, version 2 or (at
+ * your option) any later version. See the COPYING file in the top-level
+ * directory.
+ */
+
+#ifndef HW_S390_PCI_VFIO_H
+#define HW_S390_PCI_VFIO_H
+
+#include "hw/s390x/s390-pci-bus.h"
+
+void s390_pci_get_clp_info(S390PCIBusDevice *pbdev);
+
+#endif
Signature · 1 line
1.8.3.1