s390x: vfio-ap: guest dedicated crypto adapters

Introduces a VFIO based AP device. The device is defined via the QEMU command line by specifying:

-device vfio-ap,sysfsdev=<path-to-mediated-matrix-device>

There may be only one vfio-ap device configured for a guest.

The mediated matrix device is created by the VFIO AP device driver by writing a UUID to a sysfs attribute file (see docs/vfio-ap.txt). The mediated matrix device will be named after the UUID. Symbolic links to the $uuid are created in many places, so the path to the mediated matrix device $uuid can be specified in any of the following ways:

/sys/devices/vfio_ap/matrix/$uuid /sys/devices/vfio_ap/matrix/mdev_supported_types/vfio_ap-passthrough/devices/$uuid /sys/bus/mdev/devices/$uuid
/sys/bus/mdev/drivers/vfio_mdev/$uuid

When the vfio-ap device is realized, it acquires and opens the VFIO iommu group to which the mediated matrix device is bound. This causes a VFIO group notification event to be signaled. The vfio_ap device driver's group notification handler will get called at which time the device driver will configure the the AP devices to which the guest will be granted access.

Signed-off-by Tony Krowiak <[email protected]>

Tested-by Pierre Morel<[email protected]>

Acked-by Halil Pasic <[email protected]>

Tested-by Pierre Morel<[email protected]>

Tested-by Christian Borntraeger <[email protected]>

---
 MAINTAINERS                       |   2 +
 default-configs/s390x-softmmu.mak |   1 +
 hw/vfio/Makefile.objs             |   1 +
 hw/vfio/ap.c                      | 179 ++++++++++++++++++++++++++++++
 include/hw/vfio/vfio-common.h     |   1 +
 5 files changed, 184 insertions(+)
 create mode 100644 hw/vfio/ap.c

MAINTAINERS +2 -0

index b9984242820b..2730757b4482 100644
@@ -88,6 +88,7 @@ F: hw/char/terminal3270.c
 F: hw/intc/s390_flic.c
 F: hw/intc/s390_flic_kvm.c
 F: hw/s390x/
+F: hw/vfio/ap.c
 F: hw/vfio/ccw.c
 F: hw/watchdog/wdt_diag288.c
 F: include/hw/s390x/

@@ -1217,6 +1218,7 @@ F: hw/s390x/ap-device.c
 F: hw/s390x/ap-bridge.c
 F: include/hw/s390x/ap-device.h
 F: include/hw/s390x/ap-bridge.h
+F: hw/vfio/ap.c
 L: [email protected]
 
 vhost

default-configs/s390x-softmmu.mak +1 -0

index d6b67d50f0e4..5eef37592451 100644
@@ -7,3 +7,4 @@ CONFIG_S390_FLIC=y
 CONFIG_S390_FLIC_KVM=$(CONFIG_KVM)
 CONFIG_VFIO_CCW=$(CONFIG_LINUX)
 CONFIG_WDT_DIAG288=y
+CONFIG_VFIO_AP=$(CONFIG_LINUX)

hw/vfio/Makefile.objs +1 -0

index a2e7a0a7cf02..8b3f664d85f7 100644
@@ -6,4 +6,5 @@ obj-$(CONFIG_SOFTMMU) += platform.o
 obj-$(CONFIG_VFIO_XGMAC) += calxeda-xgmac.o
 obj-$(CONFIG_VFIO_AMD_XGBE) += amd-xgbe.o
 obj-$(CONFIG_SOFTMMU) += spapr.o
+obj-$(CONFIG_VFIO_AP) += ap.o
 endif

hw/vfio/ap.c new file +67 -0

new file mode 100644
index 000000000000..8ec9ad7d4d90
@@ -0,0 +1,179 @@
+/*
+ * VFIO based AP matrix device assignment
+ *
+ * Copyright 2018 IBM Corp.
+ * Author(s): Tony Krowiak <[email protected]>
+ *            Halil Pasic <[email protected]>
+ *
+ * This work is licensed under the terms of the GNU GPL, version 2 or (at
+ * your option) any later version. See the COPYING file in the top-level
+ * directory.
+ */
+
+#include <linux/vfio.h>
+#include <sys/ioctl.h>
+#include "qemu/osdep.h"
+#include "qapi/error.h"
+#include "hw/sysbus.h"
+#include "hw/vfio/vfio.h"
+#include "hw/vfio/vfio-common.h"
+#include "hw/s390x/ap-device.h"
+#include "qemu/error-report.h"
+#include "qemu/queue.h"
+#include "qemu/option.h"
+#include "qemu/config-file.h"
+#include "cpu.h"
+#include "kvm_s390x.h"
+#include "sysemu/sysemu.h"
+#include "hw/s390x/ap-bridge.h"
+#include "exec/address-spaces.h"
+
+#define VFIO_AP_DEVICE_TYPE      "vfio-ap"
+
+typedef struct VFIOAPDevice {
+    APDevice apdev;
+    VFIODevice vdev;
+} VFIOAPDevice;
+
+#define VFIO_AP_DEVICE(obj) \
+        OBJECT_CHECK(VFIOAPDevice, (obj), VFIO_AP_DEVICE_TYPE)
+
+static void vfio_ap_compute_needs_reset(VFIODevice *vdev)
+{
+    vdev->needs_reset = false;
+}
+
+/*
+ * We don't need vfio_hot_reset_multi and vfio_eoi operations for
+ * vfio-ap device now.
+ */
+struct VFIODeviceOps vfio_ap_ops = {
+    .vfio_compute_needs_reset = vfio_ap_compute_needs_reset,
+};
+
+static void vfio_ap_put_device(VFIOAPDevice *vapdev)
+{
+    g_free(vapdev->vdev.name);
+    vfio_put_base_device(&vapdev->vdev);
+}
+
+static VFIOGroup *vfio_ap_get_group(VFIOAPDevice *vapdev, Error **errp)
+{
+    GError *gerror = NULL;
+    char *symlink, *group_path;
+    int groupid;
+
+    symlink = g_strdup_printf("%s/iommu_group", vapdev->vdev.sysfsdev);
+    group_path = g_file_read_link(symlink, &gerror);

hw/vfio/ap.c new file +7 -0

+        return NULL;
+    }
+
+    g_free(group_path);
+
+    return vfio_get_group(groupid, &address_space_memory, errp);
+}

hw/vfio/ap.c new file +17 -0

+static void vfio_ap_class_init(ObjectClass *klass, void *data)
+{
+    DeviceClass *dc = DEVICE_CLASS(klass);
+
+    dc->props = vfio_ap_properties;
+    dc->vmsd = &vfio_ap_vmstate;
+    dc->desc = "VFIO-based AP device assignment";
+    dc->realize = vfio_ap_realize;
+    dc->unrealize = vfio_ap_unrealize;
+    dc->hotpluggable = false;
+    dc->reset = vfio_ap_reset;
+    dc->bus_type = TYPE_AP_BUS;
+}
+
+static const TypeInfo vfio_ap_info = {
+    .name = VFIO_AP_DEVICE_TYPE,
+    .parent = AP_DEVICE_TYPE,

This patch series is the QEMU counterpart to the KVM/kernel support for guest dedicated crypto adapters. The KVM/kernel model is built on the VFIO mediated device framework and provides the infrastructure for granting exclusive guest access to crypto devices installed on the linux host. This patch series introduces a new QEMU command line option, QEMU object model and CPU model features to exploit the KVM/kernel model.

See the detailed specifications for AP virtualization provided by this patch set in docs/vfio-ap.txt for a more complete discussion of the design introduced by this patch series.

v10 => v11 Change log:

=====================
* Replaced DO_UPCAST macros in ap.c
* Initializing GError in vfio_ap_get_group() function (BUG)
* Updated s390 maintainers with new files in MAINTAINERS
* Fixed memory leak in vfio_ap_get_group() function

v9 => v10 Change log:
====================
* Removed KVM_S390_VM_CPU_FEAT_AP feature from kvm.h
* Moved check for KVM_S390_VM_CRYPTO_ENABLE_APIE from patch 2/6 to patch 3/6
* Removed vfio from all function names in ap-bridge.c
* Removed unused macros and structure from ap-bridge.h
* Removed unused macros from ap-device.h

v8 => v9 Change log:
===================
* Removed all references to VFIO in AP bridge and bus
* Expose AP feature only if the KVM_S390_VM_CRYPTO_ENABLE_APIE VM attribute is exposed by KVM - i.e., if AP instructions are available on the linux host.
* Enable AP interpretation only if AP feature is switched on; no need to disable because it is disabled by default.

v7 => v8 Change log:
===================
* Enable SIE interpretation AP instructions if the CPU model feature for AP instructions is turned on for the guest.

v6 => v7 Change log;
===================
* Changed email address for Signed-off-by

v5 => v6 Change log:
===================
* Added reset handling fo vfio-ap device
* Added a bridge/bus to AP device object model - thanks to Halil Pasic

v4 => v5 Change log:
===================
* Added MAINTAINERS entries for VFIO AP
* Added explanation for why we are only supporting zEC12 and newer CPU models.
* Changed CPU model feature qci=on|off to apqci=on|off
* Misc. minor changes

v3 => v4 Change log:
===================
* Made vfio-ap device unpluggable for now

Quoted 11 lines · Tony Krowiak

This patch series is the QEMU counterpart to the KVM/kernel support for guest dedicated crypto adapters. The KVM/kernel model is built on the VFIO mediated device framework and provides the infrastructure for granting exclusive guest access to crypto devices installed on the linux host. This patch series introduces a new QEMU command line option, QEMU object model and CPU model features to exploit the KVM/kernel model.

See the detailed specifications for AP virtualization provided by this patch set in docs/vfio-ap.txt for a more complete discussion of the design introduced by this patch series.

Quoted 1 line

v10 => v11 Change log:

Series seems to work fine.

Christian

Quoted 13 lines · Christian Borntraeger

Quoted 11 lines · Tony Krowiak

This patch series is the QEMU counterpart to the KVM/kernel support for guest dedicated crypto adapters. The KVM/kernel model is built on the VFIO mediated device framework and provides the infrastructure for granting exclusive guest access to crypto devices installed on the linux host. This patch series introduces a new QEMU command line option, QEMU object model and CPU model features to exploit the KVM/kernel model.

See the detailed specifications for AP virtualization provided by this patch set in docs/vfio-ap.txt for a more complete discussion of the design introduced by this patch series.

Quoted 5 lines

Quoted 1 line

v10 => v11 Change log:

Series seems to work fine.

Christian

Had a quick glimpse over all patches, looks good and clean to me.

Quoted 11 lines · Tony Krowiak

This patch series is the QEMU counterpart to the KVM/kernel support for guest dedicated crypto adapters. The KVM/kernel model is built on the VFIO mediated device framework and provides the infrastructure for granting exclusive guest access to crypto devices installed on the linux host. This patch series introduces a new QEMU command line option, QEMU object model and CPU model features to exploit the KVM/kernel model.

See the detailed specifications for AP virtualization provided by this patch set in docs/vfio-ap.txt for a more complete discussion of the design introduced by this patch series.

Quoted 1 line

v10 => v11 Change log:

Series seems to work fine.

Christian

Quoted 16 lines · On Wed, 10 Oct 2018 20:15:55 +0200 Christian Borntraeger…

Quoted 12 lines · Tony Krowiak

This patch series is the QEMU counterpart to the KVM/kernel support for guest dedicated crypto adapters. The KVM/kernel model is built on the VFIO mediated device framework and provides the infrastructure for granting exclusive guest access to crypto devices installed on the linux host. This patch series introduces a new QEMU command line option, QEMU object model and CPU model features to exploit the KVM/kernel model.

See the detailed specifications for AP virtualization provided by this patch set in docs/vfio-ap.txt for a more complete discussion of the design introduced by this patch series.

v10 => v11 Change log:

Quoted 3 lines

Series seems to work fine.

Christian

Is that a Tested-by:, or just 'looks sane'?

Quoted 19 lines · Cornelia Huck

Quoted 16 lines · On Wed, 10 Oct 2018 20:15:55 +0200 Christian Borntraeger…

Quoted 12 lines · Tony Krowiak

This patch series is the QEMU counterpart to the KVM/kernel support for guest dedicated crypto adapters. The KVM/kernel model is built on the VFIO mediated device framework and provides the infrastructure for granting exclusive guest access to crypto devices installed on the linux host. This patch series introduces a new QEMU command line option, QEMU object model and CPU model features to exploit the KVM/kernel model.

See the detailed specifications for AP virtualization provided by this patch set in docs/vfio-ap.txt for a more complete discussion of the design introduced by this patch series.

v10 => v11 Change log:

Quoted 5 lines

Quoted 3 lines

Series seems to work fine.

Christian

Is that a Tested-by:, or just 'looks sane'?

It was limited to some cases, but lets make it a tested-by.

This patch series is the QEMU counterpart to the KVM/kernel support for guest dedicated crypto adapters. The KVM/kernel model is built on the VFIO mediated device framework and provides the infrastructure for granting exclusive guest access to crypto devices installed on the linux host. This patch series introduces a new QEMU command line option, QEMU object model and CPU model features to exploit the KVM/kernel model.

See the detailed specifications for AP virtualization provided by this patch set in docs/vfio-ap.txt for a more complete discussion of the design introduced by this patch series.