s390x/vfio: VFIO-AP interrupt control interception
cover s390x/vfio: VFIO-AP interrupt control interception
The S390 APQP/AQIC instruction can be intercepted by the host to configure the AP queues interruption handling for and handle the ISC used by the host and the guest and the indicator address.
This patch series define the AQIC feature in the cpumodel, extend the APDevice type for per queue interrupt handling, intercept the APQP/AQIC instruction, uses the S390 adapter interface to setup the adapter and use a VFIO ioctl to let the VFIO-AP driver handle the host instruction associated with the intercepted guest instruction.
This patch serie can be tested with the Linux/KVM patch series for the VFIO-AP driver: "s390: vfio: ap: Using GISA for AP Interrupt"
Pierre Morel (5):
s390x/vfio: ap: Linux uapi VFIO place holder s390x/cpumodel: Set up CPU model for AQIC interception s390x/vfio: ap: Definition for AP Adapter type s390x/vfio: ap: Intercepting AP Queue Interrupt Control s390x/vfio: ap: Implementing AP Queue Interrupt Control
hw/vfio/ap.c | 100 ++++++++++++++++++++++++++++++++ include/hw/s390x/ap-device.h | 55 ++++++++++++++++++ include/hw/s390x/css.h | 1 + linux-headers/linux/vfio.h | 22 +++++++ target/s390x/cpu_features.c | 1 + target/s390x/cpu_features_def.h | 1 + target/s390x/cpu_models.c | 1 + target/s390x/kvm.c | 20 +++++++ 8 files changed, 201 insertions(+)
Signature · 1 line
2.17.0
PATCH 1/5 s390x/vfio: ap: Linux uapi VFIO place holder
This file would be copied from Linux,
I put it here for the review.
--- linux-headers/linux/vfio.h | 22 ++++++++++++++++++++++ 1 file changed, 22 insertions(+)
linux-headers/linux/vfio.h +22 -0
@@ -816,6 +816,28 @@ struct vfio_iommu_spapr_tce_remove { };#define VFIO_IOMMU_SPAPR_TCE_REMOVE _IO(VFIO_TYPE, VFIO_BASE + 20) +/** +* VFIO_AP_SET_IRQ - _IOWR(VFIO_TYPE, VFIO_BASE + 21, struct vfio_ap_aqic) +* +* Setup IRQ for an AP Queue +* @cmd contains the AP queue number (apqn) +* @status receives the resulting status of the command +* @nib is the Notification Indicator byte address +* @adapter_id allows to retrieve the associated adapter +*/ +struct vfio_ap_aqic { +__u32 argsz; +__u32 flags; +/* in */ +__u64 cmd; +__u64 status; +__u64 nib; +__u32 adapter_id; +}; +#define VFIO_AP_SET_IRQ _IO(VFIO_TYPE, VFIO_BASE + 21) +#define VFIO_AP_CLEAR_IRQ _IO(VFIO_TYPE, VFIO_BASE + 22) +/* ***************************************************************** */ +#endif /* VFIO_H */
Signature · 1 line
2.17.0
PATCH 2/5 s390x/cpumodel: Set up CPU model for AQIC interception
In reply to Pierre MorelA new CPU model facilities is introduced to support AP devices interruption interception for a KVM guest.
CPU model facility:
The S390_FEAT_AP_QUEUE_INTERRUPT_CONTROL, CPU facility indicates whether AP interruption interception is available to the guest. This feature will be enabled only if the AP instructions are available on the linux host and AQIC facility is installed on the host.
This feature must be turned on from userspace to intercept AP instructions on the KVM guest. The QEMU command line to turn this feature on looks something like this:
qemu-system-s390x ... -cpu xxx,aqci=on ...
--- target/s390x/cpu_features.c | 1 + target/s390x/cpu_features_def.h | 1 + target/s390x/cpu_models.c | 1 + 3 files changed, 3 insertions(+)
target/s390x/cpu_features.c +1 -0
@@ -84,6 +84,7 @@ static const S390FeatDef s390_features[] = {FEAT_INIT ("sema" ,S390_FEAT_TYPE_STFL ,59 ,"Semaphore-assist facility" ),FEAT_INIT ("tsi" ,S390_FEAT_TYPE_STFL ,60 ,"Time-slice Instrumentation facility" ),FEAT_INIT ("ri" ,S390_FEAT_TYPE_STFL ,64 ,"CPU runtime-instrumentation facility" ), +FEAT_INIT ("aqic" ,S390_FEAT_TYPE_STFL ,65 ,"AP-Queue interruption Control facility" ),FEAT_INIT ("zpci" ,S390_FEAT_TYPE_STFL ,69 ,"z/PCI facility" ),FEAT_INIT ("aen" ,S390_FEAT_TYPE_STFL ,71 ,"General-purpose-adapter-event-notification facility" ), FEAT_INIT("ais", S390_FEAT_TYPE_STFL, 72, "General-purpose-adapter-interruption-suppression facility"),
target/s390x/cpu_features_def.h +1 -0
@@ -72,6 +72,7 @@ typedef enum {S390_FEAT_SEMAPHORE_ASSIST ,S390_FEAT_TIME_SLICE_INSTRUMENTATION ,S390_FEAT_RUNTIME_INSTRUMENTATION , +S390_FEAT_AP_QUEUE_INTERRUPT_CONTROL ,S390_FEAT_ZPCI ,S390_FEAT_ADAPTER_EVENT_NOTIFICATION , S390_FEAT_ADAPTER_INT_SUPPRESSION,
target/s390x/cpu_models.c +1 -0
@@ -788,6 +788,7 @@ static void check_consistency(const S390CPUModel *model) { S390_FEAT_SIE_KSS, S390_FEAT_SIE_F2 }, { S390_FEAT_AP_QUERY_CONFIG_INFO, S390_FEAT_AP }, { S390_FEAT_AP_FACILITIES_TEST, S390_FEAT_AP }, + { S390_FEAT_AP_QUEUE_INTERRUPT_CONTROL, S390_FEAT_AP }, };int i;
Signature · 1 line
2.17.0
PATCH 3/5 s390x/vfio: ap: Definition for AP Adapter type
In reply to Pierre MorelFrom: Pierre Morel <[email protected]>
Let's define the AP adapter type to use it with standard adapter interface.
--- include/hw/s390x/css.h | 1 + 1 file changed, 1 insertion(+)
include/hw/s390x/css.h +1 -0
@@ -217,6 +217,7 @@ IOInstEnding do_subchannel_work_passthrough(SubchDev *sub);typedef enum {CSS_IO_ADAPTER_VIRTIO =0 ,CSS_IO_ADAPTER_PCI =1 , +CSS_IO_ADAPTER_AP =2 ,CSS_IO_ADAPTER_TYPE_NUMS , }CssIoAdapterType ;
Signature · 1 line
2.17.0
PATCH 4/5 s390x/vfio: ap: Intercepting AP Queue Interrupt Control
In reply to Pierre MorelFrom: Pierre Morel <[email protected]>
We intercept the PQAP(AQIC) instruction.
Until we implement AQIC we return a PGM_OPERATION.
--- hw/vfio/ap.c | 10 ++++++++++ include/hw/s390x/ap-device.h | 9 +++++++++ target/s390x/kvm.c | 20 ++++++++++++++++++++ 3 files changed, 39 insertions(+)
hw/vfio/ap.c +10 -0
@@ -38,6 +38,16 @@ typedef struct VFIOAPDevice {#define VFIO_AP_DEVICE (obj) \ OBJECT_CHECK(VFIOAPDevice, (obj), VFIO_AP_DEVICE_TYPE) +/* +* ap_pqap +* @env: environment pointing to registers +* return value: Code Condition +*/ +int ap_pqap (CPUS390XState *env) +{ +return -PGM_OPERATION ; +} +static void vfio_ap_compute_needs_reset (VFIODevice *vdev) { vdev->needs_reset = false;
include/hw/s390x/ap-device.h +9 -0
@@ -19,4 +19,13 @@ typedef struct APDevice {#define AP_DEVICE (obj) \ OBJECT_CHECK(APDevice, (obj), AP_DEVICE_TYPE) +#define AP_DEVICE_GET_CLASS (obj) \ + OBJECT_GET_CLASS(APDeviceClass, (obj), AP_DEVICE_TYPE) + +#define AP_DEVICE_CLASS (klass) \ + OBJECT_CLASS_CHECK(APDeviceClass, (klass), AP_DEVICE_TYPE) + +#include "cpu.h" +int ap_pqap (CPUS390XState *env); +#endif /* HW_S390X_AP_DEVICE_H */
target/s390x/kvm.c +20 -0
@@ -45,6 +45,7 @@#include "trace.h" #include "hw/s390x/s390-pci-inst.h" #include "hw/s390x/s390-pci-bus.h" +#include "hw/s390x/ap-device.h" #include "hw/s390x/ipl.h" #include "hw/s390x/ebcdic.h" #include "exec/memattrs.h"
@@ -88,6 +89,7 @@#define PRIV_B2_CHSC 0x5f#define PRIV_B2_SIGA 0x74#define PRIV_B2_XSCH 0x76 +#define PRIV_B2_PQAP 0xaf#define PRIV_EB_SQBS 0x8a#define PRIV_EB_PCISTB 0xd0
@@ -1154,6 +1156,21 @@ static int kvm_sclp_service_call(S390CPU *cpu, struct kvm_run *run,return 0 ; } +static int kvm_ap_pqap (S390CPU *cpu,uint16_t ipbh0) +{ +int r; + + r =ap_pqap (&cpu->env); + +if (r <0 ) { +kvm_s390_program_interrupt (cpu, -r); + }else { +setcc (cpu, r); + } + +return 0 ; +} +static int handle_b2 (S390CPU *cpu,struct kvm_run *run,uint8_t ipa1) {CPUS390XState *env = &cpu->env;
@@ -1216,6 +1233,9 @@ static int handle_b2(S390CPU *cpu, struct kvm_run *run, uint8_t ipa1)case PRIV_B2_SCLP_CALL : rc =kvm_sclp_service_call (cpu, run, ipbh0);break ; +case PRIV_B2_PQAP : + rc =kvm_ap_pqap (cpu, ipbh0); +break ;default : rc =-1 ;DPRINTF ("KVM: unhandled PRIV: 0xb2%x\n" , ipa1);
Signature · 1 line
2.17.0
Re: [Qemu-devel] [PATCH v1 4/5] s390x/vfio: ap: Intercepting AP Queue Interrupt Control
Quoted 1 line · On Fri, 2 Nov 2018 11:30:20 +0100 Pierre Morel <[email protected]> wrote:
From: Pierre Morel <[email protected]>
Meta: you may want to rewrite your authorship to the shorter address.
Quoted 19 lines
We intercept the PQAP(AQIC) instruction.
Until we implement AQIC we return a
PGM_OPERATION.Signed-off-by Pierre Morel <[email protected]>--- hw/vfio/ap.c | 10 ++++++++++ include/hw/s390x/ap-device.h | 9 +++++++++ target/s390x/kvm.c | 20 ++++++++++++++++++++ 3 files changed, 39 insertions(+)hw/vfio/ap.c +0
-0@@ -38,6 +38,16 @@ typedef struct VFIOAPDevice {#define VFIO_AP_DEVICE (obj) \ OBJECT_CHECK(VFIOAPDevice, (obj), VFIO_AP_DEVICE_TYPE)
hw/vfio/ap.c typedef struct VFIOAPDevice { · line 49
Quoted 10 lines
hw/vfio/ap.c +9
-0+/* +* ap_pqap +* @env: environment pointing to registers +* return value: Code Condition +*/ +int ap_pqap (CPUS390XState *env) +{ +return -PGM_OPERATION ; +}
I'm not sure whether it makes sense to add such a skeleton handler here; perhaps we should merge with the next patch that actually does something for AQCI?
Quoted 9 lines
hw/vfio/ap.c +1
-0+static void vfio_ap_compute_needs_reset (VFIODevice *vdev) { vdev->needs_reset = false;include/hw/s390x/ap-device.h +0
-0
include/hw/s390x/ap-device.h typedef struct APDevice { · line 26
Quoted 8 lines
include/hw/s390x/ap-device.h +5
-0@@ -19,4 +19,13 @@ typedef struct APDevice {#define AP_DEVICE (obj) \ OBJECT_CHECK(APDevice, (obj), AP_DEVICE_TYPE) +#define AP_DEVICE_GET_CLASS (obj) \ + OBJECT_GET_CLASS(APDeviceClass, (obj), AP_DEVICE_TYPE) + +#define AP_DEVICE_CLASS (klass) \ + OBJECT_CLASS_CHECK(APDeviceClass, (klass), AP_DEVICE_TYPE)
Looks like an unrelated change -- does that maybe belong into the next patch?
include/hw/s390x/ap-device.h typedef struct APDevice { · line 31
Quoted 5 lines
include/hw/s390x/ap-device.h +4
-0+ +#include "cpu.h" +int ap_pqap (CPUS390XState *env); +#endif /* HW_S390X_AP_DEVICE_H */
The wiring up looks reasonable.
Re: [Qemu-devel] [PATCH v1 4/5] s390x/vfio: ap: Intercepting AP Queue Interrupt Control
Quoted 6 lines · Cornelia Huck
Quoted 1 line · On Fri, 2 Nov 2018 11:30:20 +0100 Pierre Morel <[email protected]> wrote:
From: Pierre Morel <[email protected]>
Meta: you may want to rewrite your authorship to the shorter address.
right thanks (this patch is in my queue since too long)
Quoted 21 lines
Quoted 21 lines
We intercept the PQAP(AQIC) instruction.
Until we implement AQIC we return a
PGM_OPERATION.Signed-off-by Pierre Morel <[email protected]>--- hw/vfio/ap.c | 10 ++++++++++ include/hw/s390x/ap-device.h | 9 +++++++++ target/s390x/kvm.c | 20 ++++++++++++++++++++ 3 files changed, 39 insertions(+)hw/vfio/ap.c +0
-0@@ -38,6 +38,16 @@ typedef struct VFIOAPDevice {#define VFIO_AP_DEVICE (obj) \ OBJECT_CHECK(VFIOAPDevice, (obj), VFIO_AP_DEVICE_TYPE)
Quoted 13 lines
hw/vfio/ap.c typedef struct VFIOAPDevice { · line 49
Quoted 9 lines
hw/vfio/ap.c +9
-0+/* +* ap_pqap +* @env: environment pointing to registers +* return value: Code Condition +*/ +int ap_pqap (CPUS390XState *env) +{ +return -PGM_OPERATION ; +}I'm not sure whether it makes sense to add such a skeleton handler here; perhaps we should merge with the next patch that actually does something for AQCI?
OK
Quoted 13 lines
Quoted 13 lines
hw/vfio/ap.c +1
-0+static void vfio_ap_compute_needs_reset (VFIODevice *vdev) { vdev->needs_reset = false;include/hw/s390x/ap-device.h +0
-0@@ -19,4 +19,13 @@ typedef struct APDevice {#define AP_DEVICE (obj) \ OBJECT_CHECK(APDevice, (obj), AP_DEVICE_TYPE)
Quoted 8 lines
include/hw/s390x/ap-device.h typedef struct APDevice { · line 26
Quoted 5 lines
include/hw/s390x/ap-device.h +5
-0+#define AP_DEVICE_GET_CLASS (obj) \ + OBJECT_GET_CLASS(APDeviceClass, (obj), AP_DEVICE_TYPE) + +#define AP_DEVICE_CLASS (klass) \ + OBJECT_CLASS_CHECK(APDeviceClass, (klass), AP_DEVICE_TYPE)Looks like an unrelated change -- does that maybe belong into the next patch?
Seems indeed.
Quoted 9 lines
include/hw/s390x/ap-device.h typedef struct APDevice { · line 31
Quoted 6 lines
include/hw/s390x/ap-device.h +4
-0+ +#include "cpu.h" +int ap_pqap (CPUS390XState *env); +#endif /* HW_S390X_AP_DEVICE_H */ The wiring up looks reasonable.
Thanks,
Pierre
Signature · 2 lines
Pierre Morel Linux/KVM/QEMU in Böblingen - Germany
PATCH 5/5 s390x/vfio: ap: Implementing AP Queue Interrupt Control
In reply to Pierre MorelWe intercept the PQAP(AQIC) instruction and transform the guest's AQIC command parameters for the host AQIC parameters.
Doing this we use the standard adapter interface to provide the adapter NIB, indicator and ISC.
We define a new structure, APQueue to keep track of the route and indicator address and we add an array of AP Queues in the VFIOAPDevice.
We call the VFIO ioctl to set or clear the interruption according to the "i" bit of the parameter.
--- hw/vfio/ap.c | 92 +++++++++++++++++++++++++++++++++++- include/hw/s390x/ap-device.h | 46 ++++++++++++++++++ 2 files changed, 137 insertions(+), 1 deletion(-)
hw/vfio/ap.c +91 -1
@@ -27,17 +27,90 @@#include "sysemu/sysemu.h" #include "hw/s390x/ap-bridge.h" #include "exec/address-spaces.h" +#include "hw/s390x/s390_flic.h" +#include "hw/s390x/css.h" #define VFIO_AP_DEVICE_TYPE "vfio-ap"typedef struct VFIOAPDevice {APDevice apdev;VFIODevice vdev; +QTAILQ_ENTRY (VFIOAPDevice) sibling; +APQueue apq[MAX_AP ][MAX_DOMAIN ]; }VFIOAPDevice ;#define VFIO_AP_DEVICE (obj) \ OBJECT_CHECK(VFIOAPDevice, (obj), VFIO_AP_DEVICE_TYPE) +VFIOAPDevice *vfio_apdev; +static APDevice *matrix; + +static int ap_aqic (CPUS390XState *env) +{ +struct pqap_cmd cmd =reg2cmd (env->regs[0 ]); +struct ap_status status =reg2status (env->regs[1 ]); +uint64_t guest_nib = env->regs[2 ]; +struct vfio_ap_aqic param = {}; +int retval; +VFIODevice *vdev; +VFIOAPDevice *ap_vdev; +APQueue *apq; + + ap_vdev =DO_UPCAST (VFIOAPDevice, apdev, matrix); + apq = &ap_vdev->apq[cmd.apid][cmd.apqi]; + vdev = &ap_vdev->vdev; + +if (status.irq) { +if (apq->nib) { + status.rc =AP_RC_BAD_STATE ; + goto error; + } + }else { +if (!apq->nib) { + status.rc =AP_RC_BAD_STATE ; + goto error; + } + } +if (!guest_nib) { + status.rc =AP_RC_INVALID_ADDR ; + goto error; + } + + apq->routes.adapter.adapter_id =css_get_adapter_id ( +CSS_IO_ADAPTER_AP , status.isc); + + apq->nib =get_indicator (ldq_p (&guest_nib),8 ); + + retval =map_indicator (&apq->routes.adapter, apq->nib); +if (retval) { + status.rc =AP_RC_INVALID_ADDR ; + env->regs[1 ] =status2reg (status); + goto error; + } + + param.cmd = env->regs[0 ]; + param.status = env->regs[1 ]; + param.nib = env->regs[2 ]; + param.adapter_id = apq->routes.adapter.adapter_id; + param.argsz =sizeof (param); + + retval =ioctl (vdev->fd,VFIO_AP_SET_IRQ , ¶m); + status =reg2status (param.status); +if (retval) { + goto err_ioctl; + } + + env->regs[1 ] = param.status; + +return 0 ; +err_ioctl: +release_indicator (&apq->routes.adapter, apq->nib); + apq->nib =NULL ; +error: + env->regs[1 ] =status2reg (status); +return 0 ; +} +/* * ap_pqap * @env: environment pointing to registers
@@ -45,7 +118,20 @@ typedef struct VFIOAPDevice {*/ int ap_pqap (CPUS390XState *env) { -return -PGM_OPERATION ; +struct pqap_cmd cmd =reg2cmd (env->regs[0 ]); +int cc =0 ; + +switch (cmd.fc) { +case AQIC : +if (!s390_has_feat (S390_FEAT_AP_QUEUE_INTERRUPT_CONTROL )) { +return -PGM_OPERATION ; + } + cc =ap_aqic (env); +break ; +default : +return -PGM_OPERATION ; + } +return cc; }static void vfio_ap_compute_needs_reset (VFIODevice *vdev)
@@ -119,6 +205,9 @@ static void vfio_ap_realize(DeviceState *dev, Error **errp) goto out_get_dev_err; } + matrix = apdev; +css_register_io_adapters (CSS_IO_ADAPTER_AP , true, false, +0 , &error_abort);return ; out_get_dev_err:
@@ -135,6 +224,7 @@ static void vfio_ap_unrealize(DeviceState *dev, Error **errp)VFIOGroup *group = vapdev->vdev.group;vfio_ap_put_device (vapdev); + matrix =NULL ;vfio_put_group (group); }
include/hw/s390x/ap-device.h +46 -0
50 lines not on this page · raw message
Signature · 1 line
2.17.0
Re: [Qemu-devel] [PATCH v1 5/5] s390x/vfio: ap: Implementing AP Queue Interrupt Control
Quoted 35 lines · On Fri, 2 Nov 2018 11:30:21 +0100 Pierre Morel <[email protected]> wrote:
We intercept the PQAP(AQIC) instruction and transform the guest's AQIC command parameters for the host AQIC parameters.
Doing this we use the standard adapter interface to provide the adapter NIB, indicator and ISC.
We define a new structure, APQueue to keep track of the route and indicator address and we add an array of AP Queues in the VFIOAPDevice.
We call the VFIO ioctl to set or clear the interruption according to the "i" bit of the parameter.
Signed-off-by Pierre Morel <[email protected]>--- hw/vfio/ap.c | 92 +++++++++++++++++++++++++++++++++++- include/hw/s390x/ap-device.h | 46 ++++++++++++++++++ 2 files changed, 137 insertions(+), 1 deletion(-)hw/vfio/ap.c +2
-0@@ -27,17 +27,90 @@#include "sysemu/sysemu.h" #include "hw/s390x/ap-bridge.h" #include "exec/address-spaces.h" +#include "hw/s390x/s390_flic.h" +#include "hw/s390x/css.h" #define VFIO_AP_DEVICE_TYPE "vfio-ap" typedef struct VFIOAPDevice { APDevice apdev;
hw/vfio/ap.c line 46
Quoted 10 lines
hw/vfio/ap.c +4
-0VFIODevice vdev; +QTAILQ_ENTRY (VFIOAPDevice) sibling; +APQueue apq[MAX_AP ][MAX_DOMAIN ]; } VFIOAPDevice;#define VFIO_AP_DEVICE (obj) \ OBJECT_CHECK(VFIOAPDevice, (obj), VFIO_AP_DEVICE_TYPE) +VFIOAPDevice *vfio_apdev; +static APDevice *matrix;
Why do you need those variables? Can't you get the target device in a different way?
hw/vfio/ap.c line 48
Quoted 2 lines
hw/vfio/ap.c +2
-0+ +static int ap_aqic (CPUS390XState *env)
<bikeshed>I think ap_pqap_aqic would be slightly better.</bikeshed>
hw/vfio/ap.c line 51
Quoted 3 lines
hw/vfio/ap.c +3
-0+{ +struct pqap_cmd cmd =reg2cmd (env->regs[0 ]); +struct ap_status status =reg2status (env->regs[1 ]);
I don't really like those reg2<whatever> helpers; they obfuscate things IMO.
Also: These are internal structs and not copied from Linux, right? Then they should be CamelCase.
hw/vfio/ap.c line 52
Quoted 1 line
hw/vfio/ap.c +1
-0+uint64_t guest_nib = env->regs[2 ];
Another point: What about endianness? Even though this is kvm-only, I would like an emulation of an instruction to be endian-clean. (Maybe it also needs a different split?)
Quoted 11 lines
hw/vfio/ap.c +11
-0+struct vfio_ap_aqic param = {}; +int retval; +VFIODevice *vdev; +VFIOAPDevice *ap_vdev; +APQueue *apq; + + ap_vdev =DO_UPCAST (VFIOAPDevice, apdev, matrix); + apq = &ap_vdev->apq[cmd.apid][cmd.apqi]; + vdev = &ap_vdev->vdev; + +if (status.irq) {
hw/vfio/ap.c line 73
Quoted 10 lines
hw/vfio/ap.c +10
-0+if (apq->nib) { + status.rc =AP_RC_BAD_STATE ; + goto error; + } + }else { +if (!apq->nib) { + status.rc =AP_RC_BAD_STATE ; + goto error; + } + }
Maybe
if (!!status.irq == !!apq->nib) {
status.rc = AP_RC_BAD_STATE;
goto error;
}?
Quoted 4 lines
hw/vfio/ap.c +4
-0+if (!guest_nib) { + status.rc =AP_RC_INVALID_ADDR ; + goto error; + }
hw/vfio/ap.c line 87
Quoted 10 lines
hw/vfio/ap.c +10
-0+ + apq->routes.adapter.adapter_id =css_get_adapter_id ( +CSS_IO_ADAPTER_AP , status.isc); + + apq->nib =get_indicator (ldq_p (&guest_nib),8 ); + + retval =map_indicator (&apq->routes.adapter, apq->nib); +if (retval) { + status.rc =AP_RC_INVALID_ADDR ; + env->regs[1 ] =status2reg (status);
I do not like the backwards conversion macros, either :(
Quoted 30 lines
hw/vfio/ap.c +26
-0+ goto error; + } + + param.cmd = env->regs[0 ]; + param.status = env->regs[1 ]; + param.nib = env->regs[2 ]; + param.adapter_id = apq->routes.adapter.adapter_id; + param.argsz =sizeof (param); + + retval =ioctl (vdev->fd,VFIO_AP_SET_IRQ , ¶m); + status =reg2status (param.status); +if (retval) { + goto err_ioctl; + } + + env->regs[1 ] = param.status; + +return 0 ; +err_ioctl: +release_indicator (&apq->routes.adapter, apq->nib); + apq->nib =NULL ; +error: + env->regs[1 ] =status2reg (status); +return 0 ; +} +/* * ap_pqap * @env: environment pointing to registers
hw/vfio/ap.c typedef struct VFIOAPDevice { · line 121
Quoted 5 lines
hw/vfio/ap.c +1
-1@@ -45,7 +118,20 @@ typedef struct VFIOAPDevice {*/ int ap_pqap (CPUS390XState *env) { -return -PGM_OPERATION ; +struct pqap_cmd cmd =reg2cmd (env->regs[0 ]);
Dito on the macro.
hw/vfio/ap.c typedef struct VFIOAPDevice { · line 124
Quoted 3 lines
hw/vfio/ap.c +3
-0+int cc =0 ; + +switch (cmd.fc) {
This probably needs some endian handling as well.
hw/vfio/ap.c typedef struct VFIOAPDevice { · line 125
Quoted 1 line
hw/vfio/ap.c +1
-0+case AQIC :
What about calling this PQAP_AQCI?
Quoted 13 lines
hw/vfio/ap.c +9
-0+if (!s390_has_feat (S390_FEAT_AP_QUEUE_INTERRUPT_CONTROL )) { +return -PGM_OPERATION ; + } + cc =ap_aqic (env); +break ; +default : +return -PGM_OPERATION ; + } +return cc; }static void vfio_ap_compute_needs_reset (VFIODevice *vdev)
hw/vfio/ap.c static void vfio_ap_realize(DeviceState *dev, Error **errp) · line 208
Quoted 4 lines
hw/vfio/ap.c +1
-0@@ -119,6 +205,9 @@ static void vfio_ap_realize(DeviceState *dev, Error **errp) goto out_get_dev_err; } + matrix = apdev;
Huh?
Quoted 31 lines
hw/vfio/ap.c +3
-0+css_register_io_adapters (CSS_IO_ADAPTER_AP , true, false, +0 , &error_abort);return ; out_get_dev_err:@@ -135,6 +224,7 @@ static void vfio_ap_unrealize(DeviceState *dev, Error **errp)VFIOGroup *group = vapdev->vdev.group;vfio_ap_put_device (vapdev); + matrix =NULL ;vfio_put_group (group); }include/hw/s390x/ap-device.h +10
-0@@ -28,4 +28,50 @@ typedef struct APDevice {#include "cpu.h" int ap_pqap (CPUS390XState *env); +#define MAX_AP 256 +#define MAX_DOMAIN 256 + +#include "hw/s390x/s390_flic.h" +#include "hw/s390x/css.h" +typedef struct APQueue { + uint32_t apid; + uint32_t apqi; + AdapterRoutes routes; + IndAddr *nib;
include/hw/s390x/ap-device.h typedef struct APDevice { · line 50
Quoted 10 lines
include/hw/s390x/ap-device.h +10
-0+} APQueue; + +/* AP PQAP commands definitions */ +#define AQIC 0x03 + +struct pqap_cmd { +uint32_t unused; +uint8_t fc; +unsigned t:1 ; +unsigned reserved:7 ;
It is better to make this an uint8_t and define an accessor for the 't' bit.
include/hw/s390x/ap-device.h typedef struct APDevice { · line 53
Quoted 3 lines
include/hw/s390x/ap-device.h +3
-0+uint8_t apid; +uint8_t apqi; +};
Also, please use typedef and CamelCase :)
Quoted 8 lines
include/hw/s390x/ap-device.h +8
-0+/* AP status returned by the AP PQAP commands */ +#define AP_RC_APQN_INVALID 0x01 +#define AP_RC_INVALID_ADDR 0x06 +#define AP_RC_BAD_STATE 0x07 + +struct ap_status { +uint16_t pad; +unsigned irq:1 ;
include/hw/s390x/ap-device.h typedef struct APDevice { · line 71
Quoted 10 lines
include/hw/s390x/ap-device.h +10
-0+ unsigned pad2:15; + unsigned e:1; + unsigned r:1; + unsigned f:1; + unsigned unused:4; + unsigned i:1; +unsigned char rc; + unsigned reserved:13; + unsigned isc:3; +};
Dito on bitfields and CamelCase :)
Quoted 6 lines
include/hw/s390x/ap-device.h +5
-0+ +#define reg2cmd (reg) (*(struct pqap_cmd *)&(reg)) +#define status2reg (status) (*((uint64_t *)&status)) +#define reg2status (reg) (*(struct ap_status *)&(reg)) +#endif /* HW_S390X_AP_DEVICE_H */
Re: [Qemu-devel] [PATCH v1 5/5] s390x/vfio: ap: Implementing AP Queue Interrupt Control
Quoted 52 lines · Cornelia Huck
Quoted 35 lines · On Fri, 2 Nov 2018 11:30:21 +0100 Pierre Morel <[email protected]> wrote:
We intercept the PQAP(AQIC) instruction and transform the guest's AQIC command parameters for the host AQIC parameters.
Doing this we use the standard adapter interface to provide the adapter NIB, indicator and ISC.
We define a new structure, APQueue to keep track of the route and indicator address and we add an array of AP Queues in the VFIOAPDevice.
We call the VFIO ioctl to set or clear the interruption according to the "i" bit of the parameter.
Signed-off-by Pierre Morel <[email protected]>--- hw/vfio/ap.c | 92 +++++++++++++++++++++++++++++++++++- include/hw/s390x/ap-device.h | 46 ++++++++++++++++++ 2 files changed, 137 insertions(+), 1 deletion(-)hw/vfio/ap.c +2
-0@@ -27,17 +27,90 @@#include "sysemu/sysemu.h" #include "hw/s390x/ap-bridge.h" #include "exec/address-spaces.h" +#include "hw/s390x/s390_flic.h" +#include "hw/s390x/css.h" #define VFIO_AP_DEVICE_TYPE "vfio-ap" typedef struct VFIOAPDevice { APDevice apdev;hw/vfio/ap.c line 46
Quoted 10 lines
hw/vfio/ap.c +4
-0VFIODevice vdev; +QTAILQ_ENTRY (VFIOAPDevice) sibling; +APQueue apq[MAX_AP ][MAX_DOMAIN ]; } VFIOAPDevice;#define VFIO_AP_DEVICE (obj) \ OBJECT_CHECK(VFIOAPDevice, (obj), VFIO_AP_DEVICE_TYPE) +VFIOAPDevice *vfio_apdev; +static APDevice *matrix;Why do you need those variables? Can't you get the target device in a different way?
Quoted 4 lines
hw/vfio/ap.c line 48
Quoted 2 lines
hw/vfio/ap.c +2
-0+ +static int ap_aqic (CPUS390XState *env)<bikeshed>I think
ap_pqap_aqicwould be slightly better.</bikeshed>
agreed.
Quoted 8 lines
hw/vfio/ap.c line 51
Quoted 4 lines
hw/vfio/ap.c +3
-0+{ +struct pqap_cmd cmd =reg2cmd (env->regs[0 ]); +struct ap_status status =reg2status (env->regs[1 ]);I don't really like those reg2<whatever> helpers; they obfuscate things IMO.
I agree, I will use masks and uint64_t/uint32_t.
Quoted 8 lines
Also: These are internal structs and not copied from Linux, right? Then they should be CamelCase.
hw/vfio/ap.c line 52
Quoted 1 line
hw/vfio/ap.c +1
-0+uint64_t guest_nib = env->regs[2 ];Another point: What about endianness? Even though this is kvm-only, I would like an emulation of an instruction to be endian-clean. (Maybe it also needs a different split?)
OK, I will look at this
Quoted 24 lines
Quoted 12 lines
hw/vfio/ap.c +11
-0+struct vfio_ap_aqic param = {}; +int retval; +VFIODevice *vdev; +VFIOAPDevice *ap_vdev; +APQueue *apq; + + ap_vdev =DO_UPCAST (VFIOAPDevice, apdev, matrix); + apq = &ap_vdev->apq[cmd.apid][cmd.apqi]; + vdev = &ap_vdev->vdev; + +if (status.irq) {hw/vfio/ap.c line 73
Quoted 10 lines
hw/vfio/ap.c +10
-0+if (apq->nib) { + status.rc =AP_RC_BAD_STATE ; + goto error; + } + }else { +if (!apq->nib) { + status.rc =AP_RC_BAD_STATE ; + goto error; + } + }Maybe
Quoted 5 lines
if (!!status.irq == !!apq->nib) { status.rc = AP_RC_BAD_STATE; goto error; }?
:)
Quoted 60 lines
Quoted 5 lines
hw/vfio/ap.c +4
-0+if (!guest_nib) { + status.rc =AP_RC_INVALID_ADDR ; + goto error; + }hw/vfio/ap.c line 87
Quoted 10 lines
hw/vfio/ap.c +10
-0+ + apq->routes.adapter.adapter_id =css_get_adapter_id ( +CSS_IO_ADAPTER_AP , status.isc); + + apq->nib =get_indicator (ldq_p (&guest_nib),8 ); + + retval =map_indicator (&apq->routes.adapter, apq->nib); +if (retval) { + status.rc =AP_RC_INVALID_ADDR ; + env->regs[1 ] =status2reg (status);I do not like the backwards conversion macros, either :(
Quoted 30 lines
hw/vfio/ap.c +26
-0+ goto error; + } + + param.cmd = env->regs[0 ]; + param.status = env->regs[1 ]; + param.nib = env->regs[2 ]; + param.adapter_id = apq->routes.adapter.adapter_id; + param.argsz =sizeof (param); + + retval =ioctl (vdev->fd,VFIO_AP_SET_IRQ , ¶m); + status =reg2status (param.status); +if (retval) { + goto err_ioctl; + } + + env->regs[1 ] = param.status; + +return 0 ; +err_ioctl: +release_indicator (&apq->routes.adapter, apq->nib); + apq->nib =NULL ; +error: + env->regs[1 ] =status2reg (status); +return 0 ; +} +/* * ap_pqap * @env: environment pointing to registers hw/vfio/ap.c typedef struct VFIOAPDevice { · line 121
Quoted 5 lines
hw/vfio/ap.c +1
-1@@ -45,7 +118,20 @@ typedef struct VFIOAPDevice {*/ int ap_pqap (CPUS390XState *env) { -return -PGM_OPERATION ; +struct pqap_cmd cmd =reg2cmd (env->regs[0 ]);Dito on the macro.
hw/vfio/ap.c typedef struct VFIOAPDevice { · line 124
Quoted 3 lines
hw/vfio/ap.c +3
-0+int cc =0 ; + +switch (cmd.fc) {
Quoted 5 lines
This probably needs some endian handling as well.
hw/vfio/ap.c typedef struct VFIOAPDevice { · line 125
Quoted 1 line
hw/vfio/ap.c +1
-0+case AQIC :What about calling this
PQAP_AQCI?
I can use it , however the function itself is called ap_pqap() so we already know it is about PQAP...
Quoted 12 lines
Quoted 12 lines
hw/vfio/ap.c +9
-0+if (!s390_has_feat (S390_FEAT_AP_QUEUE_INTERRUPT_CONTROL )) { +return -PGM_OPERATION ; + } + cc =ap_aqic (env); +break ; +default : +return -PGM_OPERATION ; + } +return cc; }
Quoted 8 lines
hw/vfio/ap.c static void vfio_ap_realize(DeviceState *dev, Error **errp) · line 208
Quoted 6 lines
hw/vfio/ap.c +1
-0static void vfio_ap_compute_needs_reset (VFIODevice *vdev)@@ -119,6 +205,9 @@ static void vfio_ap_realize(DeviceState *dev, Error **errp) goto out_get_dev_err; } + matrix = apdev;Huh?
OK, sorry, I can do better. (I hope)
Quoted 37 lines
Quoted 37 lines
hw/vfio/ap.c +3
-0+css_register_io_adapters (CSS_IO_ADAPTER_AP , true, false, +0 , &error_abort);return ; out_get_dev_err:@@ -135,6 +224,7 @@ static void vfio_ap_unrealize(DeviceState *dev, Error **errp)VFIOGroup *group = vapdev->vdev.group;vfio_ap_put_device (vapdev); + matrix =NULL ;vfio_put_group (group); }include/hw/s390x/ap-device.h +15
-0@@ -28,4 +28,50 @@ typedef struct APDevice {#include "cpu.h" int ap_pqap (CPUS390XState *env); +#define MAX_AP 256 +#define MAX_DOMAIN 256 + +#include "hw/s390x/s390_flic.h" +#include "hw/s390x/css.h" +typedef struct APQueue { +uint32_t apid; +uint32_t apqi; +AdapterRoutes routes; +IndAddr *nib; +}APQueue ; + +/* AP PQAP commands definitions */ +#define AQIC 0x03 +
Quoted 8 lines
include/hw/s390x/ap-device.h typedef struct APDevice { · line 50
Quoted 5 lines
include/hw/s390x/ap-device.h +5
-0+struct pqap_cmd { +uint32_t unused; +uint8_t fc; +unsigned t:1 ; +unsigned reserved:7 ;It is better to make this an
uint8_tand define an accessor for the 't' bit.
OK
Quoted 6 lines
include/hw/s390x/ap-device.h typedef struct APDevice { · line 53
Quoted 4 lines
include/hw/s390x/ap-device.h +3
-0+uint8_t apid; +uint8_t apqi; +};Also, please use typedef and CamelCase :)
right, more camels
Quoted 13 lines
Quoted 13 lines
include/hw/s390x/ap-device.h +12
-0+/* AP status returned by the AP PQAP commands */ +#define AP_RC_APQN_INVALID 0x01 +#define AP_RC_INVALID_ADDR 0x06 +#define AP_RC_BAD_STATE 0x07 + +struct ap_status { +uint16_t pad; +unsigned irq:1 ; +unsigned pad2:15 ; +unsigned e:1 ; +unsigned r:1 ; +unsigned f:1 ;
Quoted 8 lines
include/hw/s390x/ap-device.h typedef struct APDevice { · line 71
Quoted 6 lines
include/hw/s390x/ap-device.h +6
-0+ unsigned unused:4; + unsigned i:1; +unsigned char rc; + unsigned reserved:13; + unsigned isc:3; +};Dito on bitfields and CamelCase :)
OK again, even more Camels
Thanks,
Pierre
Quoted 8 lines
Quoted 7 lines
include/hw/s390x/ap-device.h +5
-0+ +#define reg2cmd (reg) (*(struct pqap_cmd *)&(reg)) +#define status2reg (status) (*((uint64_t *)&status)) +#define reg2status (reg) (*(struct ap_status *)&(reg)) +#endif /* HW_S390X_AP_DEVICE_H */
Signature · 2 lines
Pierre Morel Linux/KVM/QEMU in Böblingen - Germany
Quoted 21 lines · Pierre Morel
The S390 APQP/AQIC instruction can be intercepted by the host to configure the AP queues interruption handling for and handle the ISC used by the host and the guest and the indicator address.
This patch series define the AQIC feature in the cpumodel, extend the APDevice type for per queue interrupt handling, intercept the APQP/AQIC instruction, uses the S390 adapter interface to setup the adapter and use a VFIO ioctl to let the VFIO-AP driver handle the host instruction associated with the intercepted guest instruction.
This patch serie can be tested with the Linux/KVM patch series for the VFIO-AP driver: "s390: vfio: ap: Using GISA for AP Interrupt"
Pierre Morel (5):
s390x/vfio: ap: Linux uapi VFIO place holder s390x/cpumodel: Set up CPU model for AQIC interception s390x/vfio: ap: Definition for AP Adapter type s390x/vfio: ap: Intercepting AP Queue Interrupt Control s390x/vfio: ap: Implementing AP Queue Interrupt Control
Quoted 10 lines
hw/vfio/ap.c | 100 ++++++++++++++++++++++++++++++++ include/hw/s390x/ap-device.h | 55 ++++++++++++++++++ include/hw/s390x/css.h | 1 + linux-headers/linux/vfio.h | 22 +++++++ target/s390x/cpu_features.c | 1 + target/s390x/cpu_features_def.h | 1 + target/s390x/cpu_models.c | 1 + target/s390x/kvm.c | 20 +++++++ 8 files changed, 201 insertions(+)
I had a very quick high level look at it and it seems to be in general fine.
Signature · 3 lines
Thanks, David / dhildenb