Skip to contentSkip to conversation

s390x: Protected Virtualization support

PATCH Superseded Janosch Frank 4 participants 32 messages Feb 20, 2020 – Feb 25, 2020

v4 of 11 Superseded Latest is v11 →

lore
Share & tools
lore
#0 lore raw

v4 cover s390x: Protected Virtualization support

Most of the QEMU changes for PV are related to the new IPL type with subcodes 8 - 10 and the execution of the necessary Ultravisor calls to IPL secure guests. Note that we can only boot into secure mode from normal mode, i.e. stfle 161 is not active in secure mode.

The other changes related to data gathering for emulation and disabling addressing checks in secure mode, as well as CPU resets.

v4:
* Dropped reset patches which are already picked up
* Sync with KVM changes
* Review changes

V3:
* Use dedicated functions to access SIDA
* Smaller cleanups and segfault fixes
* Error reporting for Ultravisor calls
* Inject of RC of diag308 subcode 10 fails

V2:
* Split out cleanups
* Internal PV state tracking
* Review feedback

Christian Borntraeger (1):
s390x: Add unpack feature to GA1

Janosch Frank (15):
Sync pv
s390x: protvirt: Add diag308 subcodes 8 - 10
s390x: protvirt: Support unpack facility
s390x: protvirt: Add migration blocker
s390x: protvirt: Handle diag 308 subcodes 0,1,3,4
s390x: protvirt: KVM intercept changes
s390x: Add SIDA memory ops
s390x: protvirt: Move STSI data over SIDAD
s390x: protvirt: SCLP interpretation
s390x: protvirt: Set guest IPL PSW
s390x: protvirt: Move diag 308 data over SIDAD
s390x: protvirt: Disable address checks for PV guest IO emulation
s390x: protvirt: Move IO control structures over SIDA
s390x: protvirt: Handle SIGP store status correctly
docs: Add protvirt docs

 docs/protvirt.rst                   |  53 +++++++++++
 hw/s390x/Makefile.objs              |   1 +
 hw/s390x/ipl.c                      |  81 ++++++++++++++++-
 hw/s390x/ipl.h                      |  33 +++++++
 hw/s390x/pv.c                       | 106 ++++++++++++++++++++++
 hw/s390x/pv.h                       |  34 +++++++
 hw/s390x/s390-virtio-ccw.c          | 135 +++++++++++++++++++++++++++-
 hw/s390x/sclp.c                     |  17 ++++
 include/hw/s390x/s390-virtio-ccw.h  |   1 +
 include/hw/s390x/sclp.h             |   2 +
 linux-headers/linux/kvm.h           |  48 +++++++++-
 target/s390x/cpu.c                  |  27 ++++--
 target/s390x/cpu.h                  |   8 +-
 target/s390x/cpu_features_def.inc.h |   1 +
 target/s390x/diag.c                 |  61 ++++++++++---
 target/s390x/gen-features.c         |   1 +
 target/s390x/helper.c               |   4 +
 target/s390x/ioinst.c               | 113 ++++++++++++++++-------
 target/s390x/kvm.c                  |  54 ++++++++++-
 target/s390x/kvm_s390x.h            |   2 +
 target/s390x/mmu_helper.c           |   9 ++
 target/s390x/sigp.c                 |   1 +
 22 files changed, 726 insertions(+), 66 deletions(-)
 create mode 100644 docs/protvirt.rst
 create mode 100644 hw/s390x/pv.c
 create mode 100644 hw/s390x/pv.h
Signature · 1 line
2.20.1
#1 lore raw

PATCH v4 1/16 Sync pv

---
 linux-headers/linux/kvm.h | 46 +++++++++++++++++++++++++++++++++++----
 1 file changed, 42 insertions(+), 4 deletions(-)
linux-headers/linux/kvm.h +42 -4
index ec146bd52a..2e647f2d9b 100644
@@ -474,8 +474,11 @@ struct kvm_s390_mem_op {
 	__u32 size;		/* amount of bytes */
 	__u32 op;		/* type of operation */
 	__u64 buf;		/* buffer in userspace */
-	__u8 ar;		/* the access register number */
-	__u8 reserved[31];	/* should be set to 0 */
+	union {
+		__u8 ar;	/* the access register number */
+		__u32 sida_offset; /* offset into the sida */
+		__u8 reserved[32]; /* should be set to 0 */
+	};
 };
 /* types for kvm_s390_mem_op->op */
 #define KVM_S390_MEMOP_LOGICAL_READ	0
@@ -1010,6 +1013,7 @@ struct kvm_ppc_resize_hpt {
 #define KVM_CAP_ARM_NISV_TO_USER 177
 #define KVM_CAP_ARM_INJECT_EXT_DABT 178
 #define KVM_CAP_S390_VCPU_RESETS 179
+#define KVM_CAP_S390_PROTECTED 180
 
 #ifdef KVM_CAP_IRQ_ROUTING
 
@@ -1474,8 +1478,42 @@ struct kvm_enc_region {
 /* Available with KVM_CAP_ARM_SVE */
 #define KVM_ARM_VCPU_FINALIZE	  _IOW(KVMIO,  0xc2, int)
 
-#define KVM_S390_NORMAL_RESET	  _IO(KVMIO,  0xc3)
-#define KVM_S390_CLEAR_RESET	  _IO(KVMIO,  0xc4)
+/* Available with  KVM_CAP_S390_VCPU_RESETS */
+#define KVM_S390_NORMAL_RESET	_IO(KVMIO,   0xc3)
+#define KVM_S390_CLEAR_RESET	_IO(KVMIO,   0xc4)
+
+struct kvm_s390_pv_sec_parm {
+	__u64 origin;
+	__u64 length;
+};
+
+struct kvm_s390_pv_unp {
+	__u64 addr;
+	__u64 size;
+	__u64 tweak;
+};
+
+enum pv_cmd_id {
+	KVM_PV_ENABLE,
+	KVM_PV_DISABLE,
+	KVM_PV_VM_SET_SEC_PARMS,
+	KVM_PV_VM_UNPACK,
+	KVM_PV_VM_VERIFY,
+	KVM_PV_VM_PREP_RESET,
+	KVM_PV_VM_UNSHARE_ALL,
+};
+
+struct kvm_pv_cmd {
+	__u32 cmd;	/* Command to be executed */
+	__u16 rc;	/* Ultravisor return code */
+	__u16 rrc;	/* Ultravisor return reason code */
+	__u64 data;	/* Data or address */
+	__u32 flags;    /* flags for future extensions. Must be 0 for now */
+	__u32 reserved[3];
+};
+
+/* Available with KVM_CAP_S390_PROTECTED */
+#define KVM_S390_PV_COMMAND		_IOWR(KVMIO, 0xc5, struct kvm_pv_cmd)
 
 /* Secure Encrypted Virtualization command */
 enum sev_cmd_id {
Signature · 1 line
2.20.1
#2 lore raw

Re: [PATCH v4 01/16] Sync pv

Quoted 32 lines · Janosch Frank
---
 linux-headers/linux/kvm.h | 46 +++++++++++++++++++++++++++++++++++----
 1 file changed, 42 insertions(+), 4 deletions(-)
linux-headers/linux/kvm.h +6 -2
index ec146bd52a..2e647f2d9b 100644
@@ -474,8 +474,11 @@ struct kvm_s390_mem_op {
 	__u32 size;		/* amount of bytes */
 	__u32 op;		/* type of operation */
 	__u64 buf;		/* buffer in userspace */
-	__u8 ar;		/* the access register number */
-	__u8 reserved[31];	/* should be set to 0 */
+	union {
+		__u8 ar;	/* the access register number */
+		__u32 sida_offset; /* offset into the sida */
+		__u8 reserved[32]; /* should be set to 0 */
+	};
 };
 /* types for kvm_s390_mem_op->op */
 #define KVM_S390_MEMOP_LOGICAL_READ	0
@@ -1010,6 +1013,7 @@ struct kvm_ppc_resize_hpt {
 #define KVM_CAP_ARM_NISV_TO_USER 177
 #define KVM_CAP_ARM_INJECT_EXT_DABT 178
 #define KVM_CAP_S390_VCPU_RESETS 179
+#define KVM_CAP_S390_PROTECTED 180
 
 #ifdef KVM_CAP_IRQ_ROUTING
 

linux-headers/linux/kvm.h struct kvm_enc_region { · line 1483

Quoted 8 lines
linux-headers/linux/kvm.h +3 -2
@@ -1474,8 +1478,42 @@ struct kvm_enc_region {
 /* Available with KVM_CAP_ARM_SVE */
 #define KVM_ARM_VCPU_FINALIZE	  _IOW(KVMIO,  0xc2, int)
 
-#define KVM_S390_NORMAL_RESET	  _IO(KVMIO,  0xc3)
-#define KVM_S390_CLEAR_RESET	  _IO(KVMIO,  0xc4)
+/* Available with  KVM_CAP_S390_VCPU_RESETS */
+#define KVM_S390_NORMAL_RESET	_IO(KVMIO,   0xc3)
+#define KVM_S390_CLEAR_RESET	_IO(KVMIO,   0xc4)

Looks unrelated, but if that's part of the sync ...

Signature · 3 lines
Thanks,

David / dhildenb
#3 lore raw

Re: [PATCH v4 01/16] Sync pv

Quoted 36 lines · David Hildenbrand
Quoted 35 lines · Janosch Frank
---
 linux-headers/linux/kvm.h | 46 +++++++++++++++++++++++++++++++++++----
 1 file changed, 42 insertions(+), 4 deletions(-)
linux-headers/linux/kvm.h +6 -2
index ec146bd52a..2e647f2d9b 100644
@@ -474,8 +474,11 @@ struct kvm_s390_mem_op {
 	__u32 size;		/* amount of bytes */
 	__u32 op;		/* type of operation */
 	__u64 buf;		/* buffer in userspace */
-	__u8 ar;		/* the access register number */
-	__u8 reserved[31];	/* should be set to 0 */
+	union {
+		__u8 ar;	/* the access register number */
+		__u32 sida_offset; /* offset into the sida */
+		__u8 reserved[32]; /* should be set to 0 */
+	};
 };
 /* types for kvm_s390_mem_op->op */
 #define KVM_S390_MEMOP_LOGICAL_READ	0
@@ -1010,6 +1013,7 @@ struct kvm_ppc_resize_hpt {
 #define KVM_CAP_ARM_NISV_TO_USER 177
 #define KVM_CAP_ARM_INJECT_EXT_DABT 178
 #define KVM_CAP_S390_VCPU_RESETS 179
+#define KVM_CAP_S390_PROTECTED 180
 
 #ifdef KVM_CAP_IRQ_ROUTING
 
@@ -1474,8 +1478,42 @@ struct kvm_enc_region {
 /* Available with KVM_CAP_ARM_SVE */
 #define KVM_ARM_VCPU_FINALIZE	  _IOW(KVMIO,  0xc2, int)
 
Quoted 8 lines

linux-headers/linux/kvm.h struct kvm_enc_region { · line 1483

Quoted 5 lines
linux-headers/linux/kvm.h +3 -2
-#define KVM_S390_NORMAL_RESET	  _IO(KVMIO,  0xc3)
-#define KVM_S390_CLEAR_RESET	  _IO(KVMIO,  0xc4)
+/* Available with  KVM_CAP_S390_VCPU_RESETS */
+#define KVM_S390_NORMAL_RESET	_IO(KVMIO,   0xc3)
+#define KVM_S390_CLEAR_RESET	_IO(KVMIO,   0xc4)

Looks unrelated, but if that's part of the sync ...

Yeah, that doesn't belong here and was already removed a few days ago.

#4 lore raw

PATCH v4 2/16 s390x: protvirt: Add diag308 subcodes 8 - 10

In reply to Janosch Frank

For diag308 subcodes 8 - 10 we have a new ipib of type 5. The ipib holds the address and length of the secure execution header, as well as a list of guest components.

Each component is a block of memory, for example kernel or initrd, which needs to be decrypted by the Ultravisor in order to run a protected VM. The secure execution header instructs the Ultravisor on how to handle the protected VM and its components.

Subcodes 8 and 9 are similiar to 5 and 6 and subcode 10 will finally start the protected guest.

Subcodes 8-10 are not valid in protected mode, we have to do a subcode 3 and then the 8 and 10 combination for a protected reboot.

---
 hw/s390x/ipl.c      | 48 ++++++++++++++++++++++++++++++++++++++++++---
 hw/s390x/ipl.h      | 31 +++++++++++++++++++++++++++++
 target/s390x/diag.c | 27 ++++++++++++++++++++++---
 3 files changed, 100 insertions(+), 6 deletions(-)
hw/s390x/ipl.c +45 -3
index 7773499d7f..e92d989813 100644
@@ -538,15 +538,56 @@ static bool is_virtio_scsi_device(IplParameterBlock *iplb)
     return is_virtio_ccw_device_of_type(iplb, VIRTIO_ID_SCSI);
 }
 
+int s390_ipl_pv_check_components(IplParameterBlock *iplb)
+{
+    int i;
+    IPLBlockPV *ipib_pv = &iplb->pv;
+
+    if (ipib_pv->num_comp == 0) {
+        return -EINVAL;
+    }
+
+    for (i = 0; i < ipib_pv->num_comp; i++) {
+
+        /* Addr must be 4k aligned */
+        if (ipib_pv->components[i].addr & ~TARGET_PAGE_MASK) {
+            return -EINVAL;
+        }
+
+        /* Tweak prefix is monotonously increasing with each component */
+        if (i < ipib_pv->num_comp - 1 &&
+            ipib_pv->components[i].tweak_pref >
+            ipib_pv->components[i + 1].tweak_pref) {
+            return -EINVAL;
+        }
+    }
+    return 1;
+}
+
 void s390_ipl_update_diag308(IplParameterBlock *iplb)
 {
     S390IPLState *ipl = get_ipl_device();
 
-    ipl->iplb = *iplb;
-    ipl->iplb_valid = true;
+    if (iplb->pbt == S390_IPL_TYPE_PV) {
+        ipl->iplb_pv = *iplb;
+        ipl->iplb_valid_pv = true;
+    } else {
+        ipl->iplb = *iplb;
+        ipl->iplb_valid = true;
+    }
     ipl->netboot = is_virtio_net_device(iplb);
 }
 
+IplParameterBlock *s390_ipl_get_iplb_secure(void)
+{
+    S390IPLState *ipl = get_ipl_device();
+
+    if (!ipl->iplb_valid_pv) {
+        return NULL;
+    }
+    return &ipl->iplb_pv;
+}
+
 IplParameterBlock *s390_ipl_get_iplb(void)
 {
     S390IPLState *ipl = get_ipl_device();
@@ -561,7 +602,8 @@ void s390_ipl_reset_request(CPUState *cs, enum s390_reset reset_type)
 {
     S390IPLState *ipl = get_ipl_device();
 
-    if (reset_type == S390_RESET_EXTERNAL || reset_type == S390_RESET_REIPL) {
+    if (reset_type == S390_RESET_EXTERNAL || reset_type == S390_RESET_REIPL ||
+        reset_type == S390_RESET_PV) {
         /* use CPU 0 for full resets */
         ipl->reset_cpu_index = 0;
     } else {
hw/s390x/ipl.h +31 -0
index d4813105db..3c4bb66eda 100644
@@ -15,6 +15,23 @@
 #include "cpu.h"
 #include "hw/qdev-core.h"
 
+struct IPLBlockPVComp {
+    uint64_t tweak_pref;
+    uint64_t addr;
+    uint64_t size;
+} QEMU_PACKED;
+typedef struct IPLBlockPVComp IPLBlockPVComp;
+
+struct IPLBlockPV {
+    uint8_t  reserved[87];
+    uint8_t  version;
+    uint32_t num_comp;
+    uint64_t pv_header_addr;
+    uint64_t pv_header_len;
+    struct IPLBlockPVComp components[];
+} QEMU_PACKED;
+typedef struct IPLBlockPV IPLBlockPV;
+
 struct IplBlockCcw {
     uint8_t  reserved0[85];
     uint8_t  ssid;
@@ -71,6 +88,7 @@ union IplParameterBlock {
         union {
             IplBlockCcw ccw;
             IplBlockFcp fcp;
+            IPLBlockPV pv;
             IplBlockQemuScsi scsi;
         };
     } QEMU_PACKED;
@@ -84,9 +102,11 @@ union IplParameterBlock {
 typedef union IplParameterBlock IplParameterBlock;
 
 int s390_ipl_set_loadparm(uint8_t *loadparm);
+int s390_ipl_pv_check_components(IplParameterBlock *iplb);
 void s390_ipl_update_diag308(IplParameterBlock *iplb);
 void s390_ipl_prepare_cpu(S390CPU *cpu);
 IplParameterBlock *s390_ipl_get_iplb(void);
+IplParameterBlock *s390_ipl_get_iplb_secure(void);
 
 enum s390_reset {
     /* default is a reset not triggered by a CPU e.g. issued by QMP */
@@ -94,6 +114,7 @@ enum s390_reset {
     S390_RESET_REIPL,
     S390_RESET_MODIFIED_CLEAR,
     S390_RESET_LOAD_NORMAL,
+    S390_RESET_PV,
 };
 void s390_ipl_reset_request(CPUState *cs, enum s390_reset reset_type);
 void s390_ipl_get_reset_request(CPUState **cs, enum s390_reset *reset_type);
@@ -133,6 +154,7 @@ struct S390IPLState {
     /*< private >*/
     DeviceState parent_obj;
     IplParameterBlock iplb;
+    IplParameterBlock iplb_pv;
     QemuIplParameters qipl;
     uint64_t start_addr;
     uint64_t compat_start_addr;
@@ -140,6 +162,7 @@ struct S390IPLState {
     uint64_t compat_bios_start_addr;
     bool enforce_bios;
     bool iplb_valid;
+    bool iplb_valid_pv;
     bool netboot;
     /* reset related properties don't have to be migrated or reset */
     enum s390_reset reset_type;
@@ -161,9 +184,11 @@ QEMU_BUILD_BUG_MSG(offsetof(S390IPLState, iplb) & 3, "alignment of iplb wrong");
 
 #define S390_IPL_TYPE_FCP 0x00
 #define S390_IPL_TYPE_CCW 0x02
+#define S390_IPL_TYPE_PV 0x05
 #define S390_IPL_TYPE_QEMU_SCSI 0xff
 
 #define S390_IPLB_HEADER_LEN 8
+#define S390_IPLB_MIN_PV_LEN 148
 #define S390_IPLB_MIN_CCW_LEN 200
 #define S390_IPLB_MIN_FCP_LEN 384
 #define S390_IPLB_MIN_QEMU_SCSI_LEN 200
@@ -185,4 +210,10 @@ static inline bool iplb_valid_fcp(IplParameterBlock *iplb)
            iplb->pbt == S390_IPL_TYPE_FCP;
 }
 
+static inline bool iplb_valid_pv(IplParameterBlock *iplb)
+{
+    return be32_to_cpu(iplb->len) >= S390_IPLB_MIN_PV_LEN &&
+           iplb->pbt == S390_IPL_TYPE_PV;
+}
+
 #endif
target/s390x/diag.c +24 -3
index b5aec06d6b..5e77d85652 100644
@@ -52,6 +52,8 @@ int handle_diag_288(CPUS390XState *env, uint64_t r1, uint64_t r3)
 #define DIAG_308_RC_OK              0x0001
 #define DIAG_308_RC_NO_CONF         0x0102
 #define DIAG_308_RC_INVALID         0x0402
+#define DIAG_308_RC_NO_PV_CONF      0x0a02
+#define DIAG_308_RC_INVAL_FOR_PV    0x0b02
 
 #define DIAG308_RESET_MOD_CLR       0
 #define DIAG308_RESET_LOAD_NORM     1
@@ -59,6 +61,9 @@ int handle_diag_288(CPUS390XState *env, uint64_t r1, uint64_t r3)
 #define DIAG308_LOAD_NORMAL_DUMP    4
 #define DIAG308_SET                 5
 #define DIAG308_STORE               6
+#define DIAG308_PV_SET              8
+#define DIAG308_PV_STORE            9
+#define DIAG308_PV_START            10
 
 static int diag308_parm_check(CPUS390XState *env, uint64_t r1, uint64_t addr,
                               uintptr_t ra, bool write)
@@ -105,6 +110,7 @@ void handle_diag_308(CPUS390XState *env, uint64_t r1, uint64_t r3, uintptr_t ra)
         s390_ipl_reset_request(cs, S390_RESET_REIPL);
         break;
     case DIAG308_SET:
+    case DIAG308_PV_SET:
         if (diag308_parm_check(env, r1, addr, ra, false)) {
             return;
         }
@@ -117,7 +123,8 @@ void handle_diag_308(CPUS390XState *env, uint64_t r1, uint64_t r3, uintptr_t ra)
 
         cpu_physical_memory_read(addr, iplb, be32_to_cpu(iplb->len));
 
-        if (!iplb_valid_ccw(iplb) && !iplb_valid_fcp(iplb)) {
+        if (!iplb_valid_ccw(iplb) && !iplb_valid_fcp(iplb) &&
+            !(iplb_valid_pv(iplb) && s390_ipl_pv_check_components(iplb) >= 0)) {
             env->regs[r1 + 1] = DIAG_308_RC_INVALID;
             goto out;
         }
@@ -128,17 +135,31 @@ out:
         g_free(iplb);
         return;
     case DIAG308_STORE:
+    case DIAG308_PV_STORE:
         if (diag308_parm_check(env, r1, addr, ra, true)) {
             return;
         }
-        iplb = s390_ipl_get_iplb();
+        if (subcode == DIAG308_PV_STORE) {
+            iplb = s390_ipl_get_iplb_secure();
+        } else {
+            iplb = s390_ipl_get_iplb();
+        }
         if (iplb) {
             cpu_physical_memory_write(addr, iplb, be32_to_cpu(iplb->len));
             env->regs[r1 + 1] = DIAG_308_RC_OK;
         } else {
             env->regs[r1 + 1] = DIAG_308_RC_NO_CONF;
         }
-        return;
+        break;
+    case DIAG308_PV_START:
+        iplb = s390_ipl_get_iplb_secure();
+        if (!iplb || !iplb_valid_pv(iplb)) {
+            env->regs[r1 + 1] = DIAG_308_RC_NO_PV_CONF;
+            return;
+        }
+
+        s390_ipl_reset_request(cs, S390_RESET_PV);
+        break;
     default:
         s390_program_interrupt(env, PGM_SPECIFICATION, ra);
         break;
Signature · 1 line
2.20.1
#5 lore raw

Re: [PATCH v4 02/16] s390x: protvirt: Add diag308 subcodes 8 - 10

Quoted 44 lines · On Thu, 20 Feb 2020 07:56:24 -0500 Janosch Frank <[email protected]> wrote:

For diag308 subcodes 8 - 10 we have a new ipib of type 5. The ipib holds the address and length of the secure execution header, as well as a list of guest components.

Each component is a block of memory, for example kernel or initrd, which needs to be decrypted by the Ultravisor in order to run a protected VM. The secure execution header instructs the Ultravisor on how to handle the protected VM and its components.

Subcodes 8 and 9 are similiar to 5 and 6 and subcode 10 will finally start the protected guest.

Subcodes 8-10 are not valid in protected mode, we have to do a subcode 3 and then the 8 and 10 combination for a protected reboot.

---
 hw/s390x/ipl.c      | 48 ++++++++++++++++++++++++++++++++++++++++++---
 hw/s390x/ipl.h      | 31 +++++++++++++++++++++++++++++
 target/s390x/diag.c | 27 ++++++++++++++++++++++---
 3 files changed, 100 insertions(+), 6 deletions(-)
hw/s390x/ipl.c +14 -0
index 7773499d7f..e92d989813 100644
@@ -538,15 +538,56 @@ static bool is_virtio_scsi_device(IplParameterBlock *iplb)
     return is_virtio_ccw_device_of_type(iplb, VIRTIO_ID_SCSI);
 }
 
+int s390_ipl_pv_check_components(IplParameterBlock *iplb)
+{
+    int i;
+    IPLBlockPV *ipib_pv = &iplb->pv;
+
+    if (ipib_pv->num_comp == 0) {
+        return -EINVAL;
+    }
+
+    for (i = 0; i < ipib_pv->num_comp; i++) {
+
+        /* Addr must be 4k aligned */
+        if (ipib_pv->components[i].addr & ~TARGET_PAGE_MASK) {
+            return -EINVAL;

hw/s390x/ipl.c static bool is_virtio_scsi_device(IplParameterBlock *iplb) · line 564

Quoted 10 lines
hw/s390x/ipl.c +10 -0
+        }
+
+        /* Tweak prefix is monotonously increasing with each component */
+        if (i < ipib_pv->num_comp - 1 &&
+            ipib_pv->components[i].tweak_pref >
+            ipib_pv->components[i + 1].tweak_pref) {
+            return -EINVAL;
+        }
+    }
+    return 1;

Any reason why you return 1 here? 0 vs negative error is the more usual pattern.

Quoted 2 lines
hw/s390x/ipl.c +2 -0
+}
+

(...)

hw/s390x/ipl.c void handle_diag_308(CPUS390XState *env, uint64_t r1, uint64_t r3, uintptr_t ra) · line 127

Quoted 7 lines
hw/s390x/ipl.c +2 -1
@@ -117,7 +123,8 @@ void handle_diag_308(CPUS390XState *env, uint64_t r1, uint64_t r3, uintptr_t ra)
 
         cpu_physical_memory_read(addr, iplb, be32_to_cpu(iplb->len));
 
-        if (!iplb_valid_ccw(iplb) && !iplb_valid_fcp(iplb)) {
+        if (!iplb_valid_ccw(iplb) && !iplb_valid_fcp(iplb) &&
+            !(iplb_valid_pv(iplb) && s390_ipl_pv_check_components(iplb) >= 0)) {

!s390_ipl_pv_check_components() would also read nicer IMHO :)

hw/s390x/ipl.c void handle_diag_308(CPUS390XState *env, uint64_t r1, uint64_t r3, uintptr_t ra) · line 130

Quoted 3 lines
hw/s390x/ipl.c +0 -0
             env->regs[r1 + 1] = DIAG_308_RC_INVALID;
             goto out;
         }

Otherwise, looks good to me.

#6 lore raw

Re: [PATCH v4 02/16] s390x: protvirt: Add diag308 subcodes 8 - 10

Quoted 52 lines · Cornelia Huck
Quoted 49 lines · On Thu, 20 Feb 2020 07:56:24 -0500 Janosch Frank <[email protected]> wrote:

For diag308 subcodes 8 - 10 we have a new ipib of type 5. The ipib holds the address and length of the secure execution header, as well as a list of guest components.

Each component is a block of memory, for example kernel or initrd, which needs to be decrypted by the Ultravisor in order to run a protected VM. The secure execution header instructs the Ultravisor on how to handle the protected VM and its components.

Subcodes 8 and 9 are similiar to 5 and 6 and subcode 10 will finally start the protected guest.

Subcodes 8-10 are not valid in protected mode, we have to do a subcode 3 and then the 8 and 10 combination for a protected reboot.

---
 hw/s390x/ipl.c      | 48 ++++++++++++++++++++++++++++++++++++++++++---
 hw/s390x/ipl.h      | 31 +++++++++++++++++++++++++++++
 target/s390x/diag.c | 27 ++++++++++++++++++++++---
 3 files changed, 100 insertions(+), 6 deletions(-)
hw/s390x/ipl.c +19 -0
index 7773499d7f..e92d989813 100644
@@ -538,15 +538,56 @@ static bool is_virtio_scsi_device(IplParameterBlock *iplb)
     return is_virtio_ccw_device_of_type(iplb, VIRTIO_ID_SCSI);
 }
 
+int s390_ipl_pv_check_components(IplParameterBlock *iplb)
+{
+    int i;
+    IPLBlockPV *ipib_pv = &iplb->pv;
+
+    if (ipib_pv->num_comp == 0) {
+        return -EINVAL;
+    }
+
+    for (i = 0; i < ipib_pv->num_comp; i++) {
+
+        /* Addr must be 4k aligned */
+        if (ipib_pv->components[i].addr & ~TARGET_PAGE_MASK) {
+            return -EINVAL;
+        }
+
+        /* Tweak prefix is monotonously increasing with each component */
+        if (i < ipib_pv->num_comp - 1 &&
+            ipib_pv->components[i].tweak_pref >
Quoted 8 lines

hw/s390x/ipl.c static bool is_virtio_scsi_device(IplParameterBlock *iplb) · line 564

Quoted 5 lines
hw/s390x/ipl.c +5 -0
+            ipib_pv->components[i + 1].tweak_pref) {
+            return -EINVAL;
+        }
+    }
+    return 1;

Any reason why you return 1 here? 0 vs negative error is the more usual pattern.

I honestly have no idea, will fix :)

Quoted 23 lines
Quoted 3 lines
hw/s390x/ipl.c +2 -0
+}
+

(...)

hw/s390x/ipl.c void handle_diag_308(CPUS390XState *env, uint64_t r1, uint64_t r3, uintptr_t ra) · line 127

Quoted 7 lines
hw/s390x/ipl.c +2 -1
@@ -117,7 +123,8 @@ void handle_diag_308(CPUS390XState *env, uint64_t r1, uint64_t r3, uintptr_t ra)
 
         cpu_physical_memory_read(addr, iplb, be32_to_cpu(iplb->len));
 
-        if (!iplb_valid_ccw(iplb) && !iplb_valid_fcp(iplb)) {
+        if (!iplb_valid_ccw(iplb) && !iplb_valid_fcp(iplb) &&
+            !(iplb_valid_pv(iplb) && s390_ipl_pv_check_components(iplb) >= 0)) {

!s390_ipl_pv_check_components() would also read nicer IMHO :)

hw/s390x/ipl.c void handle_diag_308(CPUS390XState *env, uint64_t r1, uint64_t r3, uintptr_t ra) · line 130

Quoted 3 lines
hw/s390x/ipl.c +0 -0
             env->regs[r1 + 1] = DIAG_308_RC_INVALID;
             goto out;
         }

Otherwise, looks good to me.

#7 lore raw

PATCH v4 3/16 s390x: protvirt: Support unpack facility

In reply to Janosch Frank

When a guest has saved a ipib of type 5 and calls diagnose308 with subcode 10, we have to setup the protected processing environment via Ultravisor calls. The calls are done by KVM and are exposed via an API.

The following steps are necessary:
1. Create a VM (register it with the Ultravisor)
2. Create secure CPUs for all of our current cpus
3. Forward the secure header to the Ultravisor (has all information on how to decrypt the image and VM information)
4. Protect image pages from the host and decrypt them
5. Verify the image integrity

Only after step 5 a protected VM is allowed to run.

to machine]

---
 hw/s390x/Makefile.objs              |   1 +
 hw/s390x/ipl.c                      |  33 +++++++++
 hw/s390x/ipl.h                      |   2 +
 hw/s390x/pv.c                       | 106 ++++++++++++++++++++++++++++
 hw/s390x/pv.h                       |  34 +++++++++
 hw/s390x/s390-virtio-ccw.c          |  87 +++++++++++++++++++++++
 include/hw/s390x/s390-virtio-ccw.h  |   1 +
 target/s390x/cpu.c                  |   4 ++
 target/s390x/cpu.h                  |   1 +
 target/s390x/cpu_features_def.inc.h |   1 +
 10 files changed, 270 insertions(+)
 create mode 100644 hw/s390x/pv.c
 create mode 100644 hw/s390x/pv.h
hw/s390x/Makefile.objs +1 -0
index e02ed80b68..a46a1c7894 100644
@@ -31,6 +31,7 @@ obj-y += tod-qemu.o
 obj-$(CONFIG_KVM) += tod-kvm.o
 obj-$(CONFIG_KVM) += s390-skeys-kvm.o
 obj-$(CONFIG_KVM) += s390-stattrib-kvm.o
+obj-$(CONFIG_KVM) += pv.o
 obj-y += s390-ccw.o
 obj-y += ap-device.o
 obj-y += ap-bridge.o
hw/s390x/ipl.c +33 -0
index e92d989813..7c27f50e13 100644
@@ -33,6 +33,7 @@
 #include "qemu/cutils.h"
 #include "qemu/option.h"
 #include "exec/exec-all.h"
+#include "pv.h"
 
 #define KERN_IMAGE_START                0x010000UL
 #define LINUX_MAGIC_ADDR                0x010008UL
@@ -677,6 +678,38 @@ static void s390_ipl_prepare_qipl(S390CPU *cpu)
     cpu_physical_memory_unmap(addr, len, 1, len);
 }
 
+int s390_ipl_prepare_pv_header(void)
+{
+    S390IPLState *ipl = get_ipl_device();
+    IPLBlockPV *ipib_pv = &ipl->iplb_pv.pv;
+    void *hdr = g_malloc(ipib_pv->pv_header_len);
+    int rc;
+
+    cpu_physical_memory_read(ipib_pv->pv_header_addr, hdr,
+                             ipib_pv->pv_header_len);
+    rc = s390_pv_set_sec_parms((uint64_t)hdr,
+                          ipib_pv->pv_header_len);
+    g_free(hdr);
+    return rc;
+}
+
+int s390_ipl_pv_unpack(void)
+{
+    int i, rc = 0;
+    S390IPLState *ipl = get_ipl_device();
+    IPLBlockPV *ipib_pv = &ipl->iplb_pv.pv;
+
+    for (i = 0; i < ipib_pv->num_comp; i++) {
+        rc = s390_pv_unpack(ipib_pv->components[i].addr,
+                            TARGET_PAGE_ALIGN(ipib_pv->components[i].size),
+                            ipib_pv->components[i].tweak_pref);
+        if (rc) {
+            break;
+        }
+    }
+    return rc;
+}
+
 void s390_ipl_prepare_cpu(S390CPU *cpu)
 {
     S390IPLState *ipl = get_ipl_device();
hw/s390x/ipl.h +2 -0
index 3c4bb66eda..92467738a7 100644
@@ -104,6 +104,8 @@ typedef union IplParameterBlock IplParameterBlock;
 int s390_ipl_set_loadparm(uint8_t *loadparm);
 int s390_ipl_pv_check_components(IplParameterBlock *iplb);
 void s390_ipl_update_diag308(IplParameterBlock *iplb);
+int s390_ipl_prepare_pv_header(void);
+int s390_ipl_pv_unpack(void);
 void s390_ipl_prepare_cpu(S390CPU *cpu);
 IplParameterBlock *s390_ipl_get_iplb(void);
 IplParameterBlock *s390_ipl_get_iplb_secure(void);
hw/s390x/pv.c new file +106 -0

106 lines not on this page · raw message

hw/s390x/pv.h new file +34 -0
new file mode 100644
index 0000000000..be949d6825
@@ -0,0 +1,34 @@
+/*
+ * Protected Virtualization header
+ *
+ * Copyright IBM Corp. 2020
+ * Author(s):
+ *  Janosch Frank <[email protected]>
+ *
+ * This work is licensed under the terms of the GNU GPL, version 2 or (at
+ * your option) any later version. See the COPYING file in the top-level
+ * directory.
+ */
+
+#ifndef HW_S390_PV_H
+#define HW_S390_PV_H
+
+#ifdef CONFIG_KVM
+int s390_pv_vm_create(void);
+void s390_pv_vm_destroy(void);
+int s390_pv_set_sec_parms(uint64_t origin, uint64_t length);
+int s390_pv_unpack(uint64_t addr, uint64_t size, uint64_t tweak);
+void s390_pv_perf_clear_reset(void);
+int s390_pv_verify(void);
+void s390_pv_unshare(void);
+#else
+int s390_pv_vm_create(void) { return 0; }
+void s390_pv_vm_destroy(void) {}
+int s390_pv_set_sec_parms(uint64_t origin, uint64_t length) { return 0; }
+int s390_pv_unpack(uint64_t addr, uint64_t size, uint64_t tweak) { return 0; }
+void s390_pv_perf_clear_reset(void) {}
+int s390_pv_verify(void) { return 0; }
+void s390_pv_unshare(void) {}
+#endif
+
+#endif /* HW_S390_PV_H */
hw/s390x/s390-virtio-ccw.c +87 -0

124 lines not on this page · raw message

include/hw/s390x/s390-virtio-ccw.h +1 -0
index 8aa27199c9..cd1dccc6e3 100644
@@ -28,6 +28,7 @@ typedef struct S390CcwMachineState {
     /*< public >*/
     bool aes_key_wrap;
     bool dea_key_wrap;
+    bool pv;
     uint8_t loadparm[8];
 } S390CcwMachineState;
 
target/s390x/cpu.c +4 -0
index 8da1905485..1dbd84b9d7 100644
@@ -37,6 +37,8 @@
 #include "sysemu/hw_accel.h"
 #include "hw/qdev-properties.h"
 #ifndef CONFIG_USER_ONLY
+#include "hw/s390x/s390-virtio-ccw.h"
+#include "hw/s390x/pv.h"
 #include "hw/boards.h"
 #include "sysemu/arch_init.h"
 #include "sysemu/sysemu.h"
@@ -191,6 +193,7 @@ static void s390_cpu_realizefn(DeviceState *dev, Error **errp)
 
 #if !defined(CONFIG_USER_ONLY)
     MachineState *ms = MACHINE(qdev_get_machine());
+    S390CcwMachineState *ccw = S390_CCW_MACHINE(ms);
     unsigned int max_cpus = ms->smp.max_cpus;
     if (cpu->env.core_id >= max_cpus) {
         error_setg(&err, "Unable to add CPU with core-id: %" PRIu32
@@ -205,6 +208,7 @@ static void s390_cpu_realizefn(DeviceState *dev, Error **errp)
         goto out;
     }
 
+    cpu->env.pv = ccw->pv;
     /* sync cs->cpu_index and env->core_id. The latter is needed for TCG. */
     cs->cpu_index = cpu->env.core_id;
 #endif
target/s390x/cpu.h +1 -0
index 8a557fd8d1..cbc53c99cf 100644
@@ -114,6 +114,7 @@ struct CPUS390XState {
 
     /* Fields up to this point are cleared by a CPU reset */
     struct {} end_reset_fields;
+    bool pv; /* protected virtualization */
 
 #if !defined(CONFIG_USER_ONLY)
     uint32_t core_id; /* PoP "CPU address", same as cpu_index */
target/s390x/cpu_features_def.inc.h +1 -0
index 31dff0d84e..60db28351d 100644
@@ -107,6 +107,7 @@ DEF_FEAT(DEFLATE_BASE, "deflate-base", STFL, 151, "Deflate-conversion facility (
 DEF_FEAT(VECTOR_PACKED_DECIMAL_ENH, "vxpdeh", STFL, 152, "Vector-Packed-Decimal-Enhancement Facility")
 DEF_FEAT(MSA_EXT_9, "msa9-base", STFL, 155, "Message-security-assist-extension-9 facility (excluding subfunctions)")
 DEF_FEAT(ETOKEN, "etoken", STFL, 156, "Etoken facility")
+DEF_FEAT(UNPACK, "unpack", STFL, 161, "Unpack facility")
 
 /* Features exposed via SCLP SCCB Byte 80 - 98  (bit numbers relative to byte-80) */
 DEF_FEAT(SIE_GSLS, "gsls", SCLP_CONF_CHAR, 40, "SIE: Guest-storage-limit-suppression facility")
Signature · 1 line
2.20.1
#8 lore raw

Re: [PATCH v4 03/16] s390x: protvirt: Support unpack facility

Quoted 8 lines · Janosch Frank

When a guest has saved a ipib of type 5 and calls diagnose308 with subcode 10, we have to setup the protected processing environment via Ultravisor calls. The calls are done by KVM and are exposed via an API.

The following steps are necessary:
1. Create a VM (register it with the Ultravisor)
2. Create secure CPUs for all of our current cpus

maybe rephrase that to reflect the enable/disable change?

Quoted 10 lines

3. Forward the secure header to the Ultravisor (has all information on how to decrypt the image and VM information)
4. Protect image pages from the host and decrypt them
5. Verify the image integrity

Only after step 5 a protected VM is allowed to run.

to machine]

[...]

Quoted 19 lines
hw/s390x/pv.c new file +13 -0
new file mode 100644
index 0000000000..8ae20f31c1
@@ -0,0 +1,106 @@
+/*
+ * Secure execution functions
+ *
+ * Copyright IBM Corp. 2020
+ * Author(s):
+ *  Janosch Frank <[email protected]>
+ *
+ * This work is licensed under the terms of the GNU GPL, version 2 or (at
+ * your option) any later version. See the COPYING file in the top-level
+ * directory.
+ */
+#include "qemu/osdep.h"
+#include <sys/ioctl.h>

hw/s390x/pv.c line 23

Quoted 10 lines
hw/s390x/pv.c new file +10 -0
+
+#include <linux/kvm.h>
+
+#include "qemu/error-report.h"
+#include "sysemu/kvm.h"
+#include "pv.h"
+
+const char *cmd_names[] = {
+    "VM_CREATE",
+    "VM_DESTROY",

ENABLE/DISABLE?

hw/s390x/pv.c line 30

Quoted 7 lines
hw/s390x/pv.c new file +7 -0
+    "VM_SET_SEC_PARAMS",
+    "VM_UNPACK",
+    "VM_VERIFY",
+    "VM_PREP_RESET",
+    "VM_UNSHARE_ALL",
+    NULL
+};

Wild idea. Not sure if this is acceptable in QEMU.

These strings are only for the error_report, no?
Could we use a MACRO that uses this a as a string and b as the inter value?

-static int s390_pv_cmd(uint32_t cmd, void *data) +#define S390_PV_CMD(cmd, data) s390_pv_cmd(cmd, #cmd, data) +static int s390_pv_cmd(uint32_t cmd, const char *name, void *data)

and then replace all calls with S390_PV_CMD. (Of course also change the error report to use name).

Quoted 7 lines
hw/s390x/pv.c new file +7 -0
+
+static int s390_pv_cmd(uint32_t cmd, void *data)
+{
+    int rc;
+    struct kvm_pv_cmd pv_cmd = {
+        .cmd = cmd,
+        .data = (uint64_t)data,

hw/s390x/pv.c line 47

Quoted 10 lines
hw/s390x/pv.c new file +10 -0
+    };
+
+    rc = kvm_vm_ioctl(kvm_state, KVM_S390_PV_COMMAND, &pv_cmd);
+    if (rc) {
+        error_report("KVM PV command %d (%s) failed: header rc %x rrc %x "
+                     "IOCTL rc: %d", cmd, cmd_names[cmd], pv_cmd.rc, pv_cmd.rrc,
+                     rc);
+    }
+    return rc;
+}

[...]

Maybe also change the subject. We do more than just unpack.

#9 lore raw

Re: [PATCH v4 03/16] s390x: protvirt: Support unpack facility

Quoted 8 lines · Christian Borntraeger
Quoted 5 lines · Janosch Frank

When a guest has saved a ipib of type 5 and calls diagnose308 with subcode 10, we have to setup the protected processing environment via Ultravisor calls. The calls are done by KVM and are exposed via an API.

Quoted 5 lines
Quoted 3 lines

The following steps are necessary:
1. Create a VM (register it with the Ultravisor)
2. Create secure CPUs for all of our current cpus

maybe rephrase that to reflect the enable/disable change?

Yep

Quoted 37 lines
Quoted 11 lines

3. Forward the secure header to the Ultravisor (has all information on how to decrypt the image and VM information)
4. Protect image pages from the host and decrypt them
5. Verify the image integrity

Only after step 5 a protected VM is allowed to run.

to machine]

[...]

Quoted 23 lines
hw/s390x/pv.c new file +17 -0
new file mode 100644
index 0000000000..8ae20f31c1
@@ -0,0 +1,106 @@
+/*
+ * Secure execution functions
+ *
+ * Copyright IBM Corp. 2020
+ * Author(s):
+ *  Janosch Frank <[email protected]>
+ *
+ * This work is licensed under the terms of the GNU GPL, version 2 or (at
+ * your option) any later version. See the COPYING file in the top-level
+ * directory.
+ */
+#include "qemu/osdep.h"
+#include <sys/ioctl.h>
+
+#include <linux/kvm.h>
+
+#include "qemu/error-report.h"
Quoted 8 lines

hw/s390x/pv.c line 23

Quoted 6 lines
hw/s390x/pv.c new file +6 -0
+#include "sysemu/kvm.h"
+#include "pv.h"
+
+const char *cmd_names[] = {
+    "VM_CREATE",
+    "VM_DESTROY",

ENABLE/DISABLE?

Ack.

Quoted 41 lines

hw/s390x/pv.c line 30

Quoted 8 lines
hw/s390x/pv.c new file +7 -0
+    "VM_SET_SEC_PARAMS",
+    "VM_UNPACK",
+    "VM_VERIFY",
+    "VM_PREP_RESET",
+    "VM_UNSHARE_ALL",
+    NULL
+};

Wild idea. Not sure if this is acceptable in QEMU.

These strings are only for the error_report, no?
Could we use a MACRO that uses this a as a string and b as the inter value?

(quoted diff) +2 -1
-static int s390_pv_cmd(uint32_t cmd, void *data)
+#define S390_PV_CMD(cmd, data) s390_pv_cmd(cmd, #cmd, data)
+static int s390_pv_cmd(uint32_t cmd, const char *name, void *data)

and then replace all calls with S390_PV_CMD. (Of course also change the error report to use name).

Quoted 17 lines
hw/s390x/pv.c new file +17 -0
+
+static int s390_pv_cmd(uint32_t cmd, void *data)
+{
+    int rc;
+    struct kvm_pv_cmd pv_cmd = {
+        .cmd = cmd,
+        .data = (uint64_t)data,
+    };
+
+    rc = kvm_vm_ioctl(kvm_state, KVM_S390_PV_COMMAND, &pv_cmd);
+    if (rc) {
+        error_report("KVM PV command %d (%s) failed: header rc %x rrc %x "
+                     "IOCTL rc: %d", cmd, cmd_names[cmd], pv_cmd.rc, pv_cmd.rrc,
+                     rc);
+    }
+    return rc;
+}

[...]

Quoted 2 lines

Maybe also change the subject. We do more than just unpack.

The name is not about unpack, it's about the unpack facility

#10 lore raw

PATCH v4 4/16 s390x: protvirt: Add migration blocker

In reply to Janosch Frank

Migration is not yet supported.

---
 hw/s390x/s390-virtio-ccw.c | 17 ++++++++++++++---
 1 file changed, 14 insertions(+), 3 deletions(-)
hw/s390x/s390-virtio-ccw.c +14 -3
index aa974d294e..6fba8d9331 100644
@@ -42,6 +42,9 @@
 #include "hw/s390x/tod.h"
 #include "sysemu/sysemu.h"
 #include "hw/s390x/pv.h"
+#include "migration/blocker.h"
+
+static Error *pv_mig_blocker;
 
 S390CPU *s390_cpu_addr2state(uint16_t cpu_addr)
 {
@@ -387,6 +390,7 @@ static void s390_machine_reset(MachineState *machine)
     CPUState *cs, *t;
     S390CPU *cpu;
     S390CcwMachineState *ms = S390_CCW_MACHINE(machine);
+    static Error *local_err;
 
     /* get the reset parameters, reset them once done */
     s390_ipl_get_reset_request(&cs, &reset_type);
@@ -436,13 +440,20 @@ static void s390_machine_reset(MachineState *machine)
         }
         run_on_cpu(cs, s390_do_cpu_reset, RUN_ON_CPU_NULL);
 
-        if (s390_machine_protect(ms)) {
+        if (!pv_mig_blocker) {
+            error_setg(&pv_mig_blocker,
+                       "protected VMs are currently not migrateable.");
+        }
+        migrate_add_blocker(pv_mig_blocker, &local_err);
+        if (!local_err && s390_machine_protect(ms)) {
             s390_machine_inject_pv_error(cs);
-            s390_cpu_set_state(S390_CPU_STATE_OPERATING, cpu);
-            return;
+            migrate_del_blocker(pv_mig_blocker);
+            goto pv_err;
         }
 
         run_on_cpu(cs, s390_do_cpu_load_normal, RUN_ON_CPU_NULL);
+pv_err:
+        s390_cpu_set_state(S390_CPU_STATE_OPERATING, cpu);
         break;
     default:
         g_assert_not_reached();
Signature · 1 line
2.20.1
#11 lore raw

Re: [PATCH v4 04/16] s390x: protvirt: Add migration blocker

Quoted 36 lines · Janosch Frank

Migration is not yet supported.

---
 hw/s390x/s390-virtio-ccw.c | 17 ++++++++++++++---
 1 file changed, 14 insertions(+), 3 deletions(-)
hw/s390x/s390-virtio-ccw.c +6 -1
index aa974d294e..6fba8d9331 100644
@@ -42,6 +42,9 @@
 #include "hw/s390x/tod.h"
 #include "sysemu/sysemu.h"
 #include "hw/s390x/pv.h"
+#include "migration/blocker.h"
+
+static Error *pv_mig_blocker;
 
 S390CPU *s390_cpu_addr2state(uint16_t cpu_addr)
 {
@@ -387,6 +390,7 @@ static void s390_machine_reset(MachineState *machine)
     CPUState *cs, *t;
     S390CPU *cpu;
     S390CcwMachineState *ms = S390_CCW_MACHINE(machine);
+    static Error *local_err;
 
     /* get the reset parameters, reset them once done */
     s390_ipl_get_reset_request(&cs, &reset_type);
@@ -436,13 +440,20 @@ static void s390_machine_reset(MachineState *machine)
         }
         run_on_cpu(cs, s390_do_cpu_reset, RUN_ON_CPU_NULL);
 
-        if (s390_machine_protect(ms)) {
+        if (!pv_mig_blocker) {
+            error_setg(&pv_mig_blocker,

hw/s390x/s390-virtio-ccw.c static void s390_machine_reset(MachineState *machine) · line 452

Quoted 10 lines
hw/s390x/s390-virtio-ccw.c +6 -2
+                       "protected VMs are currently not migrateable.");
+        }
+        migrate_add_blocker(pv_mig_blocker, &local_err);
+        if (!local_err && s390_machine_protect(ms)) {
             s390_machine_inject_pv_error(cs);
-            s390_cpu_set_state(S390_CPU_STATE_OPERATING, cpu);
-            return;
+            migrate_del_blocker(pv_mig_blocker);
+            goto pv_err;
         }

I'm sorry, but that looks ... a little to interesting to me. Why can't you register/unregister the blocker where you actually perform the switch to/from PV?

Quoted 8 lines
hw/s390x/s390-virtio-ccw.c +2 -0
         run_on_cpu(cs, s390_do_cpu_load_normal, RUN_ON_CPU_NULL);
+pv_err:
+        s390_cpu_set_state(S390_CPU_STATE_OPERATING, cpu);
         break;
     default:
         g_assert_not_reached();
 
Signature · 3 lines
Thanks,

David / dhildenb
#12 lore raw

Re: [PATCH v4 04/16] s390x: protvirt: Add migration blocker

Quoted 43 lines · David Hildenbrand
Quoted 42 lines · Janosch Frank

Migration is not yet supported.

---
 hw/s390x/s390-virtio-ccw.c | 17 ++++++++++++++---
 1 file changed, 14 insertions(+), 3 deletions(-)
hw/s390x/s390-virtio-ccw.c +10 -2
index aa974d294e..6fba8d9331 100644
@@ -42,6 +42,9 @@
 #include "hw/s390x/tod.h"
 #include "sysemu/sysemu.h"
 #include "hw/s390x/pv.h"
+#include "migration/blocker.h"
+
+static Error *pv_mig_blocker;
 
 S390CPU *s390_cpu_addr2state(uint16_t cpu_addr)
 {
@@ -387,6 +390,7 @@ static void s390_machine_reset(MachineState *machine)
     CPUState *cs, *t;
     S390CPU *cpu;
     S390CcwMachineState *ms = S390_CCW_MACHINE(machine);
+    static Error *local_err;
 
     /* get the reset parameters, reset them once done */
     s390_ipl_get_reset_request(&cs, &reset_type);
@@ -436,13 +440,20 @@ static void s390_machine_reset(MachineState *machine)
         }
         run_on_cpu(cs, s390_do_cpu_reset, RUN_ON_CPU_NULL);
 
-        if (s390_machine_protect(ms)) {
+        if (!pv_mig_blocker) {
+            error_setg(&pv_mig_blocker,
+                       "protected VMs are currently not migrateable.");
+        }
+        migrate_add_blocker(pv_mig_blocker, &local_err);
+        if (!local_err && s390_machine_protect(ms)) {
             s390_machine_inject_pv_error(cs);
-            s390_cpu_set_state(S390_CPU_STATE_OPERATING, cpu);
Quoted 8 lines

hw/s390x/s390-virtio-ccw.c static void s390_machine_reset(MachineState *machine) · line 452

Quoted 4 lines
hw/s390x/s390-virtio-ccw.c +2 -1
-            return;
+            migrate_del_blocker(pv_mig_blocker);
+            goto pv_err;
         }

I'm sorry, but that looks ... a little to interesting to me. Why can't you register/unregister the blocker where you actually perform the switch to/from PV?

The sprinkled over PV lines annoyed me anyway, so I added a unprotect() function yesterday and now also moved the migration blocker into protect() and unprotect()

Quoted 11 lines
Quoted 9 lines
hw/s390x/s390-virtio-ccw.c +2 -0
         run_on_cpu(cs, s390_do_cpu_load_normal, RUN_ON_CPU_NULL);
+pv_err:
+        s390_cpu_set_state(S390_CPU_STATE_OPERATING, cpu);
         break;
     default:
         g_assert_not_reached();
 
#13 lore raw

PATCH v4 5/16 s390x: protvirt: Handle diag 308 subcodes 0,1,3,4

In reply to Janosch Frank

As we now have access to the protection state of the cpus, we can implement special handling of diag 308 subcodes for cpus in the protected state.

For subcodes 0 and 1 we need to unshare all pages before continuing, so the guest doesn't accidentally expose data when dumping.

For subcode 3/4 we tear down the protected VM and reboot into unprotected mode. We do not provide a secure reboot.

Before we can do the unshare calls, we need to mark all cpus as stopped.

---
 hw/s390x/s390-virtio-ccw.c | 37 ++++++++++++++++++++++++++++++++++---
 target/s390x/diag.c        |  4 ++++
 2 files changed, 38 insertions(+), 3 deletions(-)
hw/s390x/s390-virtio-ccw.c +34 -3
index 6fba8d9331..48dd3170ab 100644
@@ -384,6 +384,20 @@ static void s390_machine_inject_pv_error(CPUState *cs)
     env->regs[r1 + 1] = 0xa02;
 }
 
+static void s390_pv_prepare_reset(CPUS390XState *env)
+{
+    CPUState *cs;
+
+    if (!env->pv) {
+        return;
+    }
+    CPU_FOREACH(cs) {
+        s390_cpu_set_state(S390_CPU_STATE_STOPPED, S390_CPU(cs));
+    }
+    s390_pv_unshare();
+    s390_pv_perf_clear_reset();
+}
+
 static void s390_machine_reset(MachineState *machine)
 {
     enum s390_reset reset_type;
@@ -391,6 +405,7 @@ static void s390_machine_reset(MachineState *machine)
     S390CPU *cpu;
     S390CcwMachineState *ms = S390_CCW_MACHINE(machine);
     static Error *local_err;
+    CPUS390XState *env;
 
     /* get the reset parameters, reset them once done */
     s390_ipl_get_reset_request(&cs, &reset_type);
@@ -399,10 +414,16 @@ static void s390_machine_reset(MachineState *machine)
     s390_cmma_reset();
 
     cpu = S390_CPU(cs);
+    env = &cpu->env;
 
     switch (reset_type) {
     case S390_RESET_EXTERNAL:
     case S390_RESET_REIPL:
+        if (ms->pv) {
+            s390_machine_unprotect(ms);
+            migrate_del_blocker(pv_mig_blocker);
+        }
+
         qemu_devices_reset();
         s390_crypto_reset();
 
@@ -410,21 +431,31 @@ static void s390_machine_reset(MachineState *machine)
         run_on_cpu(cs, s390_do_cpu_ipl, RUN_ON_CPU_NULL);
         break;
     case S390_RESET_MODIFIED_CLEAR:
+        /*
+         * Susbsystem reset needs to be done before we unshare memory
+         * and loose access to VIRTIO structures in guest memory.
+         */
+        subsystem_reset();
+        s390_crypto_reset();
+        s390_pv_prepare_reset(env);
         CPU_FOREACH(t) {
             run_on_cpu(t, s390_do_cpu_full_reset, RUN_ON_CPU_NULL);
         }
-        subsystem_reset();
-        s390_crypto_reset();
         run_on_cpu(cs, s390_do_cpu_load_normal, RUN_ON_CPU_NULL);
         break;
     case S390_RESET_LOAD_NORMAL:
+        /*
+         * Susbsystem reset needs to be done before we unshare memory
+         * and loose access to VIRTIO structures in guest memory.
+         */
+        subsystem_reset();
+        s390_pv_prepare_reset(env);
         CPU_FOREACH(t) {
             if (t == cs) {
                 continue;
             }
             run_on_cpu(t, s390_do_cpu_reset, RUN_ON_CPU_NULL);
         }
-        subsystem_reset();
         run_on_cpu(cs, s390_do_cpu_initial_reset, RUN_ON_CPU_NULL);
         run_on_cpu(cs, s390_do_cpu_load_normal, RUN_ON_CPU_NULL);
         break;
target/s390x/diag.c +4 -0
index 5e77d85652..aedc774695 100644
@@ -68,6 +68,10 @@ int handle_diag_288(CPUS390XState *env, uint64_t r1, uint64_t r3)
 static int diag308_parm_check(CPUS390XState *env, uint64_t r1, uint64_t addr,
                               uintptr_t ra, bool write)
 {
+    /* Handled by the Ultravisor */
+    if (env->pv) {
+        return 0;
+    }
     if ((r1 & 1) || (addr & ~TARGET_PAGE_MASK)) {
         s390_program_interrupt(env, PGM_SPECIFICATION, ra);
         return -1;
Signature · 1 line
2.20.1
#14 lore raw

PATCH v4 6/16 s390x: protvirt: KVM intercept changes

In reply to Janosch Frank

Secure guests no longer intercept with code 4 for an instruction interception. Instead they have codes 104 and 108 for secure instruction interception and secure instruction notification respectively.

The 104 mirrors the 4 interception.

The 108 is a notification interception to let KVM and QEMU know that something changed and we need to update tracking information or perform specific tasks. It's currently taken for the following instructions:

* stpx (To inform about the changed prefix location)
* sclp (On incorrect SCCB values, so we can inject a IRQ)
* sigp (All but "stop and store status")
* diag308 (Subcodes 0/1)

---
 target/s390x/kvm.c | 4 ++++
 1 file changed, 4 insertions(+)
target/s390x/kvm.c +4 -0
index 1d6fd6a27b..eec0b92479 100644
@@ -115,6 +115,8 @@
 #define ICPT_CPU_STOP                   0x28
 #define ICPT_OPEREXC                    0x2c
 #define ICPT_IO                         0x40
+#define ICPT_PV_INSTR                   0x68
+#define ICPT_PV_INSTR_NOTIFICATION      0x6c
 
 #define NR_LOCAL_IRQS 32
 /*
@@ -1693,6 +1695,8 @@ static int handle_intercept(S390CPU *cpu)
             (long)cs->kvm_run->psw_addr);
     switch (icpt_code) {
         case ICPT_INSTRUCTION:
+        case ICPT_PV_INSTR:
+        case ICPT_PV_INSTR_NOTIFICATION:
             r = handle_instruction(cpu, run);
             break;
         case ICPT_PROGRAM:
Signature · 1 line
2.20.1
Reviewed-by on v4 6/16
#15 lore raw

Re: [PATCH v4 06/16] s390x: protvirt: KVM intercept changes

Quoted 37 lines · Janosch Frank

Secure guests no longer intercept with code 4 for an instruction interception. Instead they have codes 104 and 108 for secure instruction interception and secure instruction notification respectively.

The 104 mirrors the 4 interception.

The 108 is a notification interception to let KVM and QEMU know that something changed and we need to update tracking information or perform specific tasks. It's currently taken for the following instructions:

* stpx (To inform about the changed prefix location)
* sclp (On incorrect SCCB values, so we can inject a IRQ)
* sigp (All but "stop and store status")
* diag308 (Subcodes 0/1)

---
 target/s390x/kvm.c | 4 ++++
 1 file changed, 4 insertions(+)
target/s390x/kvm.c +2 -0
index 1d6fd6a27b..eec0b92479 100644
@@ -115,6 +115,8 @@
 #define ICPT_CPU_STOP                   0x28
 #define ICPT_OPEREXC                    0x2c
 #define ICPT_IO                         0x40
+#define ICPT_PV_INSTR                   0x68
+#define ICPT_PV_INSTR_NOTIFICATION      0x6c
 
 #define NR_LOCAL_IRQS 32
 /*

target/s390x/kvm.c static int handle_intercept(S390CPU *cpu) · line 1702

Quoted 8 lines
target/s390x/kvm.c +2 -0
@@ -1693,6 +1695,8 @@ static int handle_intercept(S390CPU *cpu)
             (long)cs->kvm_run->psw_addr);
     switch (icpt_code) {
         case ICPT_INSTRUCTION:
+        case ICPT_PV_INSTR:
+        case ICPT_PV_INSTR_NOTIFICATION:
             r = handle_instruction(cpu, run);
             break;
         case ICPT_PROGRAM:
Signature · 3 lines
Thanks,

David / dhildenb
#16 lore raw

Re: [PATCH v4 06/16] s390x: protvirt: KVM intercept changes

Quoted 34 lines · David Hildenbrand
Quoted 33 lines · Janosch Frank

Secure guests no longer intercept with code 4 for an instruction interception. Instead they have codes 104 and 108 for secure instruction interception and secure instruction notification respectively.

The 104 mirrors the 4 interception.

The 108 is a notification interception to let KVM and QEMU know that something changed and we need to update tracking information or perform specific tasks. It's currently taken for the following instructions:

* stpx (To inform about the changed prefix location)
* sclp (On incorrect SCCB values, so we can inject a IRQ)
* sigp (All but "stop and store status")
* diag308 (Subcodes 0/1)

---
 target/s390x/kvm.c | 4 ++++
 1 file changed, 4 insertions(+)
target/s390x/kvm.c +2 -0
index 1d6fd6a27b..eec0b92479 100644
@@ -115,6 +115,8 @@
 #define ICPT_CPU_STOP                   0x28
 #define ICPT_OPEREXC                    0x2c
 #define ICPT_IO                         0x40
+#define ICPT_PV_INSTR                   0x68
+#define ICPT_PV_INSTR_NOTIFICATION      0x6c
 
Quoted 15 lines

target/s390x/kvm.c static int handle_intercept(S390CPU *cpu) · line 1702

Quoted 12 lines
target/s390x/kvm.c +2 -0
 #define NR_LOCAL_IRQS 32
 /*
@@ -1693,6 +1695,8 @@ static int handle_intercept(S390CPU *cpu)
             (long)cs->kvm_run->psw_addr);
     switch (icpt_code) {
         case ICPT_INSTRUCTION:
+        case ICPT_PV_INSTR:
+        case ICPT_PV_INSTR_NOTIFICATION:
             r = handle_instruction(cpu, run);
             break;
         case ICPT_PROGRAM:

Thanks!

#17 lore raw

PATCH v4 7/16 s390x: Add SIDA memory ops

In reply to Janosch Frank

Protected guests save the instruction control blocks in the SIDA instead of QEMU/KVM directly accessing the guest's memory.

Let's introduce new functions to access the SIDA.

---
 linux-headers/linux/kvm.h |  2 ++
 target/s390x/cpu.h        |  7 ++++++-
 target/s390x/kvm.c        | 23 +++++++++++++++++++++++
 target/s390x/kvm_s390x.h  |  2 ++
 target/s390x/mmu_helper.c |  9 +++++++++
 5 files changed, 42 insertions(+), 1 deletion(-)
linux-headers/linux/kvm.h +2 -0
index 2e647f2d9b..7ccf5988d2 100644
@@ -483,6 +483,8 @@ struct kvm_s390_mem_op {
 /* types for kvm_s390_mem_op->op */
 #define KVM_S390_MEMOP_LOGICAL_READ	0
 #define KVM_S390_MEMOP_LOGICAL_WRITE	1
+#define KVM_S390_MEMOP_SIDA_READ	2
+#define KVM_S390_MEMOP_SIDA_WRITE	3
 /* flags for kvm_s390_mem_op->flags */
 #define KVM_S390_MEMOP_F_CHECK_ONLY		(1ULL << 0)
 #define KVM_S390_MEMOP_F_INJECT_EXCEPTION	(1ULL << 1)
target/s390x/cpu.h +6 -1
index cbc53c99cf..491d6860a8 100644
@@ -823,7 +823,12 @@ int s390_cpu_virt_mem_rw(S390CPU *cpu, vaddr laddr, uint8_t ar, void *hostbuf,
 #define s390_cpu_virt_mem_check_write(cpu, laddr, ar, len)   \
         s390_cpu_virt_mem_rw(cpu, laddr, ar, NULL, len, true)
 void s390_cpu_virt_mem_handle_exc(S390CPU *cpu, uintptr_t ra);
-
+int s390_cpu_pv_mem_rw(S390CPU *cpu, unsigned int offset,  void *hostbuf,
+                       int len, bool is_write);
+#define s390_cpu_pv_mem_read(cpu, offset, dest, len)    \
+        s390_cpu_pv_mem_rw(cpu, offset, dest, len, false)
+#define s390_cpu_pv_mem_write(cpu, offset, dest, len)       \
+        s390_cpu_pv_mem_rw(cpu, offset, dest, len, true)
 
 /* sigp.c */
 int s390_cpu_restart(S390CPU *cpu);
target/s390x/kvm.c +23 -0
index eec0b92479..f222836df5 100644
@@ -846,6 +846,29 @@ int kvm_s390_mem_op(S390CPU *cpu, vaddr addr, uint8_t ar, void *hostbuf,
     return ret;
 }
 
+int kvm_s390_mem_op_pv(S390CPU *cpu, uint64_t offset, void *hostbuf,
+                       int len, bool is_write)
+{
+    int ret = 0;
+    struct kvm_s390_mem_op mem_op = {
+        .sida_offset = offset,
+        .size = len,
+        .op = is_write ? KVM_S390_MEMOP_SIDA_WRITE
+                       : KVM_S390_MEMOP_SIDA_READ,
+        .buf = (uint64_t)hostbuf,
+    };
+
+    if (!cap_mem_op) {
+        return -ENOSYS;
+    }
+
+    ret = kvm_vcpu_ioctl(CPU(cpu), KVM_S390_MEM_OP, &mem_op);
+    if (ret < 0) {
+        warn_report("KVM_S390_MEM_OP failed: %s", strerror(-ret));
+    }
+    return ret;
+}
+
 /*
  * Legacy layout for s390:
  * Older S390 KVM requires the topmost vma of the RAM to be
target/s390x/kvm_s390x.h +2 -0
index 0b21789796..9c38f6ccce 100644
@@ -19,6 +19,8 @@ void kvm_s390_vcpu_interrupt(S390CPU *cpu, struct kvm_s390_irq *irq);
 void kvm_s390_access_exception(S390CPU *cpu, uint16_t code, uint64_t te_code);
 int kvm_s390_mem_op(S390CPU *cpu, vaddr addr, uint8_t ar, void *hostbuf,
                     int len, bool is_write);
+int kvm_s390_mem_op_pv(S390CPU *cpu, vaddr addr, void *hostbuf, int len,
+                       bool is_write);
 void kvm_s390_program_interrupt(S390CPU *cpu, uint16_t code);
 int kvm_s390_set_cpu_state(S390CPU *cpu, uint8_t cpu_state);
 void kvm_s390_vcpu_interrupt_pre_save(S390CPU *cpu);
target/s390x/mmu_helper.c +9 -0
index c9f3f34750..ad485399db 100644
@@ -474,6 +474,15 @@ static int translate_pages(S390CPU *cpu, vaddr addr, int nr_pages,
     return 0;
 }
 
+int s390_cpu_pv_mem_rw(S390CPU *cpu, unsigned int offset, void *hostbuf,
+                       int len, bool is_write)
+{
+    int ret;
+
+    ret = kvm_s390_mem_op_pv(cpu, offset, hostbuf, len, is_write);
+    return ret;
+}
+
 /**
  * s390_cpu_virt_mem_rw:
  * @laddr:     the logical start address
Signature · 1 line
2.20.1
#18 lore raw

Re: [PATCH v4 07/16] s390x: Add SIDA memory ops

Quoted 50 lines · Janosch Frank

Protected guests save the instruction control blocks in the SIDA instead of QEMU/KVM directly accessing the guest's memory.

Let's introduce new functions to access the SIDA.

---
 linux-headers/linux/kvm.h |  2 ++
 target/s390x/cpu.h        |  7 ++++++-
 target/s390x/kvm.c        | 23 +++++++++++++++++++++++
 target/s390x/kvm_s390x.h  |  2 ++
 target/s390x/mmu_helper.c |  9 +++++++++
 5 files changed, 42 insertions(+), 1 deletion(-)
linux-headers/linux/kvm.h +2 -0
index 2e647f2d9b..7ccf5988d2 100644
@@ -483,6 +483,8 @@ struct kvm_s390_mem_op {
 /* types for kvm_s390_mem_op->op */
 #define KVM_S390_MEMOP_LOGICAL_READ	0
 #define KVM_S390_MEMOP_LOGICAL_WRITE	1
+#define KVM_S390_MEMOP_SIDA_READ	2
+#define KVM_S390_MEMOP_SIDA_WRITE	3
 /* flags for kvm_s390_mem_op->flags */
 #define KVM_S390_MEMOP_F_CHECK_ONLY		(1ULL << 0)
 #define KVM_S390_MEMOP_F_INJECT_EXCEPTION	(1ULL << 1)
target/s390x/cpu.h +6 -1
index cbc53c99cf..491d6860a8 100644
@@ -823,7 +823,12 @@ int s390_cpu_virt_mem_rw(S390CPU *cpu, vaddr laddr, uint8_t ar, void *hostbuf,
 #define s390_cpu_virt_mem_check_write(cpu, laddr, ar, len)   \
         s390_cpu_virt_mem_rw(cpu, laddr, ar, NULL, len, true)
 void s390_cpu_virt_mem_handle_exc(S390CPU *cpu, uintptr_t ra);
-
+int s390_cpu_pv_mem_rw(S390CPU *cpu, unsigned int offset,  void *hostbuf,
+                       int len, bool is_write);
+#define s390_cpu_pv_mem_read(cpu, offset, dest, len)    \
+        s390_cpu_pv_mem_rw(cpu, offset, dest, len, false)
+#define s390_cpu_pv_mem_write(cpu, offset, dest, len)       \
+        s390_cpu_pv_mem_rw(cpu, offset, dest, len, true)
 
 /* sigp.c */
 int s390_cpu_restart(S390CPU *cpu);
target/s390x/kvm.c +0 -0
index eec0b92479..f222836df5 100644

target/s390x/kvm.c int kvm_s390_mem_op(S390CPU *cpu, vaddr addr, uint8_t ar, void *hostbuf, · line 852

Quoted 7 lines
target/s390x/kvm.c +4 -0
@@ -846,6 +846,29 @@ int kvm_s390_mem_op(S390CPU *cpu, vaddr addr, uint8_t ar, void *hostbuf,
     return ret;
 }
 
+int kvm_s390_mem_op_pv(S390CPU *cpu, uint64_t offset, void *hostbuf,
+                       int len, bool is_write)
+{
+    int ret = 0;

move that after the struct declaration/definition. No need to initialize to 0.

target/s390x/kvm.c int kvm_s390_mem_op(S390CPU *cpu, vaddr addr, uint8_t ar, void *hostbuf, · line 863

Quoted 11 lines
target/s390x/kvm.c +11 -0
+    struct kvm_s390_mem_op mem_op = {
+        .sida_offset = offset,
+        .size = len,
+        .op = is_write ? KVM_S390_MEMOP_SIDA_WRITE
+                       : KVM_S390_MEMOP_SIDA_READ,
+        .buf = (uint64_t)hostbuf,
+    };
+
+    if (!cap_mem_op) {
+        return -ENOSYS;
+    }

if (!cap_mem_op || !cap_protected)

And move cap_protected from patch 15 in here.

target/s390x/kvm.c int kvm_s390_mem_op(S390CPU *cpu, vaddr addr, uint8_t ar, void *hostbuf, · line 867

Quoted 4 lines
target/s390x/kvm.c +4 -0
+
+    ret = kvm_vcpu_ioctl(CPU(cpu), KVM_S390_MEM_OP, &mem_op);
+    if (ret < 0) {
+        warn_report("KVM_S390_MEM_OP failed: %s", strerror(-ret));

I'd even error_report(). After all, this should never fail.

Quoted 25 lines
target/s390x/kvm.c +4 -0
+    }
+    return ret;
+}
+
 /*
  * Legacy layout for s390:
  * Older S390 KVM requires the topmost vma of the RAM to be
target/s390x/kvm_s390x.h +2 -0
index 0b21789796..9c38f6ccce 100644
@@ -19,6 +19,8 @@ void kvm_s390_vcpu_interrupt(S390CPU *cpu, struct kvm_s390_irq *irq);
 void kvm_s390_access_exception(S390CPU *cpu, uint16_t code, uint64_t te_code);
 int kvm_s390_mem_op(S390CPU *cpu, vaddr addr, uint8_t ar, void *hostbuf,
                     int len, bool is_write);
+int kvm_s390_mem_op_pv(S390CPU *cpu, vaddr addr, void *hostbuf, int len,
+                       bool is_write);
 void kvm_s390_program_interrupt(S390CPU *cpu, uint16_t code);
 int kvm_s390_set_cpu_state(S390CPU *cpu, uint8_t cpu_state);
 void kvm_s390_vcpu_interrupt_pre_save(S390CPU *cpu);
target/s390x/mmu_helper.c +0 -0
index c9f3f34750..ad485399db 100644

target/s390x/mmu_helper.c static int translate_pages(S390CPU *cpu, vaddr addr, int nr_pages, · line 483

Quoted 10 lines
target/s390x/mmu_helper.c +7 -0
@@ -474,6 +474,15 @@ static int translate_pages(S390CPU *cpu, vaddr addr, int nr_pages,
     return 0;
 }
 
+int s390_cpu_pv_mem_rw(S390CPU *cpu, unsigned int offset, void *hostbuf,
+                       int len, bool is_write)
+{
+    int ret;
+
+    ret = kvm_s390_mem_op_pv(cpu, offset, hostbuf, len, is_write);
+    return ret;

if (kvm_enabled()) {
...

} else {
    /* Helpful comment */
    g_assert_not_reached();
}
Signature · 3 lines
Thanks,

David / dhildenb
#19 lore raw

Re: [PATCH v4 07/16] s390x: Add SIDA memory ops

Quoted 54 lines · David Hildenbrand
Quoted 53 lines · Janosch Frank

Protected guests save the instruction control blocks in the SIDA instead of QEMU/KVM directly accessing the guest's memory.

Let's introduce new functions to access the SIDA.

---
 linux-headers/linux/kvm.h |  2 ++
 target/s390x/cpu.h        |  7 ++++++-
 target/s390x/kvm.c        | 23 +++++++++++++++++++++++
 target/s390x/kvm_s390x.h  |  2 ++
 target/s390x/mmu_helper.c |  9 +++++++++
 5 files changed, 42 insertions(+), 1 deletion(-)
linux-headers/linux/kvm.h +2 -0
index 2e647f2d9b..7ccf5988d2 100644
@@ -483,6 +483,8 @@ struct kvm_s390_mem_op {
 /* types for kvm_s390_mem_op->op */
 #define KVM_S390_MEMOP_LOGICAL_READ	0
 #define KVM_S390_MEMOP_LOGICAL_WRITE	1
+#define KVM_S390_MEMOP_SIDA_READ	2
+#define KVM_S390_MEMOP_SIDA_WRITE	3
 /* flags for kvm_s390_mem_op->flags */
 #define KVM_S390_MEMOP_F_CHECK_ONLY		(1ULL << 0)
 #define KVM_S390_MEMOP_F_INJECT_EXCEPTION	(1ULL << 1)
target/s390x/cpu.h +6 -1
index cbc53c99cf..491d6860a8 100644
@@ -823,7 +823,12 @@ int s390_cpu_virt_mem_rw(S390CPU *cpu, vaddr laddr, uint8_t ar, void *hostbuf,
 #define s390_cpu_virt_mem_check_write(cpu, laddr, ar, len)   \
         s390_cpu_virt_mem_rw(cpu, laddr, ar, NULL, len, true)
 void s390_cpu_virt_mem_handle_exc(S390CPU *cpu, uintptr_t ra);
-
+int s390_cpu_pv_mem_rw(S390CPU *cpu, unsigned int offset,  void *hostbuf,
+                       int len, bool is_write);
+#define s390_cpu_pv_mem_read(cpu, offset, dest, len)    \
+        s390_cpu_pv_mem_rw(cpu, offset, dest, len, false)
+#define s390_cpu_pv_mem_write(cpu, offset, dest, len)       \
+        s390_cpu_pv_mem_rw(cpu, offset, dest, len, true)
 
 /* sigp.c */
 int s390_cpu_restart(S390CPU *cpu);
target/s390x/kvm.c +0 -0
index eec0b92479..f222836df5 100644
@@ -846,6 +846,29 @@ int kvm_s390_mem_op(S390CPU *cpu, vaddr addr, uint8_t ar, void *hostbuf,
     return ret;
 }
 
Quoted 7 lines

target/s390x/kvm.c int kvm_s390_mem_op(S390CPU *cpu, vaddr addr, uint8_t ar, void *hostbuf, · line 852

Quoted 4 lines
target/s390x/kvm.c +4 -0
+int kvm_s390_mem_op_pv(S390CPU *cpu, uint64_t offset, void *hostbuf,
+                       int len, bool is_write)
+{
+    int ret = 0;

move that after the struct declaration/definition. No need to initialize to 0.

Ack

Quoted 13 lines

target/s390x/kvm.c int kvm_s390_mem_op(S390CPU *cpu, vaddr addr, uint8_t ar, void *hostbuf, · line 863

Quoted 12 lines
target/s390x/kvm.c +11 -0
+    struct kvm_s390_mem_op mem_op = {
+        .sida_offset = offset,
+        .size = len,
+        .op = is_write ? KVM_S390_MEMOP_SIDA_WRITE
+                       : KVM_S390_MEMOP_SIDA_READ,
+        .buf = (uint64_t)hostbuf,
+    };
+
+    if (!cap_mem_op) {
+        return -ENOSYS;
+    }
Quoted 3 lines

if (!cap_mem_op || !cap_protected)

And move cap_protected from patch 15 in here.

Ack

Quoted 7 lines

target/s390x/kvm.c int kvm_s390_mem_op(S390CPU *cpu, vaddr addr, uint8_t ar, void *hostbuf, · line 867

Quoted 5 lines
target/s390x/kvm.c +4 -0
+
+    ret = kvm_vcpu_ioctl(CPU(cpu), KVM_S390_MEM_OP, &mem_op);
+    if (ret < 0) {
+        warn_report("KVM_S390_MEM_OP failed: %s", strerror(-ret));

I'd even error_report(). After all, this should never fail.

Hmm, might even make sense to exit here.
In my experience a failed memop will soon result in bigger problems as it effectively stops a cpu.

Quoted 39 lines
Quoted 26 lines
target/s390x/kvm.c +4 -0
+    }
+    return ret;
+}
+
 /*
  * Legacy layout for s390:
  * Older S390 KVM requires the topmost vma of the RAM to be
target/s390x/kvm_s390x.h +2 -0
index 0b21789796..9c38f6ccce 100644
@@ -19,6 +19,8 @@ void kvm_s390_vcpu_interrupt(S390CPU *cpu, struct kvm_s390_irq *irq);
 void kvm_s390_access_exception(S390CPU *cpu, uint16_t code, uint64_t te_code);
 int kvm_s390_mem_op(S390CPU *cpu, vaddr addr, uint8_t ar, void *hostbuf,
                     int len, bool is_write);
+int kvm_s390_mem_op_pv(S390CPU *cpu, vaddr addr, void *hostbuf, int len,
+                       bool is_write);
 void kvm_s390_program_interrupt(S390CPU *cpu, uint16_t code);
 int kvm_s390_set_cpu_state(S390CPU *cpu, uint8_t cpu_state);
 void kvm_s390_vcpu_interrupt_pre_save(S390CPU *cpu);
target/s390x/mmu_helper.c +0 -0
index c9f3f34750..ad485399db 100644

target/s390x/mmu_helper.c static int translate_pages(S390CPU *cpu, vaddr addr, int nr_pages, · line 483

Quoted 10 lines
target/s390x/mmu_helper.c +7 -0
@@ -474,6 +474,15 @@ static int translate_pages(S390CPU *cpu, vaddr addr, int nr_pages,
     return 0;
 }
 
+int s390_cpu_pv_mem_rw(S390CPU *cpu, unsigned int offset, void *hostbuf,
+                       int len, bool is_write)
+{
+    int ret;
+
+    ret = kvm_s390_mem_op_pv(cpu, offset, hostbuf, len, is_write);
+    return ret;

if (kvm_enabled()) {
...

Quoted 5 lines
} else {
    /* Helpful comment */
    g_assert_not_reached();
}

Ack.

Quoted 1 line
#20 lore raw

PATCH v4 8/16 s390x: protvirt: Move STSI data over SIDAD

In reply to Janosch Frank

For protected guests, we need to put the STSI emulation results into the SIDA, so SIE will write them into the guest at the next entry.

---
 target/s390x/kvm.c | 15 ++++++++++++---
 1 file changed, 12 insertions(+), 3 deletions(-)
target/s390x/kvm.c +12 -3
index f222836df5..3a5a5146e3 100644
@@ -1795,11 +1795,16 @@ static int handle_tsch(S390CPU *cpu)
 
 static void insert_stsi_3_2_2(S390CPU *cpu, __u64 addr, uint8_t ar)
 {
+    CPUS390XState *env = &cpu->env;
     SysIB_322 sysib;
     int del;
 
-    if (s390_cpu_virt_mem_read(cpu, addr, ar, &sysib, sizeof(sysib))) {
-        return;
+    if (env->pv) {
+        s390_cpu_pv_mem_read(cpu, 0, &sysib, sizeof(sysib));
+    } else {
+        if (s390_cpu_virt_mem_read(cpu, addr, ar, &sysib, sizeof(sysib))) {
+            return;
+        }
     }
     /* Shift the stack of Extended Names to prepare for our own data */
     memmove(&sysib.ext_names[1], &sysib.ext_names[0],
@@ -1838,7 +1843,11 @@ static void insert_stsi_3_2_2(S390CPU *cpu, __u64 addr, uint8_t ar)
     /* Insert UUID */
     memcpy(sysib.vm[0].uuid, &qemu_uuid, sizeof(sysib.vm[0].uuid));
 
-    s390_cpu_virt_mem_write(cpu, addr, ar, &sysib, sizeof(sysib));
+    if (env->pv) {
+        s390_cpu_pv_mem_write(cpu, 0, &sysib, sizeof(sysib));
+    } else {
+        s390_cpu_virt_mem_write(cpu, addr, ar, &sysib, sizeof(sysib));
+    }
 }
 
 static int handle_stsi(S390CPU *cpu)
Signature · 1 line
2.20.1
Acked-by on v4 8/16
#21 lore raw

Re: [PATCH v4 08/16] s390x: protvirt: Move STSI data over SIDAD

Quoted 35 lines · Janosch Frank

For protected guests, we need to put the STSI emulation results into the SIDA, so SIE will write them into the guest at the next entry.

---
 target/s390x/kvm.c | 15 ++++++++++++---
 1 file changed, 12 insertions(+), 3 deletions(-)
target/s390x/kvm.c +7 -2
index f222836df5..3a5a5146e3 100644
@@ -1795,11 +1795,16 @@ static int handle_tsch(S390CPU *cpu)
 
 static void insert_stsi_3_2_2(S390CPU *cpu, __u64 addr, uint8_t ar)
 {
+    CPUS390XState *env = &cpu->env;
     SysIB_322 sysib;
     int del;
 
-    if (s390_cpu_virt_mem_read(cpu, addr, ar, &sysib, sizeof(sysib))) {
-        return;
+    if (env->pv) {
+        s390_cpu_pv_mem_read(cpu, 0, &sysib, sizeof(sysib));
+    } else {
+        if (s390_cpu_virt_mem_read(cpu, addr, ar, &sysib, sizeof(sysib))) {
+            return;
+        }
     }
     /* Shift the stack of Extended Names to prepare for our own data */
     memmove(&sysib.ext_names[1], &sysib.ext_names[0],
@@ -1838,7 +1843,11 @@ static void insert_stsi_3_2_2(S390CPU *cpu, __u64 addr, uint8_t ar)
     /* Insert UUID */
     memcpy(sysib.vm[0].uuid, &qemu_uuid, sizeof(sysib.vm[0].uuid));

target/s390x/kvm.c static void insert_stsi_3_2_2(S390CPU *cpu, __u64 addr, uint8_t ar) · line 1853

Quoted 10 lines
target/s390x/kvm.c +5 -1
 
-    s390_cpu_virt_mem_write(cpu, addr, ar, &sysib, sizeof(sysib));
+    if (env->pv) {
+        s390_cpu_pv_mem_write(cpu, 0, &sysib, sizeof(sysib));
+    } else {
+        s390_cpu_virt_mem_write(cpu, addr, ar, &sysib, sizeof(sysib));
+    }
 }
 
 static int handle_stsi(S390CPU *cpu)
Signature · 3 lines
Thanks,

David / dhildenb
#22 lore raw

PATCH v4 9/16 s390x: protvirt: SCLP interpretation

In reply to Janosch Frank

SCLP for a protected guest is done over the SIDAD, so we need to use the s390_cpu_virt_mem_* functions to access the SIDAD instead of guest memory when reading/writing SCBs.

To not confuse the sclp emulation, we set 0x4000 as the SCCB address, since the function that injects the sclp external interrupt would reject a zero sccb address.

---
 hw/s390x/sclp.c         | 17 +++++++++++++++++
 include/hw/s390x/sclp.h |  2 ++
 target/s390x/kvm.c      |  5 +++++
 3 files changed, 24 insertions(+)
hw/s390x/sclp.c +17 -0
index af0bfbc2ec..5136f5fcbe 100644
@@ -193,6 +193,23 @@ static void sclp_execute(SCLPDevice *sclp, SCCB *sccb, uint32_t code)
     }
 }
 
+#define SCLP_PV_DUMMY_ADDR 0x4000
+int sclp_service_call_protected(CPUS390XState *env, uint64_t sccb,
+                                uint32_t code)
+{
+    SCLPDevice *sclp = get_sclp_device();
+    SCLPDeviceClass *sclp_c = SCLP_GET_CLASS(sclp);
+    SCCB work_sccb;
+    hwaddr sccb_len = sizeof(SCCB);
+
+    s390_cpu_pv_mem_read(env_archcpu(env), 0, &work_sccb, sccb_len);
+    sclp_c->execute(sclp, &work_sccb, code);
+    s390_cpu_pv_mem_write(env_archcpu(env), 0, &work_sccb,
+                          be16_to_cpu(work_sccb.h.length));
+    sclp_c->service_interrupt(sclp, SCLP_PV_DUMMY_ADDR);
+    return 0;
+}
+
 int sclp_service_call(CPUS390XState *env, uint64_t sccb, uint32_t code)
 {
     SCLPDevice *sclp = get_sclp_device();
include/hw/s390x/sclp.h +2 -0
index c54413b78c..c0a3faa37d 100644
@@ -217,5 +217,7 @@ void s390_sclp_init(void);
 void sclp_service_interrupt(uint32_t sccb);
 void raise_irq_cpu_hotplug(void);
 int sclp_service_call(CPUS390XState *env, uint64_t sccb, uint32_t code);
+int sclp_service_call_protected(CPUS390XState *env, uint64_t sccb,
+                                uint32_t code);
 
 #endif
target/s390x/kvm.c +5 -0
index 3a5a5146e3..31dd49729b 100644
@@ -1224,6 +1224,11 @@ static void kvm_sclp_service_call(S390CPU *cpu, struct kvm_run *run,
     sccb = env->regs[ipbh0 & 0xf];
     code = env->regs[(ipbh0 & 0xf0) >> 4];
 
+    if (run->s390_sieic.icptcode == ICPT_PV_INSTR) {
+        sclp_service_call_protected(env, sccb, code);
+        return;
+    }
+
     r = sclp_service_call(env, sccb, code);
     if (r < 0) {
         kvm_s390_program_interrupt(cpu, -r);
Signature · 1 line
2.20.1
#23 lore raw

PATCH v4 10/16 s390x: protvirt: Set guest IPL PSW

In reply to Janosch Frank

Handling of CPU reset and setting of the IPL psw from guest storage at offset 0 is done by a Ultravisor call. Let's only fetch it if necessary.

---
 target/s390x/cpu.c | 23 ++++++++++++++---------
 1 file changed, 14 insertions(+), 9 deletions(-)
target/s390x/cpu.c +14 -9
index 1dbd84b9d7..fe2ac450d2 100644
@@ -78,16 +78,21 @@ static bool s390_cpu_has_work(CPUState *cs)
 static void s390_cpu_load_normal(CPUState *s)
 {
     S390CPU *cpu = S390_CPU(s);
-    uint64_t spsw = ldq_phys(s->as, 0);
-
-    cpu->env.psw.mask = spsw & 0xffffffff80000000ULL;
-    /*
-     * Invert short psw indication, so SIE will report a specification
-     * exception if it was not set.
-     */
-    cpu->env.psw.mask ^= PSW_MASK_SHORTPSW;
-    cpu->env.psw.addr = spsw & 0x7fffffffULL;
+    CPUS390XState *env = &cpu->env;
+    uint64_t spsw;
 
+    if (!env->pv) {
+        spsw = ldq_phys(s->as, 0);
+        cpu->env.psw.mask = spsw & 0xffffffff80000000ULL;
+        /*
+         * Invert short psw indication, so SIE will report a specification
+         * exception if it was not set.
+         */
+        cpu->env.psw.mask ^= PSW_MASK_SHORTPSW;
+        cpu->env.psw.addr = spsw & 0x7fffffffULL;
+    } else {
+        s390_cpu_set_state(S390_CPU_STATE_LOAD, cpu);
+    }
     s390_cpu_set_state(S390_CPU_STATE_OPERATING, cpu);
 }
 #endif
Signature · 1 line
2.20.1
#24 lore raw

PATCH v4 11/16 s390x: protvirt: Move diag 308 data over SIDAD

In reply to Janosch Frank

For protected guests the IPIB is written/read to/from the satellite block, so we need those accesses to go through s390_cpu_pv_mem_read/write().

---
 target/s390x/diag.c | 30 +++++++++++++++++++++++-------
 1 file changed, 23 insertions(+), 7 deletions(-)
target/s390x/diag.c +23 -7
index aedc774695..be80c0367b 100644
@@ -88,6 +88,7 @@ static int diag308_parm_check(CPUS390XState *env, uint64_t r1, uint64_t addr,
 void handle_diag_308(CPUS390XState *env, uint64_t r1, uint64_t r3, uintptr_t ra)
 {
     CPUState *cs = env_cpu(env);
+    S390CPU *cpu = S390_CPU(cs);
     uint64_t addr =  env->regs[r1];
     uint64_t subcode = env->regs[r3];
     IplParameterBlock *iplb;
@@ -119,13 +120,22 @@ void handle_diag_308(CPUS390XState *env, uint64_t r1, uint64_t r3, uintptr_t ra)
             return;
         }
         iplb = g_new0(IplParameterBlock, 1);
-        cpu_physical_memory_read(addr, iplb, sizeof(iplb->len));
+        if (!env->pv) {
+            cpu_physical_memory_read(addr, iplb, sizeof(iplb->len));
+        } else {
+            s390_cpu_pv_mem_read(cpu, 0, iplb, sizeof(iplb->len));
+        }
+
         if (!iplb_valid_len(iplb)) {
             env->regs[r1 + 1] = DIAG_308_RC_INVALID;
             goto out;
         }
 
-        cpu_physical_memory_read(addr, iplb, be32_to_cpu(iplb->len));
+        if (!env->pv) {
+            cpu_physical_memory_read(addr, iplb, be32_to_cpu(iplb->len));
+        } else {
+            s390_cpu_pv_mem_read(cpu, 0, iplb, be32_to_cpu(iplb->len));
+        }
 
         if (!iplb_valid_ccw(iplb) && !iplb_valid_fcp(iplb) &&
             !(iplb_valid_pv(iplb) && s390_ipl_pv_check_components(iplb) >= 0)) {
@@ -137,7 +147,7 @@ void handle_diag_308(CPUS390XState *env, uint64_t r1, uint64_t r3, uintptr_t ra)
         env->regs[r1 + 1] = DIAG_308_RC_OK;
 out:
         g_free(iplb);
-        return;
+        break;
     case DIAG308_STORE:
     case DIAG308_PV_STORE:
         if (diag308_parm_check(env, r1, addr, ra, true)) {
@@ -148,12 +158,18 @@ out:
         } else {
             iplb = s390_ipl_get_iplb();
         }
-        if (iplb) {
-            cpu_physical_memory_write(addr, iplb, be32_to_cpu(iplb->len));
-            env->regs[r1 + 1] = DIAG_308_RC_OK;
-        } else {
+        if (!iplb) {
             env->regs[r1 + 1] = DIAG_308_RC_NO_CONF;
+            return;
         }
+
+        if (!env->pv) {
+            cpu_physical_memory_write(addr, iplb, be32_to_cpu(iplb->len));
+        } else {
+            s390_cpu_pv_mem_write(cpu, 0, iplb, be32_to_cpu(iplb->len));
+        }
+
+        env->regs[r1 + 1] = DIAG_308_RC_OK;
         break;
     case DIAG308_PV_START:
         iplb = s390_ipl_get_iplb_secure();
Signature · 1 line
2.20.1
#25 lore raw

PATCH v4 12/16 s390x: protvirt: Disable address checks for PV guest IO emulation

In reply to Janosch Frank

IO instruction data is routed through SIDAD for protected guests, so adresses do not need to be checked, as this is kernel memory.

---
 target/s390x/ioinst.c | 26 +++++++++++++++++++-------
 1 file changed, 19 insertions(+), 7 deletions(-)
target/s390x/ioinst.c +19 -7
index c437a1d8c6..e4102430aa 100644
@@ -17,6 +17,16 @@
 #include "trace.h"
 #include "hw/s390x/s390-pci-bus.h"
 
+static uint64_t get_address_from_regs(CPUS390XState *env, uint32_t ipb,
+                                      uint8_t *ar)
+{
+    if (env->pv) {
+        *ar = 0;
+        return 0;
+    }
+    return decode_basedisp_s(env, ipb, ar);
+}
+
 int ioinst_disassemble_sch_ident(uint32_t value, int *m, int *cssid, int *ssid,
                                  int *schid)
 {
@@ -114,7 +124,7 @@ void ioinst_handle_msch(S390CPU *cpu, uint64_t reg1, uint32_t ipb, uintptr_t ra)
     CPUS390XState *env = &cpu->env;
     uint8_t ar;
 
-    addr = decode_basedisp_s(env, ipb, &ar);
+    addr = get_address_from_regs(env, ipb, &ar);
     if (addr & 3) {
         s390_program_interrupt(env, PGM_SPECIFICATION, ra);
         return;
@@ -171,7 +181,7 @@ void ioinst_handle_ssch(S390CPU *cpu, uint64_t reg1, uint32_t ipb, uintptr_t ra)
     CPUS390XState *env = &cpu->env;
     uint8_t ar;
 
-    addr = decode_basedisp_s(env, ipb, &ar);
+    addr = get_address_from_regs(env, ipb, &ar);
     if (addr & 3) {
         s390_program_interrupt(env, PGM_SPECIFICATION, ra);
         return;
@@ -203,7 +213,7 @@ void ioinst_handle_stcrw(S390CPU *cpu, uint32_t ipb, uintptr_t ra)
     CPUS390XState *env = &cpu->env;
     uint8_t ar;
 
-    addr = decode_basedisp_s(env, ipb, &ar);
+    addr = get_address_from_regs(env, ipb, &ar);
     if (addr & 3) {
         s390_program_interrupt(env, PGM_SPECIFICATION, ra);
         return;
@@ -234,7 +244,7 @@ void ioinst_handle_stsch(S390CPU *cpu, uint64_t reg1, uint32_t ipb,
     CPUS390XState *env = &cpu->env;
     uint8_t ar;
 
-    addr = decode_basedisp_s(env, ipb, &ar);
+    addr = get_address_from_regs(env, ipb, &ar);
     if (addr & 3) {
         s390_program_interrupt(env, PGM_SPECIFICATION, ra);
         return;
@@ -303,7 +313,7 @@ int ioinst_handle_tsch(S390CPU *cpu, uint64_t reg1, uint32_t ipb, uintptr_t ra)
         return -EIO;
     }
     trace_ioinst_sch_id("tsch", cssid, ssid, schid);
-    addr = decode_basedisp_s(env, ipb, &ar);
+    addr = get_address_from_regs(env, ipb, &ar);
     if (addr & 3) {
         s390_program_interrupt(env, PGM_SPECIFICATION, ra);
         return -EIO;
@@ -601,7 +611,7 @@ void ioinst_handle_chsc(S390CPU *cpu, uint32_t ipb, uintptr_t ra)
 {
     ChscReq *req;
     ChscResp *res;
-    uint64_t addr;
+    uint64_t addr = 0;
     int reg;
     uint16_t len;
     uint16_t command;
@@ -610,7 +620,9 @@ void ioinst_handle_chsc(S390CPU *cpu, uint32_t ipb, uintptr_t ra)
 
     trace_ioinst("chsc");
     reg = (ipb >> 20) & 0x00f;
-    addr = env->regs[reg];
+    if (!env->pv) {
+        addr = env->regs[reg];
+    }
     /* Page boundary? */
     if (addr & 0xfff) {
         s390_program_interrupt(env, PGM_SPECIFICATION, ra);
Signature · 1 line
2.20.1
#26 lore raw

PATCH v4 13/16 s390x: protvirt: Move IO control structures over SIDA

In reply to Janosch Frank

For protected guests, we need to put the IO emulation results into the SIDA, so SIE will write them into the guest at the next entry.

---
 target/s390x/ioinst.c | 87 ++++++++++++++++++++++++++++++-------------
 1 file changed, 61 insertions(+), 26 deletions(-)
target/s390x/ioinst.c +61 -26
index e4102430aa..330b04d79a 100644
@@ -129,9 +129,13 @@ void ioinst_handle_msch(S390CPU *cpu, uint64_t reg1, uint32_t ipb, uintptr_t ra)
         s390_program_interrupt(env, PGM_SPECIFICATION, ra);
         return;
     }
-    if (s390_cpu_virt_mem_read(cpu, addr, ar, &schib, sizeof(schib))) {
-        s390_cpu_virt_mem_handle_exc(cpu, ra);
-        return;
+    if (env->pv) {
+        s390_cpu_pv_mem_read(cpu, addr, &schib, sizeof(schib));
+    } else {
+        if (s390_cpu_virt_mem_read(cpu, addr, ar, &schib, sizeof(schib))) {
+            s390_cpu_virt_mem_handle_exc(cpu, ra);
+            return;
+        }
     }
     if (ioinst_disassemble_sch_ident(reg1, &m, &cssid, &ssid, &schid) ||
         !ioinst_schib_valid(&schib)) {
@@ -186,9 +190,13 @@ void ioinst_handle_ssch(S390CPU *cpu, uint64_t reg1, uint32_t ipb, uintptr_t ra)
         s390_program_interrupt(env, PGM_SPECIFICATION, ra);
         return;
     }
-    if (s390_cpu_virt_mem_read(cpu, addr, ar, &orig_orb, sizeof(orb))) {
-        s390_cpu_virt_mem_handle_exc(cpu, ra);
-        return;
+    if (env->pv) {
+        s390_cpu_pv_mem_read(cpu, addr, &orig_orb, sizeof(orb));
+    } else {
+        if (s390_cpu_virt_mem_read(cpu, addr, ar, &orig_orb, sizeof(orb))) {
+            s390_cpu_virt_mem_handle_exc(cpu, ra);
+            return;
+        }
     }
     copy_orb_from_guest(&orb, &orig_orb);
     if (ioinst_disassemble_sch_ident(reg1, &m, &cssid, &ssid, &schid) ||
@@ -222,14 +230,19 @@ void ioinst_handle_stcrw(S390CPU *cpu, uint32_t ipb, uintptr_t ra)
     cc = css_do_stcrw(&crw);
     /* 0 - crw stored, 1 - zeroes stored */
 
-    if (s390_cpu_virt_mem_write(cpu, addr, ar, &crw, sizeof(crw)) == 0) {
+    if (env->pv) {
+        s390_cpu_pv_mem_write(cpu, addr, &crw, sizeof(crw));
         setcc(cpu, cc);
     } else {
-        if (cc == 0) {
-            /* Write failed: requeue CRW since STCRW is suppressing */
-            css_undo_stcrw(&crw);
+        if (s390_cpu_virt_mem_write(cpu, addr, ar, &crw, sizeof(crw)) == 0) {
+            setcc(cpu, cc);
+        } else {
+            if (cc == 0) {
+                /* Write failed: requeue CRW since STCRW is suppressing */
+                css_undo_stcrw(&crw);
+            }
+            s390_cpu_virt_mem_handle_exc(cpu, ra);
         }
-        s390_cpu_virt_mem_handle_exc(cpu, ra);
     }
 }
 
@@ -251,6 +264,9 @@ void ioinst_handle_stsch(S390CPU *cpu, uint64_t reg1, uint32_t ipb,
     }
 
     if (ioinst_disassemble_sch_ident(reg1, &m, &cssid, &ssid, &schid)) {
+        if (env->pv) {
+            return;
+        }
         /*
          * As operand exceptions have a lower priority than access exceptions,
          * we check whether the memory area is writeable (injecting the
@@ -283,14 +299,19 @@ void ioinst_handle_stsch(S390CPU *cpu, uint64_t reg1, uint32_t ipb,
         }
     }
     if (cc != 3) {
-        if (s390_cpu_virt_mem_write(cpu, addr, ar, &schib,
-                                    sizeof(schib)) != 0) {
-            s390_cpu_virt_mem_handle_exc(cpu, ra);
-            return;
+        if (env->pv) {
+            s390_cpu_pv_mem_write(cpu, addr, &schib, sizeof(schib));
+        } else {
+            if (s390_cpu_virt_mem_write(cpu, addr, ar, &schib,
+                                        sizeof(schib)) != 0) {
+                s390_cpu_virt_mem_handle_exc(cpu, ra);
+                return;
+            }
         }
     } else {
         /* Access exceptions have a higher priority than cc3 */
-        if (s390_cpu_virt_mem_check_write(cpu, addr, ar, sizeof(schib)) != 0) {
+        if (!env->pv &&
+            s390_cpu_virt_mem_check_write(cpu, addr, ar, sizeof(schib)) != 0) {
             s390_cpu_virt_mem_handle_exc(cpu, ra);
             return;
         }
@@ -327,15 +348,20 @@ int ioinst_handle_tsch(S390CPU *cpu, uint64_t reg1, uint32_t ipb, uintptr_t ra)
     }
     /* 0 - status pending, 1 - not status pending, 3 - not operational */
     if (cc != 3) {
-        if (s390_cpu_virt_mem_write(cpu, addr, ar, &irb, irb_len) != 0) {
-            s390_cpu_virt_mem_handle_exc(cpu, ra);
-            return -EFAULT;
+        if (env->pv) {
+            s390_cpu_pv_mem_write(cpu, addr, &irb, irb_len);
+        } else {
+            if (s390_cpu_virt_mem_write(cpu, addr, ar, &irb, irb_len) != 0) {
+                s390_cpu_virt_mem_handle_exc(cpu, ra);
+                return -EFAULT;
+            }
         }
         css_do_tsch_update_subch(sch);
     } else {
         irb_len = sizeof(irb) - sizeof(irb.emw);
         /* Access exceptions have a higher priority than cc3 */
-        if (s390_cpu_virt_mem_check_write(cpu, addr, ar, irb_len) != 0) {
+        if (!env->pv &&
+            s390_cpu_virt_mem_check_write(cpu, addr, ar, irb_len) != 0) {
             s390_cpu_virt_mem_handle_exc(cpu, ra);
             return -EFAULT;
         }
@@ -633,9 +659,13 @@ void ioinst_handle_chsc(S390CPU *cpu, uint32_t ipb, uintptr_t ra)
      * present CHSC sub-handlers ... if we ever need more, we should take
      * care of req->len here first.
      */
-    if (s390_cpu_virt_mem_read(cpu, addr, reg, buf, sizeof(ChscReq))) {
-        s390_cpu_virt_mem_handle_exc(cpu, ra);
-        return;
+    if (env->pv) {
+        s390_cpu_pv_mem_read(cpu, addr, buf, sizeof(ChscReq));
+    } else {
+        if (s390_cpu_virt_mem_read(cpu, addr, reg, buf, sizeof(ChscReq))) {
+            s390_cpu_virt_mem_handle_exc(cpu, ra);
+            return;
+        }
     }
     req = (ChscReq *)buf;
     len = be16_to_cpu(req->len);
@@ -666,11 +696,16 @@ void ioinst_handle_chsc(S390CPU *cpu, uint32_t ipb, uintptr_t ra)
         break;
     }
 
-    if (!s390_cpu_virt_mem_write(cpu, addr + len, reg, res,
-                                 be16_to_cpu(res->len))) {
+    if (env->pv) {
+        s390_cpu_pv_mem_write(cpu, addr + len, res, be16_to_cpu(res->len));
         setcc(cpu, 0);    /* Command execution complete */
     } else {
-        s390_cpu_virt_mem_handle_exc(cpu, ra);
+        if (!s390_cpu_virt_mem_write(cpu, addr + len, reg, res,
+                                     be16_to_cpu(res->len))) {
+            setcc(cpu, 0);    /* Command execution complete */
+        } else {
+            s390_cpu_virt_mem_handle_exc(cpu, ra);
+        }
     }
 }
 
Signature · 1 line
2.20.1
#27 lore raw

PATCH v4 14/16 s390x: protvirt: Handle SIGP store status correctly

In reply to Janosch Frank

Status storing is not done by QEMU anymore, but is handled by SIE.

---
 target/s390x/helper.c | 4 ++++
 target/s390x/sigp.c   | 1 +
 2 files changed, 5 insertions(+)
target/s390x/helper.c +4 -0
index a3a49164e4..3800c4b395 100644
@@ -246,6 +246,10 @@ int s390_store_status(S390CPU *cpu, hwaddr addr, bool store_arch)
     hwaddr len = sizeof(*sa);
     int i;
 
+    if (cpu->env.pv) {
+        return 0;
+    }
+
     sa = cpu_physical_memory_map(addr, &len, 1);
     if (!sa) {
         return -EFAULT;
target/s390x/sigp.c +1 -0
index c604f17710..e1c8071464 100644
@@ -497,6 +497,7 @@ void do_stop_interrupt(CPUS390XState *env)
     if (s390_cpu_set_state(S390_CPU_STATE_STOPPED, cpu) == 0) {
         qemu_system_shutdown_request(SHUTDOWN_CAUSE_GUEST_SHUTDOWN);
     }
+    /* Storing will occur on next SIE entry for protected VMs */
     if (cpu->env.sigp_order == SIGP_STOP_STORE_STATUS) {
         s390_store_status(cpu, S390_STORE_STATUS_DEF_ADDR, true);
     }
Signature · 1 line
2.20.1
#28 lore raw

PATCH v4 15/16 s390x: Add unpack feature to GA1

In reply to Janosch Frank

From: Christian Borntraeger <[email protected]>

---
 target/s390x/gen-features.c | 1 +
 target/s390x/kvm.c          | 7 +++++++
 2 files changed, 8 insertions(+)
target/s390x/gen-features.c +1 -0
index 6278845b12..8ddeebc544 100644
@@ -562,6 +562,7 @@ static uint16_t full_GEN15_GA1[] = {
     S390_FEAT_GROUP_MSA_EXT_9,
     S390_FEAT_GROUP_MSA_EXT_9_PCKMO,
     S390_FEAT_ETOKEN,
+    S390_FEAT_UNPACK,
 };
 
 /* Default features (in order of release)
target/s390x/kvm.c +7 -0
index 31dd49729b..5b6a7ca466 100644
@@ -154,6 +154,7 @@ static int cap_ri;
 static int cap_gs;
 static int cap_hpage_1m;
 static int cap_vcpu_resets;
+static int cap_protected;
 
 static int active_cmma;
 
@@ -346,6 +347,7 @@ int kvm_arch_init(MachineState *ms, KVMState *s)
     cap_mem_op = kvm_check_extension(s, KVM_CAP_S390_MEM_OP);
     cap_s390_irq = kvm_check_extension(s, KVM_CAP_S390_INJECT_IRQ);
     cap_vcpu_resets = kvm_check_extension(s, KVM_CAP_S390_VCPU_RESETS);
+    cap_protected = kvm_check_extension(s, KVM_CAP_S390_PROTECTED);
 
     if (!kvm_check_extension(s, KVM_CAP_S390_GMAP)
         || !kvm_check_extension(s, KVM_CAP_S390_COW)) {
@@ -2394,6 +2396,11 @@ void kvm_s390_get_host_cpu_model(S390CPUModel *model, Error **errp)
         clear_bit(S390_FEAT_BPB, model->features);
     }
 
+    /* we do have the IPL enhancements */
+    if (cap_protected) {
+        set_bit(S390_FEAT_UNPACK, model->features);
+    }
+
     /* We emulate a zPCI bus and AEN, therefore we don't need HW support */
     set_bit(S390_FEAT_ZPCI, model->features);
     set_bit(S390_FEAT_ADAPTER_EVENT_NOTIFICATION, model->features);
Signature · 1 line
2.20.1
#29 lore raw

Re: [PATCH v4 15/16] s390x: Add unpack feature to GA1

Quoted 2 lines · Janosch Frank

From: Christian Borntraeger <[email protected]>

Needs some love. What does the facility unlock/allow. Why can we enable it now. What's in place, what's missing.

Quoted 41 lines
---
 target/s390x/gen-features.c | 1 +
 target/s390x/kvm.c          | 7 +++++++
 2 files changed, 8 insertions(+)
target/s390x/gen-features.c +1 -0
index 6278845b12..8ddeebc544 100644
@@ -562,6 +562,7 @@ static uint16_t full_GEN15_GA1[] = {
     S390_FEAT_GROUP_MSA_EXT_9,
     S390_FEAT_GROUP_MSA_EXT_9_PCKMO,
     S390_FEAT_ETOKEN,
+    S390_FEAT_UNPACK,
 };
 
 /* Default features (in order of release)
target/s390x/kvm.c +2 -0
index 31dd49729b..5b6a7ca466 100644
@@ -154,6 +154,7 @@ static int cap_ri;
 static int cap_gs;
 static int cap_hpage_1m;
 static int cap_vcpu_resets;
+static int cap_protected;
 
 static int active_cmma;
 
@@ -346,6 +347,7 @@ int kvm_arch_init(MachineState *ms, KVMState *s)
     cap_mem_op = kvm_check_extension(s, KVM_CAP_S390_MEM_OP);
     cap_s390_irq = kvm_check_extension(s, KVM_CAP_S390_INJECT_IRQ);
     cap_vcpu_resets = kvm_check_extension(s, KVM_CAP_S390_VCPU_RESETS);
+    cap_protected = kvm_check_extension(s, KVM_CAP_S390_PROTECTED);
 
     if (!kvm_check_extension(s, KVM_CAP_S390_GMAP)
         || !kvm_check_extension(s, KVM_CAP_S390_COW)) {
@@ -2394,6 +2396,11 @@ void kvm_s390_get_host_cpu_model(S390CPUModel *model, Error **errp)
         clear_bit(S390_FEAT_BPB, model->features);

target/s390x/kvm.c void kvm_s390_get_host_cpu_model(S390CPUModel *model, Error **errp) · line 2406

Quoted 10 lines
target/s390x/kvm.c +5 -0
     }
 
+    /* we do have the IPL enhancements */
+    if (cap_protected) {
+        set_bit(S390_FEAT_UNPACK, model->features);
+    }
+
     /* We emulate a zPCI bus and AEN, therefore we don't need HW support */
     set_bit(S390_FEAT_ZPCI, model->features);
     set_bit(S390_FEAT_ADAPTER_EVENT_NOTIFICATION, model->features);

In general, LGTM.

Signature · 3 lines
Thanks,

David / dhildenb
#30 lore raw

PATCH v4 16/16 docs: Add protvirt docs

In reply to Janosch Frank

Lets add some documentation for the Protected VM functionality.

---
 docs/protvirt.rst | 53 +++++++++++++++++++++++++++++++++++++++++++++++
 1 file changed, 53 insertions(+)
 create mode 100644 docs/protvirt.rst
docs/protvirt.rst new file +53 -0
new file mode 100644
index 0000000000..8bfa72be01
@@ -0,1 +1,53 @@
+Protected Virtualization on s390x
+========================
+The memory and most of the register contents of Protected Virtual
+Machines (PVMs) are inaccessible to the hypervisor, effectively
+prohibiting VM introspection when the VM is running. At rest, PVMs are
+encrypted and can only be decrypted by the firmware of specific IBM Z
+machines.
+
+
+Prerequisites
+-------------
+To run PVMs, you need to have a machine with the Protected
+Virtualization feature, which is indicated by the Ultravisor Call
+facility (stfle bit 158). This is a KVM only feature, therefore you
+need a KVM which is able to support PVMs and activate the Ultravisor
+initialization by setting "prot_virt=1" on the kernel command line.
+
+If those requirements are met, the capability "KVM_CAP_S390_PROTECTED"
+will indicate that KVM can support PVMs on that LPAR.
+
+
+QEMU Settings
+-------------
+To indicate to the VM that it can move into protected mode, the
+"Unpack facility" (stfle bit 161) needs to be part of the cpu model of
+the VM.
+
+All I/O devices need to use the IOMMU.
+Passthrough devices are currently not supported.
+
+Host huge page backings are not supported. The guest however can use
+huge pages as indicated by its facilities.
+
+
+Boot Process
+-----------------
+A secure guest image can be booted from disk and using the QEMU
+command line. Booting from disk is done by the unmodified s390-ccw
+BIOS. I.e., the bootmap is interpreted and a number of components is
+read into memory and control is transferred to one of the components
+(zipl stage3), which does some fixups and then transfers control to
+some program residing in guest memory, which is normally the OS
+kernel. The secure image has another component prepended (stage3a)
+which uses the new diag308 subcodes 8 and 10 to trigger the transition
+into secure mode.
+
+Booting from the command line requires that the file passed
+via -kernel has the same memory layout as would result from the disk
+boot. This memory layout includes the encrypted components (kernel,
+initrd, cmdline), the stage3a loader and metadata. In case this boot
+method is used, the command line options -initrd and -cmdline are
+ineffective.  The preparation of secure guest image is done by a
+program (name tbd) of the s390-tools package.
Signature · 1 line
2.20.1
#31 lore raw

Re: [PATCH v4 16/16] docs: Add protvirt docs

Quoted 7 lines · On Thu, 20 Feb 2020 07:56:38 -0500 Janosch Frank <[email protected]> wrote:

Lets add some documentation for the Protected VM functionality.

---
 docs/protvirt.rst | 53 +++++++++++++++++++++++++++++++++++++++++++++++
 1 file changed, 53 insertions(+)
 create mode 100644 docs/protvirt.rst

You'll probably want to add that file to an index as well, so that it gets built properly.

docs/protvirt.rst line 2

Quoted 9 lines
docs/protvirt.rst new file +2 -0
new file mode 100644
index 0000000000..8bfa72be01
@@ -0,1 +1,53 @@
+Protected Virtualization on s390x
+========================

Please lengthen the underlining :)

Also, it might improve readability of the text doc if you added an empty line beneath the headers.

Quoted 4 lines
docs/protvirt.rst new file +4 -0
+The memory and most of the register contents of Protected Virtual
+Machines (PVMs) are inaccessible to the hypervisor, effectively
+prohibiting VM introspection when the VM is running. At rest, PVMs are
+encrypted and can only be decrypted by the firmware of specific IBM Z

docs/protvirt.rst line 16

Quoted 10 lines
docs/protvirt.rst new file +10 -0
+machines.
+
+
+Prerequisites
+-------------
+To run PVMs, you need to have a machine with the Protected
+Virtualization feature, which is indicated by the Ultravisor Call
+facility (stfle bit 158). This is a KVM only feature, therefore you
+need a KVM which is able to support PVMs and activate the Ultravisor
+initialization by setting "prot_virt=1" on the kernel command line.

prot_virt=1, so that it gets rendered as a literal in html?

docs/protvirt.rst line 18

Quoted 2 lines
docs/protvirt.rst new file +2 -0
+
+If those requirements are met, the capability "KVM_CAP_S390_PROTECTED"

KVM_CAP_S390_PROTECTED

docs/protvirt.rst line 26

Quoted 8 lines
docs/protvirt.rst new file +8 -0
+will indicate that KVM can support PVMs on that LPAR.
+
+
+QEMU Settings
+-------------
+To indicate to the VM that it can move into protected mode, the
+"Unpack facility" (stfle bit 161) needs to be part of the cpu model of
+the VM.

Add an example invocation here?

docs/protvirt.rst line 29

Quoted 3 lines
docs/protvirt.rst new file +3 -0
+
+All I/O devices need to use the IOMMU.
+Passthrough devices are currently not supported.

s/Passthrough devices/Passthrough (vfio) devices/ ?

docs/protvirt.rst line 36

Quoted 7 lines
docs/protvirt.rst new file +7 -0
+
+Host huge page backings are not supported. The guest however can use
+huge pages as indicated by its facilities.
+
+
+Boot Process
+-----------------

Underlining too long :)

docs/protvirt.rst line 37

Quoted 1 line
docs/protvirt.rst new file +1 -0
+A secure guest image can be booted from disk and using the QEMU

"can be both booted from..." ?

Quoted 6 lines
docs/protvirt.rst new file +6 -0
+command line. Booting from disk is done by the unmodified s390-ccw
+BIOS. I.e., the bootmap is interpreted and a number of components is
+read into memory and control is transferred to one of the components
+(zipl stage3), which does some fixups and then transfers control to
+some program residing in guest memory, which is normally the OS
+kernel. The secure image has another component prepended (stage3a)

docs/protvirt.rst line 53

Quoted 10 lines
docs/protvirt.rst new file +10 -0
+which uses the new diag308 subcodes 8 and 10 to trigger the transition
+into secure mode.
+
+Booting from the command line requires that the file passed
+via -kernel has the same memory layout as would result from the disk
+boot. This memory layout includes the encrypted components (kernel,
+initrd, cmdline), the stage3a loader and metadata. In case this boot
+method is used, the command line options -initrd and -cmdline are
+ineffective.  The preparation of secure guest image is done by a
+program (name tbd) of the s390-tools package.

Hm... do you have an ETA for that tbd program?