Skip to contentSkip to conversation

s390: diagnose 318 info reset and migration support

PATCH In review Collin Walling 5 participants 38 messages May 1, 2019 – May 16, 2019

lore
Share & tools
lore
#0 lore raw

PATCH v4 1/1 s390: diagnose 318 info reset and migration support

DIAGNOSE 0x318 (diag318) is a privileged s390x instruction that must be intercepted by SIE and handled via KVM. Let's introduce some functions to communicate between QEMU and KVM via ioctls. These will be used to get/set the diag318 related information (also known as the "Control Program Code" or "CPC"), as well as check the system if KVM supports handling this instruction.

The availability of this instruction is determined by byte 134, bit 0 of the Read Info block. This coincidentally expands into the space used for CPU entries, which means VMs running with the diag318 capability will have a reduced maximum CPU count. To alleviate this, let's calculate the actual max CPU entry space by subtracting the size of Read Info from the SCCB size then dividing that number by the size of a CPU entry. If this value is less than the value denoted by S390_MAX_CPUS, then let's reduce the max cpus for s390 from 248 to 240 in an effort to anticipate this potentially happening again in the future.

In order to simplify the migration and system reset requirements of the diag318 data, let's introduce it as a device class and include a VMStateDescription.

Diag318 is reset on during modified clear and load normal.

---
 hw/s390x/Makefile.objs       |   1 +
 hw/s390x/diag318.c           | 100 +++++++++++++++++++++++++++++++++++++++++++
 hw/s390x/diag318.h           |  39 +++++++++++++++++
 hw/s390x/s390-virtio-ccw.c   |  23 ++++++++++
 hw/s390x/sclp.c              |   5 +++
 include/hw/s390x/sclp.h      |   2 +
 linux-headers/asm-s390/kvm.h |   4 ++
 target/s390x/kvm-stub.c      |  15 +++++++
 target/s390x/kvm.c           |  32 ++++++++++++++
 target/s390x/kvm_s390x.h     |   3 ++
 10 files changed, 224 insertions(+)
 create mode 100644 hw/s390x/diag318.c
 create mode 100644 hw/s390x/diag318.h
hw/s390x/Makefile.objs +1 -0
index e02ed80..93621dc 100644
@@ -34,3 +34,4 @@ obj-$(CONFIG_KVM) += s390-stattrib-kvm.o
 obj-y += s390-ccw.o
 obj-y += ap-device.o
 obj-y += ap-bridge.o
+obj-y += diag318.o
hw/s390x/diag318.c new file +100 -0

100 lines not on this page · raw message

hw/s390x/diag318.h new file +39 -0
new file mode 100644
index 0000000..c154b0a
@@ -0,0 +1,39 @@
+/*
+ * DIAGNOSE 0x318 functions for reset and migration
+ *
+ * Copyright IBM, Corp. 2019
+ *
+ * Authors:
+ *  Collin Walling <[email protected]>
+ *
+ * This work is licensed under the terms of the GNU GPL, version 2 or (at your
+ * option) any later version. See the COPYING file in the top-level directory.
+ */
+
+ #ifndef DIAG318_H
+ #define DIAG318_H
+
+#include "qemu/osdep.h"
+#include "hw/qdev.h"
+
+#define TYPE_S390_DIAG318 "diag318"
+#define DIAG318(obj) \
+    OBJECT_CHECK(DIAG318State, (obj), TYPE_S390_DIAG318)
+
+typedef struct DIAG318State {
+    /*< private >*/
+    DeviceState parent_obj;
+
+    /*< public >*/
+    uint64_t cpc;
+    bool enabled;
+} DIAG318State;
+
+typedef struct DIAG318Class {
+    /*< private >*/
+    DeviceClass parent_class;
+
+    /*< public >*/
+} DIAG318Class;
+
+#endif /* DIAG318_H */
hw/s390x/s390-virtio-ccw.c +23 -0
index d11069b..44a424b 100644
@@ -36,6 +36,7 @@
 #include "cpu_models.h"
 #include "hw/nmi.h"
 #include "hw/s390x/tod.h"
+#include "hw/s390x/diag318.h"
 
 S390CPU *s390_cpu_addr2state(uint16_t cpu_addr)
 {
@@ -92,6 +93,7 @@ static const char *const reset_dev_types[] = {
     "s390-sclp-event-facility",
     "s390-flic",
     "diag288",
+    TYPE_S390_DIAG318,
 };
 
 static void subsystem_reset(void)
@@ -246,6 +248,17 @@ static void s390_create_sclpconsole(const char *type, Chardev *chardev)
     qdev_init_nofail(dev);
 }
 
+static void s390_init_diag318(void)
+{
+    Object *new = object_new(TYPE_S390_DIAG318);
+    DeviceState *dev = DEVICE(new);
+
+    object_property_add_child(qdev_get_machine(), TYPE_S390_DIAG318,
+                              new, NULL);
+    object_unref(new);
+    qdev_init_nofail(dev);
+}
+
 static void ccw_init(MachineState *machine)
 {
     int ret;
@@ -302,6 +315,8 @@ static void ccw_init(MachineState *machine)
 
     /* init the TOD clock */
     s390_init_tod();
+
+    s390_init_diag318();
 }
 
 static void s390_cpu_plug(HotplugHandler *hotplug_dev,
@@ -570,6 +585,7 @@ static void machine_set_loadparm(Object *obj, const char *val, Error **errp)
         ms->loadparm[i] = ' '; /* pad right with spaces */
     }
 }
+
 static inline void s390_machine_initfn(Object *obj)
 {
     object_property_add_bool(obj, "aes-key-wrap",
@@ -652,6 +668,13 @@ static void ccw_machine_4_0_instance_options(MachineState *machine)
 
 static void ccw_machine_4_0_class_options(MachineClass *mc)
 {
+    /*
+     * Read Info might reveal more bytes used to detect facilities, thus
+     * reducing the number of CPU entries. Let's reduce the max CPUs by
+     * an arbitrary number in effort to anticipate future facility bytes.
+     */
+    if ((SCCB_SIZE - sizeof(ReadInfo)) / sizeof(CPUEntry) < S390_MAX_CPUS)
+        mc->max_cpus = S390_MAX_CPUS - 8;
 }
 DEFINE_CCW_MACHINE(4_0, "4.0", true);
 
hw/s390x/sclp.c +5 -0
index 4510a80..9cfa188 100644
@@ -22,6 +22,7 @@
 #include "hw/s390x/event-facility.h"
 #include "hw/s390x/s390-pci-bus.h"
 #include "hw/s390x/ipl.h"
+#include "kvm_s390x.h"
 
 static inline SCLPDevice *get_sclp_device(void)
 {
@@ -74,6 +75,10 @@ static void read_SCP_info(SCLPDevice *sclp, SCCB *sccb)
     s390_get_feat_block(S390_FEAT_TYPE_SCLP_CONF_CHAR_EXT,
                          read_info->conf_char_ext);
 
+    /* Enable diag318 for guest if KVM supports emulation */
+    if (kvm_s390_has_diag318())
+        read_info->fac134 = 0x80;
+
     read_info->facilities = cpu_to_be64(SCLP_HAS_CPU_INFO |
                                         SCLP_HAS_IOA_RECONFIG);
 
include/hw/s390x/sclp.h +2 -0
index f9db243..d47e10a 100644
@@ -133,6 +133,8 @@ typedef struct ReadInfo {
     uint16_t highest_cpu;
     uint8_t  _reserved5[124 - 122];     /* 122-123 */
     uint32_t hmfai;
+    uint8_t  _reserved7[134 - 128];     /* 128-133 */
+    uint8_t  fac134;
     struct CPUEntry entries[0];
 } QEMU_PACKED ReadInfo;
 
linux-headers/asm-s390/kvm.h +4 -0
index 0265482..735f5a4 100644
@@ -74,6 +74,7 @@ struct kvm_s390_io_adapter_req {
 #define KVM_S390_VM_CRYPTO		2
 #define KVM_S390_VM_CPU_MODEL		3
 #define KVM_S390_VM_MIGRATION		4
+#define KVM_S390_VM_MISC		5
 
 /* kvm attributes for mem_ctrl */
 #define KVM_S390_VM_MEM_ENABLE_CMMA	0
@@ -168,6 +169,9 @@ struct kvm_s390_vm_cpu_subfunc {
 #define KVM_S390_VM_MIGRATION_START	1
 #define KVM_S390_VM_MIGRATION_STATUS	2
 
+/* kvm attributes for KVM_S390_VM_MISC */
+#define KVM_S390_VM_MISC_CPC		0
+
 /* for KVM_GET_REGS and KVM_SET_REGS */
 struct kvm_regs {
 	/* general purpose regs for s390 */
target/s390x/kvm-stub.c +15 -0
index bf7795e..7861ccd 100644
@@ -104,3 +104,18 @@ void kvm_s390_stop_interrupt(S390CPU *cpu)
 void kvm_s390_restart_interrupt(S390CPU *cpu)
 {
 }
+
+int kvm_s390_get_cpc(uint64_t *cpc)
+{
+    return 0;
+}
+
+int kvm_s390_set_cpc(uint64_t cpc)
+{
+    return 0;
+}
+
+bool kvm_s390_has_diag318(void)
+{
+    return false;
+}
target/s390x/kvm.c +32 -0
index 19530fb..225e516 100644
@@ -747,6 +747,38 @@ int kvm_s390_set_clock_ext(uint8_t tod_high, uint64_t tod_low)
     return kvm_vm_ioctl(kvm_state, KVM_SET_DEVICE_ATTR, &attr);
 }
 
+int kvm_s390_get_cpc(uint64_t *cpc)
+{
+    struct kvm_device_attr attr = {
+        .group = KVM_S390_VM_MISC,
+        .attr = KVM_S390_VM_MISC_CPC,
+        .addr = (uint64_t)cpc,
+    };
+
+    return kvm_vm_ioctl(kvm_state, KVM_GET_DEVICE_ATTR, &attr);
+}
+
+int kvm_s390_set_cpc(uint64_t cpc)
+{
+    struct kvm_device_attr attr = {
+        .group = KVM_S390_VM_MISC,
+        .attr = KVM_S390_VM_MISC_CPC,
+        .addr = (uint64_t)&cpc,
+    };
+
+    return kvm_vm_ioctl(kvm_state, KVM_SET_DEVICE_ATTR, &attr);
+}
+
+bool kvm_s390_has_diag318(void)
+{
+    struct kvm_device_attr attr = {
+        .group = KVM_S390_VM_MISC,
+        .attr = KVM_S390_VM_MISC_CPC,
+    };
+
+    return kvm_vm_ioctl(kvm_state, KVM_HAS_DEVICE_ATTR, &attr) == 0;
+}
+
 /**
  * kvm_s390_mem_op:
  * @addr:      the logical start address in guest memory
target/s390x/kvm_s390x.h +3 -0
index 6e52287..53f165f 100644
@@ -29,6 +29,9 @@ int kvm_s390_get_clock(uint8_t *tod_high, uint64_t *tod_clock);
 int kvm_s390_get_clock_ext(uint8_t *tod_high, uint64_t *tod_clock);
 int kvm_s390_set_clock(uint8_t tod_high, uint64_t tod_clock);
 int kvm_s390_set_clock_ext(uint8_t tod_high, uint64_t tod_clock);
+int kvm_s390_get_cpc(uint64_t *cpc);
+int kvm_s390_set_cpc(uint64_t cpc);
+bool kvm_s390_has_diag318(void);
 void kvm_s390_enable_css_support(S390CPU *cpu);
 int kvm_s390_assign_subch_ioeventfd(EventNotifier *notifier, uint32_t sch,
                                     int vq, bool assign);
Signature · 1 line
2.7.4
#1 lore raw
Quoted 69 lines · Collin Walling

DIAGNOSE 0x318 (diag318) is a privileged s390x instruction that must be intercepted by SIE and handled via KVM. Let's introduce some functions to communicate between QEMU and KVM via ioctls. These will be used to get/set the diag318 related information (also known as the "Control Program Code" or "CPC"), as well as check the system if KVM supports handling this instruction.

The availability of this instruction is determined by byte 134, bit 0 of the Read Info block. This coincidentally expands into the space used for CPU entries, which means VMs running with the diag318 capability will have a reduced maximum CPU count. To alleviate this, let's calculate the actual max CPU entry space by subtracting the size of Read Info from the SCCB size then dividing that number by the size of a CPU entry. If this value is less than the value denoted by S390_MAX_CPUS, then let's reduce the max cpus for s390 from 248 to 240 in an effort to anticipate this potentially happening again in the future.

In order to simplify the migration and system reset requirements of the diag318 data, let's introduce it as a device class and include a VMStateDescription.

Diag318 is reset on during modified clear and load normal.

---
 hw/s390x/Makefile.objs       |   1 +
 hw/s390x/diag318.c           | 100 +++++++++++++++++++++++++++++++++++++++++++
 hw/s390x/diag318.h           |  39 +++++++++++++++++
 hw/s390x/s390-virtio-ccw.c   |  23 ++++++++++
 hw/s390x/sclp.c              |   5 +++
 include/hw/s390x/sclp.h      |   2 +
 linux-headers/asm-s390/kvm.h |   4 ++
 target/s390x/kvm-stub.c      |  15 +++++++
 target/s390x/kvm.c           |  32 ++++++++++++++
 target/s390x/kvm_s390x.h     |   3 ++
 10 files changed, 224 insertions(+)
 create mode 100644 hw/s390x/diag318.c
 create mode 100644 hw/s390x/diag318.h
hw/s390x/Makefile.objs +1 -0
index e02ed80..93621dc 100644
@@ -34,3 +34,4 @@ obj-$(CONFIG_KVM) += s390-stattrib-kvm.o
 obj-y += s390-ccw.o
 obj-y += ap-device.o
 obj-y += ap-bridge.o
+obj-y += diag318.o
hw/s390x/diag318.c new file +15 -0
new file mode 100644
index 0000000..94b44da
@@ -0,0 +1,100 @@
+/*
+ * DIAGNOSE 0x318 functions for reset and migration
+ *
+ * Copyright IBM, Corp. 2019
+ *
+ * Authors:
+ *  Collin Walling <[email protected]>
+ *
+ * This work is licensed under the terms of the GNU GPL, version 2 or (at your
+ * option) any later version. See the COPYING file in the top-level directory.
+ */
+
+#include "hw/s390x/diag318.h"
+#include "qapi/error.h"
+#include "kvm_s390x.h"

hw/s390x/diag318.c line 25

Quoted 10 lines
hw/s390x/diag318.c new file +10 -0
+#include "sysemu/kvm.h"
+
+static int diag318_post_load(void *opaque, int version_id)
+{
+    DIAG318State *d = opaque;
+
+    kvm_s390_set_cpc(d->cpc);
+
+    /* It is not necessary to retain a copy of the cpc after migration. */
+    d->cpc = 0;

But we also do not need to zero it out? Can't you just drop these 2 lines?

Quoted 8 lines
hw/s390x/diag318.c new file +8 -0
+
+    return 0;
+}
+
+static int diag318_pre_save(void *opaque)
+{
+    DIAG318State *d = opaque;
+

hw/s390x/diag318.c line 43

Quoted 10 lines
hw/s390x/diag318.c new file +10 -0
+    kvm_s390_get_cpc(&d->cpc);
+    return 0;
+}
+
+static bool diag318_needed(void *opaque)
+{
+    DIAG318State *d = opaque;
+
+    return d->enabled;
+}

I would like to have a cpumodel entry that allows to disable that feature. And we should then check for this.

Quoted 11 lines
hw/s390x/diag318.c new file +11 -0
+
+const VMStateDescription vmstate_diag318 = {
+    .name = "vmstate_diag318",
+    .post_load = diag318_post_load,
+    .pre_save = diag318_pre_save,
+    .version_id = 1,
+    .minimum_version_id = 1,
+    .needed = diag318_needed,
+    .fields = (VMStateField[]) {
+        VMSTATE_UINT64(cpc, DIAG318State),
+        VMSTATE_END_OF_LIST()

hw/s390x/diag318.c line 64

Quoted 10 lines
hw/s390x/diag318.c new file +10 -0
+    }
+};
+
+static void s390_diag318_realize(DeviceState *dev, Error **errp)
+{
+    DIAG318State *d = DIAG318(dev);
+
+    if (kvm_s390_has_diag318()) {
+        d->enabled = true;
+    }

same here -> cpumodel
Then we can get rid of the enabled bool.

Quoted 137 lines
hw/s390x/diag318.c new file +36 -0
+}
+
+static void s390_diag318_reset(DeviceState *dev)
+{
+    DIAG318State *d = DIAG318(dev);
+
+    if (d->enabled) {
+        kvm_s390_set_cpc(0);
+    }
+}
+
+static void s390_diag318_class_init(ObjectClass *klass, void *data)
+{
+    DeviceClass *dc = DEVICE_CLASS(klass);
+
+    dc->realize = s390_diag318_realize;
+    dc->reset = s390_diag318_reset;
+    dc->vmsd = &vmstate_diag318;
+    dc->hotpluggable = false;
+    /* Reason: Set automatically during IPL */
+    dc->user_creatable = false;
+}
+
+static const TypeInfo s390_diag318_info = {
+    .class_init = s390_diag318_class_init,
+    .parent = TYPE_DEVICE,
+    .name = TYPE_S390_DIAG318,
+    .instance_size = sizeof(DIAG318State),
+};
+
+static void s390_diag318_register_types(void)
+{
+    type_register_static(&s390_diag318_info);
+}
+
+type_init(s390_diag318_register_types)
hw/s390x/diag318.h new file +39 -0
new file mode 100644
index 0000000..c154b0a
@@ -0,0 +1,39 @@
+/*
+ * DIAGNOSE 0x318 functions for reset and migration
+ *
+ * Copyright IBM, Corp. 2019
+ *
+ * Authors:
+ *  Collin Walling <[email protected]>
+ *
+ * This work is licensed under the terms of the GNU GPL, version 2 or (at your
+ * option) any later version. See the COPYING file in the top-level directory.
+ */
+
+ #ifndef DIAG318_H
+ #define DIAG318_H
+
+#include "qemu/osdep.h"
+#include "hw/qdev.h"
+
+#define TYPE_S390_DIAG318 "diag318"
+#define DIAG318(obj) \
+    OBJECT_CHECK(DIAG318State, (obj), TYPE_S390_DIAG318)
+
+typedef struct DIAG318State {
+    /*< private >*/
+    DeviceState parent_obj;
+
+    /*< public >*/
+    uint64_t cpc;
+    bool enabled;
+} DIAG318State;
+
+typedef struct DIAG318Class {
+    /*< private >*/
+    DeviceClass parent_class;
+
+    /*< public >*/
+} DIAG318Class;
+
+#endif /* DIAG318_H */
hw/s390x/s390-virtio-ccw.c +16 -0
index d11069b..44a424b 100644
@@ -36,6 +36,7 @@
 #include "cpu_models.h"
 #include "hw/nmi.h"
 #include "hw/s390x/tod.h"
+#include "hw/s390x/diag318.h"
 
 S390CPU *s390_cpu_addr2state(uint16_t cpu_addr)
 {
@@ -92,6 +93,7 @@ static const char *const reset_dev_types[] = {
     "s390-sclp-event-facility",
     "s390-flic",
     "diag288",
+    TYPE_S390_DIAG318,
 };
 
 static void subsystem_reset(void)
@@ -246,6 +248,17 @@ static void s390_create_sclpconsole(const char *type, Chardev *chardev)
     qdev_init_nofail(dev);
 }
 
+static void s390_init_diag318(void)
+{
+    Object *new = object_new(TYPE_S390_DIAG318);
+    DeviceState *dev = DEVICE(new);
+
+    object_property_add_child(qdev_get_machine(), TYPE_S390_DIAG318,
+                              new, NULL);
+    object_unref(new);
+    qdev_init_nofail(dev);
+}
+
 static void ccw_init(MachineState *machine)
 {
     int ret;
@@ -302,6 +315,8 @@ static void ccw_init(MachineState *machine)
 
     /* init the TOD clock */
     s390_init_tod();
+
+    s390_init_diag318();
 }
 
 static void s390_cpu_plug(HotplugHandler *hotplug_dev,
@@ -570,6 +585,7 @@ static void machine_set_loadparm(Object *obj, const char *val, Error **errp)
         ms->loadparm[i] = ' '; /* pad right with spaces */
     }
 }
+
 static inline void s390_machine_initfn(Object *obj)
 {
     object_property_add_bool(obj, "aes-key-wrap",

hw/s390x/s390-virtio-ccw.c static void ccw_machine_4_0_instance_options(MachineState *machine) · line 677

Quoted 10 lines
hw/s390x/s390-virtio-ccw.c +7 -0
@@ -652,6 +668,13 @@ static void ccw_machine_4_0_instance_options(MachineState *machine)
 
 static void ccw_machine_4_0_class_options(MachineClass *mc)
 {
+    /*
+     * Read Info might reveal more bytes used to detect facilities, thus
+     * reducing the number of CPU entries. Let's reduce the max CPUs by
+     * an arbitrary number in effort to anticipate future facility bytes.
+     */
+    if ((SCCB_SIZE - sizeof(ReadInfo)) / sizeof(CPUEntry) < S390_MAX_CPUS)
+        mc->max_cpus = S390_MAX_CPUS - 8;

Maybe this should depend on the presence of this feature in the cpumodel?

Quoted 16 lines
hw/s390x/s390-virtio-ccw.c +0 -0
 }
 DEFINE_CCW_MACHINE(4_0, "4.0", true);
 
hw/s390x/sclp.c +1 -0
index 4510a80..9cfa188 100644
@@ -22,6 +22,7 @@
 #include "hw/s390x/event-facility.h"
 #include "hw/s390x/s390-pci-bus.h"
 #include "hw/s390x/ipl.h"
+#include "kvm_s390x.h"
 
 static inline SCLPDevice *get_sclp_device(void)
 {

hw/s390x/sclp.c static void read_SCP_info(SCLPDevice *sclp, SCCB *sccb) · line 80

Quoted 6 lines
hw/s390x/sclp.c +3 -0
@@ -74,6 +75,10 @@ static void read_SCP_info(SCLPDevice *sclp, SCCB *sccb)
     s390_get_feat_block(S390_FEAT_TYPE_SCLP_CONF_CHAR_EXT,
                          read_info->conf_char_ext);
 
+    /* Enable diag318 for guest if KVM supports emulation */
+    if (kvm_s390_has_diag318())
+        read_info->fac134 = 0x80;

I think we should rather make this part of the cpumodel

Quoted 22 lines
hw/s390x/sclp.c +1 -0
+
     read_info->facilities = cpu_to_be64(SCLP_HAS_CPU_INFO |
                                         SCLP_HAS_IOA_RECONFIG);
 
include/hw/s390x/sclp.h +2 -0
index f9db243..d47e10a 100644
@@ -133,6 +133,8 @@ typedef struct ReadInfo {
     uint16_t highest_cpu;
     uint8_t  _reserved5[124 - 122];     /* 122-123 */
     uint32_t hmfai;
+    uint8_t  _reserved7[134 - 128];     /* 128-133 */
+    uint8_t  fac134;
     struct CPUEntry entries[0];
 } QEMU_PACKED ReadInfo;
 
linux-headers/asm-s390/kvm.h +0 -0
index 0265482..735f5a4 100644

linux-headers/asm-s390/kvm.h struct kvm_s390_io_adapter_req { · line 77

Quoted 4 lines
linux-headers/asm-s390/kvm.h +1 -0
@@ -74,6 +74,7 @@ struct kvm_s390_io_adapter_req {
 #define KVM_S390_VM_CRYPTO		2
 #define KVM_S390_VM_CPU_MODEL		3
 #define KVM_S390_VM_MIGRATION		4
+#define KVM_S390_VM_MISC		5

Can you split that out into a separate patch (header sync). (We also need the relevant kernel patch first, I could not find it right now).

Quoted 94 lines
linux-headers/asm-s390/kvm.h +3 -0
 /* kvm attributes for mem_ctrl */
 #define KVM_S390_VM_MEM_ENABLE_CMMA	0
@@ -168,6 +169,9 @@ struct kvm_s390_vm_cpu_subfunc {
 #define KVM_S390_VM_MIGRATION_START	1
 #define KVM_S390_VM_MIGRATION_STATUS	2
 
+/* kvm attributes for KVM_S390_VM_MISC */
+#define KVM_S390_VM_MISC_CPC		0
+
 /* for KVM_GET_REGS and KVM_SET_REGS */
 struct kvm_regs {
 	/* general purpose regs for s390 */
target/s390x/kvm-stub.c +15 -0
index bf7795e..7861ccd 100644
@@ -104,3 +104,18 @@ void kvm_s390_stop_interrupt(S390CPU *cpu)
 void kvm_s390_restart_interrupt(S390CPU *cpu)
 {
 }
+
+int kvm_s390_get_cpc(uint64_t *cpc)
+{
+    return 0;
+}
+
+int kvm_s390_set_cpc(uint64_t cpc)
+{
+    return 0;
+}
+
+bool kvm_s390_has_diag318(void)
+{
+    return false;
+}
target/s390x/kvm.c +32 -0
index 19530fb..225e516 100644
@@ -747,6 +747,38 @@ int kvm_s390_set_clock_ext(uint8_t tod_high, uint64_t tod_low)
     return kvm_vm_ioctl(kvm_state, KVM_SET_DEVICE_ATTR, &attr);
 }
 
+int kvm_s390_get_cpc(uint64_t *cpc)
+{
+    struct kvm_device_attr attr = {
+        .group = KVM_S390_VM_MISC,
+        .attr = KVM_S390_VM_MISC_CPC,
+        .addr = (uint64_t)cpc,
+    };
+
+    return kvm_vm_ioctl(kvm_state, KVM_GET_DEVICE_ATTR, &attr);
+}
+
+int kvm_s390_set_cpc(uint64_t cpc)
+{
+    struct kvm_device_attr attr = {
+        .group = KVM_S390_VM_MISC,
+        .attr = KVM_S390_VM_MISC_CPC,
+        .addr = (uint64_t)&cpc,
+    };
+
+    return kvm_vm_ioctl(kvm_state, KVM_SET_DEVICE_ATTR, &attr);
+}
+
+bool kvm_s390_has_diag318(void)
+{
+    struct kvm_device_attr attr = {
+        .group = KVM_S390_VM_MISC,
+        .attr = KVM_S390_VM_MISC_CPC,
+    };
+
+    return kvm_vm_ioctl(kvm_state, KVM_HAS_DEVICE_ATTR, &attr) == 0;
+}
+
 /**
  * kvm_s390_mem_op:
  * @addr:      the logical start address in guest memory
target/s390x/kvm_s390x.h +3 -0
index 6e52287..53f165f 100644
@@ -29,6 +29,9 @@ int kvm_s390_get_clock(uint8_t *tod_high, uint64_t *tod_clock);
 int kvm_s390_get_clock_ext(uint8_t *tod_high, uint64_t *tod_clock);
 int kvm_s390_set_clock(uint8_t tod_high, uint64_t tod_clock);
 int kvm_s390_set_clock_ext(uint8_t tod_high, uint64_t tod_clock);
+int kvm_s390_get_cpc(uint64_t *cpc);
+int kvm_s390_set_cpc(uint64_t cpc);
+bool kvm_s390_has_diag318(void);
 void kvm_s390_enable_css_support(S390CPU *cpu);
 int kvm_s390_assign_subch_ioeventfd(EventNotifier *notifier, uint32_t sch,
                                     int vq, bool assign);
#2 lore raw
Quoted 301 lines · Christian Borntraeger
Quoted 69 lines · Collin Walling

DIAGNOSE 0x318 (diag318) is a privileged s390x instruction that must be intercepted by SIE and handled via KVM. Let's introduce some functions to communicate between QEMU and KVM via ioctls. These will be used to get/set the diag318 related information (also known as the "Control Program Code" or "CPC"), as well as check the system if KVM supports handling this instruction.

The availability of this instruction is determined by byte 134, bit 0 of the Read Info block. This coincidentally expands into the space used for CPU entries, which means VMs running with the diag318 capability will have a reduced maximum CPU count. To alleviate this, let's calculate the actual max CPU entry space by subtracting the size of Read Info from the SCCB size then dividing that number by the size of a CPU entry. If this value is less than the value denoted by S390_MAX_CPUS, then let's reduce the max cpus for s390 from 248 to 240 in an effort to anticipate this potentially happening again in the future.

In order to simplify the migration and system reset requirements of the diag318 data, let's introduce it as a device class and include a VMStateDescription.

Diag318 is reset on during modified clear and load normal.

---
 hw/s390x/Makefile.objs       |   1 +
 hw/s390x/diag318.c           | 100 +++++++++++++++++++++++++++++++++++++++++++
 hw/s390x/diag318.h           |  39 +++++++++++++++++
 hw/s390x/s390-virtio-ccw.c   |  23 ++++++++++
 hw/s390x/sclp.c              |   5 +++
 include/hw/s390x/sclp.h      |   2 +
 linux-headers/asm-s390/kvm.h |   4 ++
 target/s390x/kvm-stub.c      |  15 +++++++
 target/s390x/kvm.c           |  32 ++++++++++++++
 target/s390x/kvm_s390x.h     |   3 ++
 10 files changed, 224 insertions(+)
 create mode 100644 hw/s390x/diag318.c
 create mode 100644 hw/s390x/diag318.h
hw/s390x/Makefile.objs +1 -0
index e02ed80..93621dc 100644
@@ -34,3 +34,4 @@ obj-$(CONFIG_KVM) += s390-stattrib-kvm.o
 obj-y += s390-ccw.o
 obj-y += ap-device.o
 obj-y += ap-bridge.o
+obj-y += diag318.o
hw/s390x/diag318.c new file +15 -0
new file mode 100644
index 0000000..94b44da
@@ -0,0 +1,100 @@
+/*
+ * DIAGNOSE 0x318 functions for reset and migration
+ *
+ * Copyright IBM, Corp. 2019
+ *
+ * Authors:
+ *  Collin Walling <[email protected]>
+ *
+ * This work is licensed under the terms of the GNU GPL, version 2 or (at your
+ * option) any later version. See the COPYING file in the top-level directory.
+ */
+
+#include "hw/s390x/diag318.h"
+#include "qapi/error.h"
+#include "kvm_s390x.h"

hw/s390x/diag318.c line 25

Quoted 10 lines
hw/s390x/diag318.c new file +10 -0
+#include "sysemu/kvm.h"
+
+static int diag318_post_load(void *opaque, int version_id)
+{
+    DIAG318State *d = opaque;
+
+    kvm_s390_set_cpc(d->cpc);
+
+    /* It is not necessary to retain a copy of the cpc after migration. */
+    d->cpc = 0;

But we also do not need to zero it out? Can't you just drop these 2 lines?

Quoted 8 lines
hw/s390x/diag318.c new file +8 -0
+
+    return 0;
+}
+
+static int diag318_pre_save(void *opaque)
+{
+    DIAG318State *d = opaque;
+

hw/s390x/diag318.c line 43

Quoted 10 lines
hw/s390x/diag318.c new file +10 -0
+    kvm_s390_get_cpc(&d->cpc);
+    return 0;
+}
+
+static bool diag318_needed(void *opaque)
+{
+    DIAG318State *d = opaque;
+
+    return d->enabled;
+}

I would like to have a cpumodel entry that allows to disable that feature. And we should then check for this.

Quoted 11 lines
hw/s390x/diag318.c new file +11 -0
+
+const VMStateDescription vmstate_diag318 = {
+    .name = "vmstate_diag318",
+    .post_load = diag318_post_load,
+    .pre_save = diag318_pre_save,
+    .version_id = 1,
+    .minimum_version_id = 1,
+    .needed = diag318_needed,
+    .fields = (VMStateField[]) {
+        VMSTATE_UINT64(cpc, DIAG318State),
+        VMSTATE_END_OF_LIST()

hw/s390x/diag318.c line 64

Quoted 10 lines
hw/s390x/diag318.c new file +10 -0
+    }
+};
+
+static void s390_diag318_realize(DeviceState *dev, Error **errp)
+{
+    DIAG318State *d = DIAG318(dev);
+
+    if (kvm_s390_has_diag318()) {
+        d->enabled = true;
+    }

same here -> cpumodel
Then we can get rid of the enabled bool.

Quoted 137 lines
hw/s390x/diag318.c new file +36 -0
+}
+
+static void s390_diag318_reset(DeviceState *dev)
+{
+    DIAG318State *d = DIAG318(dev);
+
+    if (d->enabled) {
+        kvm_s390_set_cpc(0);
+    }
+}
+
+static void s390_diag318_class_init(ObjectClass *klass, void *data)
+{
+    DeviceClass *dc = DEVICE_CLASS(klass);
+
+    dc->realize = s390_diag318_realize;
+    dc->reset = s390_diag318_reset;
+    dc->vmsd = &vmstate_diag318;
+    dc->hotpluggable = false;
+    /* Reason: Set automatically during IPL */
+    dc->user_creatable = false;
+}
+
+static const TypeInfo s390_diag318_info = {
+    .class_init = s390_diag318_class_init,
+    .parent = TYPE_DEVICE,
+    .name = TYPE_S390_DIAG318,
+    .instance_size = sizeof(DIAG318State),
+};
+
+static void s390_diag318_register_types(void)
+{
+    type_register_static(&s390_diag318_info);
+}
+
+type_init(s390_diag318_register_types)
hw/s390x/diag318.h new file +39 -0
new file mode 100644
index 0000000..c154b0a
@@ -0,0 +1,39 @@
+/*
+ * DIAGNOSE 0x318 functions for reset and migration
+ *
+ * Copyright IBM, Corp. 2019
+ *
+ * Authors:
+ *  Collin Walling <[email protected]>
+ *
+ * This work is licensed under the terms of the GNU GPL, version 2 or (at your
+ * option) any later version. See the COPYING file in the top-level directory.
+ */
+
+ #ifndef DIAG318_H
+ #define DIAG318_H
+
+#include "qemu/osdep.h"
+#include "hw/qdev.h"
+
+#define TYPE_S390_DIAG318 "diag318"
+#define DIAG318(obj) \
+    OBJECT_CHECK(DIAG318State, (obj), TYPE_S390_DIAG318)
+
+typedef struct DIAG318State {
+    /*< private >*/
+    DeviceState parent_obj;
+
+    /*< public >*/
+    uint64_t cpc;
+    bool enabled;
+} DIAG318State;
+
+typedef struct DIAG318Class {
+    /*< private >*/
+    DeviceClass parent_class;
+
+    /*< public >*/
+} DIAG318Class;
+
+#endif /* DIAG318_H */
hw/s390x/s390-virtio-ccw.c +16 -0
index d11069b..44a424b 100644
@@ -36,6 +36,7 @@
 #include "cpu_models.h"
 #include "hw/nmi.h"
 #include "hw/s390x/tod.h"
+#include "hw/s390x/diag318.h"
 
 S390CPU *s390_cpu_addr2state(uint16_t cpu_addr)
 {
@@ -92,6 +93,7 @@ static const char *const reset_dev_types[] = {
     "s390-sclp-event-facility",
     "s390-flic",
     "diag288",
+    TYPE_S390_DIAG318,
 };
 
 static void subsystem_reset(void)
@@ -246,6 +248,17 @@ static void s390_create_sclpconsole(const char *type, Chardev *chardev)
     qdev_init_nofail(dev);
 }
 
+static void s390_init_diag318(void)
+{
+    Object *new = object_new(TYPE_S390_DIAG318);
+    DeviceState *dev = DEVICE(new);
+
+    object_property_add_child(qdev_get_machine(), TYPE_S390_DIAG318,
+                              new, NULL);
+    object_unref(new);
+    qdev_init_nofail(dev);
+}
+
 static void ccw_init(MachineState *machine)
 {
     int ret;
@@ -302,6 +315,8 @@ static void ccw_init(MachineState *machine)
 
     /* init the TOD clock */
     s390_init_tod();
+
+    s390_init_diag318();
 }
 
 static void s390_cpu_plug(HotplugHandler *hotplug_dev,
@@ -570,6 +585,7 @@ static void machine_set_loadparm(Object *obj, const char *val, Error **errp)
         ms->loadparm[i] = ' '; /* pad right with spaces */
     }
 }
+
 static inline void s390_machine_initfn(Object *obj)
 {
     object_property_add_bool(obj, "aes-key-wrap",

hw/s390x/s390-virtio-ccw.c static void ccw_machine_4_0_instance_options(MachineState *machine) · line 677

Quoted 10 lines
hw/s390x/s390-virtio-ccw.c +7 -0
@@ -652,6 +668,13 @@ static void ccw_machine_4_0_instance_options(MachineState *machine)
 
 static void ccw_machine_4_0_class_options(MachineClass *mc)
 {
+    /*
+     * Read Info might reveal more bytes used to detect facilities, thus
+     * reducing the number of CPU entries. Let's reduce the max CPUs by
+     * an arbitrary number in effort to anticipate future facility bytes.
+     */
+    if ((SCCB_SIZE - sizeof(ReadInfo)) / sizeof(CPUEntry) < S390_MAX_CPUS)
+        mc->max_cpus = S390_MAX_CPUS - 8;

Maybe this should depend on the presence of this feature in the cpumodel?

Quoted 19 lines
hw/s390x/s390-virtio-ccw.c +0 -0
 }
 DEFINE_CCW_MACHINE(4_0, "4.0", true);
 
hw/s390x/sclp.c +1 -0
index 4510a80..9cfa188 100644
@@ -22,6 +22,7 @@
 #include "hw/s390x/event-facility.h"
 #include "hw/s390x/s390-pci-bus.h"
 #include "hw/s390x/ipl.h"
+#include "kvm_s390x.h"
 
 static inline SCLPDevice *get_sclp_device(void)
 {
@@ -74,6 +75,10 @@ static void read_SCP_info(SCLPDevice *sclp, SCCB *sccb)
     s390_get_feat_block(S390_FEAT_TYPE_SCLP_CONF_CHAR_EXT,
                          read_info->conf_char_ext);
 
Quoted 5 lines

hw/s390x/sclp.c static void read_SCP_info(SCLPDevice *sclp, SCCB *sccb) · line 80

Quoted 3 lines
hw/s390x/sclp.c +3 -0
+    /* Enable diag318 for guest if KVM supports emulation */
+    if (kvm_s390_has_diag318())
+        read_info->fac134 = 0x80;

I think we should rather make this part of the cpumodel

+1, had the same idea when skimming over this.

Signature · 3 lines
Thanks,

David / dhildenb
#3 lore raw
In reply to Christian Borntraeger
Quoted 76 lines · Christian Borntraeger
Quoted 73 lines · Collin Walling

DIAGNOSE 0x318 (diag318) is a privileged s390x instruction that must be intercepted by SIE and handled via KVM. Let's introduce some functions to communicate between QEMU and KVM via ioctls. These will be used to get/set the diag318 related information (also known as the "Control Program Code" or "CPC"), as well as check the system if KVM supports handling this instruction.

The availability of this instruction is determined by byte 134, bit 0 of the Read Info block. This coincidentally expands into the space used for CPU entries, which means VMs running with the diag318 capability will have a reduced maximum CPU count. To alleviate this, let's calculate the actual max CPU entry space by subtracting the size of Read Info from the SCCB size then dividing that number by the size of a CPU entry. If this value is less than the value denoted by S390_MAX_CPUS, then let's reduce the max cpus for s390 from 248 to 240 in an effort to anticipate this potentially happening again in the future.

In order to simplify the migration and system reset requirements of the diag318 data, let's introduce it as a device class and include a VMStateDescription.

Diag318 is reset on during modified clear and load normal.

---
  hw/s390x/Makefile.objs       |   1 +
  hw/s390x/diag318.c           | 100 +++++++++++++++++++++++++++++++++++++++++++
  hw/s390x/diag318.h           |  39 +++++++++++++++++
  hw/s390x/s390-virtio-ccw.c   |  23 ++++++++++
  hw/s390x/sclp.c              |   5 +++
  include/hw/s390x/sclp.h      |   2 +
  linux-headers/asm-s390/kvm.h |   4 ++
  target/s390x/kvm-stub.c      |  15 +++++++
  target/s390x/kvm.c           |  32 ++++++++++++++
  target/s390x/kvm_s390x.h     |   3 ++
  10 files changed, 224 insertions(+)

create mode 100644 hw/s390x/diag318.c
create mode 100644 hw/s390x/diag318.h

hw/s390x/Makefile.objs +1 -0
index e02ed80..93621dc 100644
@@ -34,3 +34,4 @@ obj-$(CONFIG_KVM) += s390-stattrib-kvm.o
  obj-y += s390-ccw.o
  obj-y += ap-device.o
  obj-y += ap-bridge.o
+obj-y += diag318.o
hw/s390x/diag318.c new file +19 -0
new file mode 100644
index 0000000..94b44da
@@ -0,0 +1,100 @@
+/*
+ * DIAGNOSE 0x318 functions for reset and migration
+ *
+ * Copyright IBM, Corp. 2019
+ *
+ * Authors:
+ *  Collin Walling <[email protected]>
+ *
+ * This work is licensed under the terms of the GNU GPL, version 2 or (at your
+ * option) any later version. See the COPYING file in the top-level directory.
+ */
+
+#include "hw/s390x/diag318.h"
+#include "qapi/error.h"
+#include "kvm_s390x.h"
+#include "sysemu/kvm.h"
+
+static int diag318_post_load(void *opaque, int version_id)
+{
Quoted 10 lines

hw/s390x/diag318.c line 25

Quoted 6 lines
hw/s390x/diag318.c new file +6 -0
+    DIAG318State *d = opaque;
+
+    kvm_s390_set_cpc(d->cpc);
+
+    /* It is not necessary to retain a copy of the cpc after migration. */
+    d->cpc = 0;

But we also do not need to zero it out? Can't you just drop these 2 lines?

Absolutely. I'll drop them.

Quoted 13 lines
Quoted 13 lines
hw/s390x/diag318.c new file +13 -0
+
+    return 0;
+}
+
+static int diag318_pre_save(void *opaque)
+{
+    DIAG318State *d = opaque;
+
+    kvm_s390_get_cpc(&d->cpc);
+    return 0;
+}
+
+static bool diag318_needed(void *opaque)
Quoted 9 lines

hw/s390x/diag318.c line 43

Quoted 5 lines
hw/s390x/diag318.c new file +5 -0
+{
+    DIAG318State *d = opaque;
+
+    return d->enabled;
+}

I would like to have a cpumodel entry that allows to disable that feature. And we should then check for this.

Noted.

Quoted 16 lines
Quoted 16 lines
hw/s390x/diag318.c new file +16 -0
+
+const VMStateDescription vmstate_diag318 = {
+    .name = "vmstate_diag318",
+    .post_load = diag318_post_load,
+    .pre_save = diag318_pre_save,
+    .version_id = 1,
+    .minimum_version_id = 1,
+    .needed = diag318_needed,
+    .fields = (VMStateField[]) {
+        VMSTATE_UINT64(cpc, DIAG318State),
+        VMSTATE_END_OF_LIST()
+    }
+};
+
+static void s390_diag318_realize(DeviceState *dev, Error **errp)
+{
Quoted 8 lines

hw/s390x/diag318.c line 64

Quoted 5 lines
hw/s390x/diag318.c new file +5 -0
+    DIAG318State *d = DIAG318(dev);
+
+    if (kvm_s390_has_diag318()) {
+        d->enabled = true;
+    }

same here -> cpumodel
Then we can get rid of the enabled bool.

Noted.

[...]

Quoted 12 lines
Quoted 12 lines
hw/s390x/diag318.c new file +1 -0
  static void s390_cpu_plug(HotplugHandler *hotplug_dev,
@@ -570,6 +585,7 @@ static void machine_set_loadparm(Object *obj, const char *val, Error **errp)
          ms->loadparm[i] = ' '; /* pad right with spaces */
      }
  }
+
  static inline void s390_machine_initfn(Object *obj)
  {
      object_property_add_bool(obj, "aes-key-wrap",
@@ -652,6 +668,13 @@ static void ccw_machine_4_0_instance_options(MachineState *machine)
 
Quoted 12 lines

hw/s390x/diag318.c static void ccw_machine_4_0_instance_options(MachineState *machine) · line 677

Quoted 9 lines
hw/s390x/diag318.c new file +7 -0
  static void ccw_machine_4_0_class_options(MachineClass *mc)
  {
+    /*
+     * Read Info might reveal more bytes used to detect facilities, thus
+     * reducing the number of CPU entries. Let's reduce the max CPUs by
+     * an arbitrary number in effort to anticipate future facility bytes.
+     */
+    if ((SCCB_SIZE - sizeof(ReadInfo)) / sizeof(CPUEntry) < S390_MAX_CPUS)
+        mc->max_cpus = S390_MAX_CPUS - 8;

Maybe this should depend on the presence of this feature in the cpumodel?

That's a good idea. This would allow the user to fine tune their VM to either allow the original max 248 _or_ enable the diag318 instruction.

Do you think the reduction to 240 max CPUs still makes sense? I do not know what the future of the RSCPI looks like, but if we end up with more bytes indicating facility bits, we could simply reduce the the max CPUs dynamically depending on which features the user / CPU model enables.

In this case, if diag318 is enabled then reduce the max CPUs to 247. CPU models can protect us WRT migration -- we wouldn't have to concern ourselves if the destination machine can support the same number of CPUs.

In the future, if feature X is enabled, then reduce the max CPUs to as many as we can fit in the remainder of the SCCB.

Quoted 19 lines
Quoted 19 lines
hw/s390x/diag318.c new file +0 -0
  }
  DEFINE_CCW_MACHINE(4_0, "4.0", true);
 
hw/s390x/sclp.c +1 -0
index 4510a80..9cfa188 100644
@@ -22,6 +22,7 @@
  #include "hw/s390x/event-facility.h"
  #include "hw/s390x/s390-pci-bus.h"
  #include "hw/s390x/ipl.h"
+#include "kvm_s390x.h"
 
  static inline SCLPDevice *get_sclp_device(void)
  {
@@ -74,6 +75,10 @@ static void read_SCP_info(SCLPDevice *sclp, SCCB *sccb)
      s390_get_feat_block(S390_FEAT_TYPE_SCLP_CONF_CHAR_EXT,
                           read_info->conf_char_ext);
 
Quoted 7 lines

hw/s390x/sclp.c static void read_SCP_info(SCLPDevice *sclp, SCCB *sccb) · line 80

Quoted 3 lines
hw/s390x/sclp.c +3 -0
+    /* Enable diag318 for guest if KVM supports emulation */
+    if (kvm_s390_has_diag318())
+        read_info->fac134 = 0x80;

I think we should rather make this part of the cpumodel

As per an older discussion, does it still make sense to restrict this feature starting with a particular CPU model (we decided to allow this feature starting with the zEC12 model). We have the opportunity to simply enable this feature for *all* CPU models since this only requires KVM to support emulation.

Pseudo:

if diag318_cpu_feat_is_enabled
if kvm_has_diag318
set facility bit for diag318
else
error "KVM does not support emulation of diag318" <terminate>

The next question would be this: do we want this feature to be enabled by default, or only if it is specified by the user?

Default enabled: VMs will have the diag318 capability, but max CPUs will also be reduced by default. Use will have to explicitly disable diag318 to allow the max 248 CPUs.

User enabled: VMs will have to be configured to enable this capability (users might not realize / care to add this feature themselves). This is as simple as providing the CPU model in a guest XML or command line and providing the diag318feature. The max CPUs will not be altered by default.

Quoted 17 lines
Quoted 17 lines
hw/s390x/sclp.c +1 -0
+
      read_info->facilities = cpu_to_be64(SCLP_HAS_CPU_INFO |
                                          SCLP_HAS_IOA_RECONFIG);
 
include/hw/s390x/sclp.h +2 -0
index f9db243..d47e10a 100644
@@ -133,6 +133,8 @@ typedef struct ReadInfo {
      uint16_t highest_cpu;
      uint8_t  _reserved5[124 - 122];     /* 122-123 */
      uint32_t hmfai;
+    uint8_t  _reserved7[134 - 128];     /* 128-133 */
+    uint8_t  fac134;
      struct CPUEntry entries[0];
  } QEMU_PACKED ReadInfo;
 
Quoted 12 lines

linux-headers/asm-s390/kvm.h struct kvm_s390_io_adapter_req { · line 77

Quoted 9 lines
linux-headers/asm-s390/kvm.h +1 -0
index 0265482..735f5a4 100644
@@ -74,6 +74,7 @@ struct kvm_s390_io_adapter_req {
  #define KVM_S390_VM_CRYPTO		2
  #define KVM_S390_VM_CPU_MODEL		3
  #define KVM_S390_VM_MIGRATION		4
+#define KVM_S390_VM_MISC		5

Can you split that out into a separate patch (header sync). (We also need the relevant kernel patch first, I could not find it right now).

Gah, I keep forgetting about splitting this. Thanks for the reminder :)

[...]

#4 lore raw
Quoted 178 lines · Collin Walling
Quoted 82 lines · Christian Borntraeger
Quoted 79 lines · Collin Walling

DIAGNOSE 0x318 (diag318) is a privileged s390x instruction that must be intercepted by SIE and handled via KVM. Let's introduce some functions to communicate between QEMU and KVM via ioctls. These will be used to get/set the diag318 related information (also known as the "Control Program Code" or "CPC"), as well as check the system if KVM supports handling this instruction.

The availability of this instruction is determined by byte 134, bit 0 of the Read Info block. This coincidentally expands into the space used for CPU entries, which means VMs running with the diag318 capability will have a reduced maximum CPU count. To alleviate this, let's calculate
the actual max CPU entry space by subtracting the size of Read Info from the SCCB size then dividing that number by the size of a CPU entry. If this value is less than the value denoted by S390_MAX_CPUS, then let's reduce the max cpus for s390 from 248 to 240 in an effort to anticipate this potentially happening again in the future.

In order to simplify the migration and system reset requirements of the diag318 data, let's introduce it as a device class and include a VMStateDescription.

Diag318 is reset on during modified clear and load normal.

---
  hw/s390x/Makefile.objs       |   1 + hw/s390x/diag318.c           | 100
+++++++++++++++++++++++++++++++++++++++++++
  hw/s390x/diag318.h           |  39 +++++++++++++++++ hw/s390x/s390-virtio-ccw.c   |  23 ++++++++++ hw/s390x/sclp.c              |   5 +++ include/hw/s390x/sclp.h      |   2 + linux-headers/asm-s390/kvm.h |   4 ++ target/s390x/kvm-stub.c      |  15 +++++++ target/s390x/kvm.c           |  32 ++++++++++++++ target/s390x/kvm_s390x.h     |   3 ++

  10 files changed, 224 insertions(+)

  create mode 100644 hw/s390x/diag318.c
  create mode 100644 hw/s390x/diag318.h

hw/s390x/Makefile.objs +0 -0
index e02ed80..93621dc 100644
@@ -34,3 +34,4 @@ obj-$(CONFIG_KVM) += s390-stattrib-kvm.o

  obj-y += s390-ccw.o
  obj-y += ap-device.o
  obj-y += ap-bridge.o

hw/s390x/Makefile.objs +1 -0
+obj-y += diag318.o
hw/s390x/diag318.c new file +9 -0
new file mode 100644
index 0000000..94b44da
@@ -0,0 +1,100 @@
+/*
+ * DIAGNOSE 0x318 functions for reset and migration
+ *
+ * Copyright IBM, Corp. 2019
+ *
+ * Authors:
+ *  Collin Walling <[email protected]>
+ *
+ * This work is licensed under the terms of the GNU GPL, version 2

or (at your
+ * option) any later version. See the COPYING file in the top-level directory.
+ */

hw/s390x/diag318.c new file +10 -0
+
+#include "hw/s390x/diag318.h"
+#include "qapi/error.h"
+#include "kvm_s390x.h"
+#include "sysemu/kvm.h"
+
+static int diag318_post_load(void *opaque, int version_id)
+{
+    DIAG318State *d = opaque;
+
Quoted 10 lines
Quoted 5 lines

+    kvm_s390_set_cpc(d->cpc);
+
+    /* It is not necessary to retain a copy of the cpc after migration. */
+    d->cpc = 0;

But we also do not need to zero it out? Can't you just drop these 2 lines?

Absolutely. I'll drop them.

Quoted 14 lines
Quoted 14 lines
hw/s390x/diag318.c new file +14 -0
+
+    return 0;
+}
+
+static int diag318_pre_save(void *opaque)
+{
+    DIAG318State *d = opaque;
+
+    kvm_s390_get_cpc(&d->cpc);
+    return 0;
+}
+
+static bool diag318_needed(void *opaque)
+{
Quoted 9 lines

hw/s390x/diag318.c line 43

Quoted 4 lines

+    DIAG318State *d = opaque;

hw/s390x/diag318.c new file +3 -0
+
+    return d->enabled;
+}

I would like to have a cpumodel entry that allows to disable that feature. And we should
then check for this.

Noted.

Quoted 16 lines
Quoted 16 lines
hw/s390x/diag318.c new file +16 -0
+
+const VMStateDescription vmstate_diag318 = {
+    .name = "vmstate_diag318",
+    .post_load = diag318_post_load,
+    .pre_save = diag318_pre_save,
+    .version_id = 1,
+    .minimum_version_id = 1,
+    .needed = diag318_needed,
+    .fields = (VMStateField[]) {
+        VMSTATE_UINT64(cpc, DIAG318State),
+        VMSTATE_END_OF_LIST()
+    }
+};
+
+static void s390_diag318_realize(DeviceState *dev, Error **errp)
+{
Quoted 8 lines
Quoted 5 lines

+    DIAG318State *d = DIAG318(dev);
+
+    if (kvm_s390_has_diag318()) {
+        d->enabled = true;
+    }

same here -> cpumodel
Then we can get rid of the enabled bool.

Noted.

[...]

Quoted 18 lines
Quoted 18 lines

    static void s390_cpu_plug(HotplugHandler *hotplug_dev,

hw/s390x/diag318.c new file +0 -0
@@ -570,6 +585,7 @@ static void machine_set_loadparm(Object *obj,
 const char *val, Error **errp)

          ms->loadparm[i] = ' '; /* pad right with spaces */ }
  }
+
  static inline void s390_machine_initfn(Object *obj) {
      object_property_add_bool(obj, "aes-key-wrap",

hw/s390x/diag318.c new file +2 -0
@@ -652,6 +668,13 @@ static void
 ccw_machine_4_0_instance_options(MachineState *machine)
     static void ccw_machine_4_0_class_options(MachineClass *mc)
   {
+    /*
+     * Read Info might reveal more bytes used to detect facilities,

thus
+     * reducing the number of CPU entries. Let's reduce the max CPUs by

Quoted 9 lines
Quoted 6 lines

+     * an arbitrary number in effort to anticipate future facility bytes.
+     */
+    if ((SCCB_SIZE - sizeof(ReadInfo)) / sizeof(CPUEntry) < S390_MAX_CPUS)
+        mc->max_cpus = S390_MAX_CPUS - 8;

Maybe this should depend on the presence of this feature in the cpumodel?

Quoted 2 lines

That's a good idea. This would allow the user to fine tune their VM to either allow the original max 248 _or_ enable the diag318 instruction.

Please also not that ccw_machine_4_0_class_options() is called from ccw_machine_3_1_class_options() and thus all older machine class option functions - you have to make sure to restore the 248 for them.

As a test, please check that you can migrate a s390-ccw-virtio-3.1 machine with 248 CPUs from a qemu without your patch to a QEMU with your patch and back.

Thomas

#5 lore raw
In reply to Collin Walling
Quoted 255 lines · Collin Walling

DIAGNOSE 0x318 (diag318) is a privileged s390x instruction that must be intercepted by SIE and handled via KVM. Let's introduce some functions to communicate between QEMU and KVM via ioctls. These will be used to get/set the diag318 related information (also known as the "Control Program Code" or "CPC"), as well as check the system if KVM supports handling this instruction.

The availability of this instruction is determined by byte 134, bit 0 of the Read Info block. This coincidentally expands into the space used for CPU entries, which means VMs running with the diag318 capability will have a reduced maximum CPU count. To alleviate this, let's calculate the actual max CPU entry space by subtracting the size of Read Info from the SCCB size then dividing that number by the size of a CPU entry. If this value is less than the value denoted by S390_MAX_CPUS, then let's reduce the max cpus for s390 from 248 to 240 in an effort to anticipate this potentially happening again in the future.

In order to simplify the migration and system reset requirements of the diag318 data, let's introduce it as a device class and include a VMStateDescription.

Diag318 is reset on during modified clear and load normal.

---
 hw/s390x/Makefile.objs       |   1 +
 hw/s390x/diag318.c           | 100 +++++++++++++++++++++++++++++++++++++++++++
 hw/s390x/diag318.h           |  39 +++++++++++++++++
 hw/s390x/s390-virtio-ccw.c   |  23 ++++++++++
 hw/s390x/sclp.c              |   5 +++
 include/hw/s390x/sclp.h      |   2 +
 linux-headers/asm-s390/kvm.h |   4 ++
 target/s390x/kvm-stub.c      |  15 +++++++
 target/s390x/kvm.c           |  32 ++++++++++++++
 target/s390x/kvm_s390x.h     |   3 ++
 10 files changed, 224 insertions(+)
 create mode 100644 hw/s390x/diag318.c
 create mode 100644 hw/s390x/diag318.h
hw/s390x/Makefile.objs +1 -0
index e02ed80..93621dc 100644
@@ -34,3 +34,4 @@ obj-$(CONFIG_KVM) += s390-stattrib-kvm.o
 obj-y += s390-ccw.o
 obj-y += ap-device.o
 obj-y += ap-bridge.o
+obj-y += diag318.o
hw/s390x/diag318.c new file +100 -0
new file mode 100644
index 0000000..94b44da
@@ -0,0 +1,100 @@
+/*
+ * DIAGNOSE 0x318 functions for reset and migration
+ *
+ * Copyright IBM, Corp. 2019
+ *
+ * Authors:
+ *  Collin Walling <[email protected]>
+ *
+ * This work is licensed under the terms of the GNU GPL, version 2 or (at your
+ * option) any later version. See the COPYING file in the top-level directory.
+ */
+
+#include "hw/s390x/diag318.h"
+#include "qapi/error.h"
+#include "kvm_s390x.h"
+#include "sysemu/kvm.h"
+
+static int diag318_post_load(void *opaque, int version_id)
+{
+    DIAG318State *d = opaque;
+
+    kvm_s390_set_cpc(d->cpc);
+
+    /* It is not necessary to retain a copy of the cpc after migration. */
+    d->cpc = 0;
+
+    return 0;
+}
+
+static int diag318_pre_save(void *opaque)
+{
+    DIAG318State *d = opaque;
+
+    kvm_s390_get_cpc(&d->cpc);
+    return 0;
+}
+
+static bool diag318_needed(void *opaque)
+{
+    DIAG318State *d = opaque;
+
+    return d->enabled;
+}
+
+const VMStateDescription vmstate_diag318 = {
+    .name = "vmstate_diag318",
+    .post_load = diag318_post_load,
+    .pre_save = diag318_pre_save,
+    .version_id = 1,
+    .minimum_version_id = 1,
+    .needed = diag318_needed,
+    .fields = (VMStateField[]) {
+        VMSTATE_UINT64(cpc, DIAG318State),
+        VMSTATE_END_OF_LIST()
+    }
+};
+
+static void s390_diag318_realize(DeviceState *dev, Error **errp)
+{
+    DIAG318State *d = DIAG318(dev);
+
+    if (kvm_s390_has_diag318()) {
+        d->enabled = true;
+    }
+}
+
+static void s390_diag318_reset(DeviceState *dev)
+{
+    DIAG318State *d = DIAG318(dev);
+
+    if (d->enabled) {
+        kvm_s390_set_cpc(0);
+    }
+}
+
+static void s390_diag318_class_init(ObjectClass *klass, void *data)
+{
+    DeviceClass *dc = DEVICE_CLASS(klass);
+
+    dc->realize = s390_diag318_realize;
+    dc->reset = s390_diag318_reset;
+    dc->vmsd = &vmstate_diag318;
+    dc->hotpluggable = false;
+    /* Reason: Set automatically during IPL */
+    dc->user_creatable = false;
+}
+
+static const TypeInfo s390_diag318_info = {
+    .class_init = s390_diag318_class_init,
+    .parent = TYPE_DEVICE,
+    .name = TYPE_S390_DIAG318,
+    .instance_size = sizeof(DIAG318State),
+};
+
+static void s390_diag318_register_types(void)
+{
+    type_register_static(&s390_diag318_info);
+}
+
+type_init(s390_diag318_register_types)
hw/s390x/diag318.h new file +39 -0
new file mode 100644
index 0000000..c154b0a
@@ -0,0 +1,39 @@
+/*
+ * DIAGNOSE 0x318 functions for reset and migration
+ *
+ * Copyright IBM, Corp. 2019
+ *
+ * Authors:
+ *  Collin Walling <[email protected]>
+ *
+ * This work is licensed under the terms of the GNU GPL, version 2 or (at your
+ * option) any later version. See the COPYING file in the top-level directory.
+ */
+
+ #ifndef DIAG318_H
+ #define DIAG318_H
+
+#include "qemu/osdep.h"
+#include "hw/qdev.h"
+
+#define TYPE_S390_DIAG318 "diag318"
+#define DIAG318(obj) \
+    OBJECT_CHECK(DIAG318State, (obj), TYPE_S390_DIAG318)
+
+typedef struct DIAG318State {
+    /*< private >*/
+    DeviceState parent_obj;
+
+    /*< public >*/
+    uint64_t cpc;
+    bool enabled;
+} DIAG318State;
+
+typedef struct DIAG318Class {
+    /*< private >*/
+    DeviceClass parent_class;
+
+    /*< public >*/
+} DIAG318Class;
+
+#endif /* DIAG318_H */
hw/s390x/s390-virtio-ccw.c +16 -0
index d11069b..44a424b 100644
@@ -36,6 +36,7 @@
 #include "cpu_models.h"
 #include "hw/nmi.h"
 #include "hw/s390x/tod.h"
+#include "hw/s390x/diag318.h"
 
 S390CPU *s390_cpu_addr2state(uint16_t cpu_addr)
 {
@@ -92,6 +93,7 @@ static const char *const reset_dev_types[] = {
     "s390-sclp-event-facility",
     "s390-flic",
     "diag288",
+    TYPE_S390_DIAG318,
 };
 
 static void subsystem_reset(void)
@@ -246,6 +248,17 @@ static void s390_create_sclpconsole(const char *type, Chardev *chardev)
     qdev_init_nofail(dev);
 }
 
+static void s390_init_diag318(void)
+{
+    Object *new = object_new(TYPE_S390_DIAG318);
+    DeviceState *dev = DEVICE(new);
+
+    object_property_add_child(qdev_get_machine(), TYPE_S390_DIAG318,
+                              new, NULL);
+    object_unref(new);
+    qdev_init_nofail(dev);
+}
+
 static void ccw_init(MachineState *machine)
 {
     int ret;
@@ -302,6 +315,8 @@ static void ccw_init(MachineState *machine)
 
     /* init the TOD clock */
     s390_init_tod();
+
+    s390_init_diag318();
 }
 
 static void s390_cpu_plug(HotplugHandler *hotplug_dev,
@@ -570,6 +585,7 @@ static void machine_set_loadparm(Object *obj, const char *val, Error **errp)
         ms->loadparm[i] = ' '; /* pad right with spaces */
     }
 }
+
 static inline void s390_machine_initfn(Object *obj)
 {
     object_property_add_bool(obj, "aes-key-wrap",

hw/s390x/s390-virtio-ccw.c static void ccw_machine_4_0_instance_options(MachineState *machine) · line 677

Quoted 10 lines
hw/s390x/s390-virtio-ccw.c +7 -0
@@ -652,6 +668,13 @@ static void ccw_machine_4_0_instance_options(MachineState *machine)
 
 static void ccw_machine_4_0_class_options(MachineClass *mc)
 {
+    /*
+     * Read Info might reveal more bytes used to detect facilities, thus
+     * reducing the number of CPU entries. Let's reduce the max CPUs by
+     * an arbitrary number in effort to anticipate future facility bytes.
+     */
+    if ((SCCB_SIZE - sizeof(ReadInfo)) / sizeof(CPUEntry) < S390_MAX_CPUS)
+        mc->max_cpus = S390_MAX_CPUS - 8;

This is too complicated, just set it always to 240.

However, I am still not sure how to best handle this scenario. One solution is

1. Set it statically to 240 for machine > 4.1
2. Keep the old machines unmodifed
3. Don't indicate the CPU feature for machines <= 4.0

#3 is the problematic part, as it mixes host CPU features and machines. Bad. The host CPU model should always look the same on all machines. I don't like this.

Signature · 3 lines
Thanks,

David / dhildenb
#6 lore raw
Quoted 272 lines · David Hildenbrand
Quoted 255 lines · Collin Walling

DIAGNOSE 0x318 (diag318) is a privileged s390x instruction that must be intercepted by SIE and handled via KVM. Let's introduce some functions to communicate between QEMU and KVM via ioctls. These will be used to get/set the diag318 related information (also known as the "Control Program Code" or "CPC"), as well as check the system if KVM supports handling this instruction.

The availability of this instruction is determined by byte 134, bit 0 of the Read Info block. This coincidentally expands into the space used for CPU entries, which means VMs running with the diag318 capability will have a reduced maximum CPU count. To alleviate this, let's calculate the actual max CPU entry space by subtracting the size of Read Info from the SCCB size then dividing that number by the size of a CPU entry. If this value is less than the value denoted by S390_MAX_CPUS, then let's reduce the max cpus for s390 from 248 to 240 in an effort to anticipate this potentially happening again in the future.

In order to simplify the migration and system reset requirements of the diag318 data, let's introduce it as a device class and include a VMStateDescription.

Diag318 is reset on during modified clear and load normal.

---
 hw/s390x/Makefile.objs       |   1 +
 hw/s390x/diag318.c           | 100 +++++++++++++++++++++++++++++++++++++++++++
 hw/s390x/diag318.h           |  39 +++++++++++++++++
 hw/s390x/s390-virtio-ccw.c   |  23 ++++++++++
 hw/s390x/sclp.c              |   5 +++
 include/hw/s390x/sclp.h      |   2 +
 linux-headers/asm-s390/kvm.h |   4 ++
 target/s390x/kvm-stub.c      |  15 +++++++
 target/s390x/kvm.c           |  32 ++++++++++++++
 target/s390x/kvm_s390x.h     |   3 ++
 10 files changed, 224 insertions(+)
 create mode 100644 hw/s390x/diag318.c
 create mode 100644 hw/s390x/diag318.h
hw/s390x/Makefile.objs +1 -0
index e02ed80..93621dc 100644
@@ -34,3 +34,4 @@ obj-$(CONFIG_KVM) += s390-stattrib-kvm.o
 obj-y += s390-ccw.o
 obj-y += ap-device.o
 obj-y += ap-bridge.o
+obj-y += diag318.o
hw/s390x/diag318.c new file +100 -0
new file mode 100644
index 0000000..94b44da
@@ -0,0 +1,100 @@
+/*
+ * DIAGNOSE 0x318 functions for reset and migration
+ *
+ * Copyright IBM, Corp. 2019
+ *
+ * Authors:
+ *  Collin Walling <[email protected]>
+ *
+ * This work is licensed under the terms of the GNU GPL, version 2 or (at your
+ * option) any later version. See the COPYING file in the top-level directory.
+ */
+
+#include "hw/s390x/diag318.h"
+#include "qapi/error.h"
+#include "kvm_s390x.h"
+#include "sysemu/kvm.h"
+
+static int diag318_post_load(void *opaque, int version_id)
+{
+    DIAG318State *d = opaque;
+
+    kvm_s390_set_cpc(d->cpc);
+
+    /* It is not necessary to retain a copy of the cpc after migration. */
+    d->cpc = 0;
+
+    return 0;
+}
+
+static int diag318_pre_save(void *opaque)
+{
+    DIAG318State *d = opaque;
+
+    kvm_s390_get_cpc(&d->cpc);
+    return 0;
+}
+
+static bool diag318_needed(void *opaque)
+{
+    DIAG318State *d = opaque;
+
+    return d->enabled;
+}
+
+const VMStateDescription vmstate_diag318 = {
+    .name = "vmstate_diag318",
+    .post_load = diag318_post_load,
+    .pre_save = diag318_pre_save,
+    .version_id = 1,
+    .minimum_version_id = 1,
+    .needed = diag318_needed,
+    .fields = (VMStateField[]) {
+        VMSTATE_UINT64(cpc, DIAG318State),
+        VMSTATE_END_OF_LIST()
+    }
+};
+
+static void s390_diag318_realize(DeviceState *dev, Error **errp)
+{
+    DIAG318State *d = DIAG318(dev);
+
+    if (kvm_s390_has_diag318()) {
+        d->enabled = true;
+    }
+}
+
+static void s390_diag318_reset(DeviceState *dev)
+{
+    DIAG318State *d = DIAG318(dev);
+
+    if (d->enabled) {
+        kvm_s390_set_cpc(0);
+    }
+}
+
+static void s390_diag318_class_init(ObjectClass *klass, void *data)
+{
+    DeviceClass *dc = DEVICE_CLASS(klass);
+
+    dc->realize = s390_diag318_realize;
+    dc->reset = s390_diag318_reset;
+    dc->vmsd = &vmstate_diag318;
+    dc->hotpluggable = false;
+    /* Reason: Set automatically during IPL */
+    dc->user_creatable = false;
+}
+
+static const TypeInfo s390_diag318_info = {
+    .class_init = s390_diag318_class_init,
+    .parent = TYPE_DEVICE,
+    .name = TYPE_S390_DIAG318,
+    .instance_size = sizeof(DIAG318State),
+};
+
+static void s390_diag318_register_types(void)
+{
+    type_register_static(&s390_diag318_info);
+}
+
+type_init(s390_diag318_register_types)
hw/s390x/diag318.h new file +39 -0
new file mode 100644
index 0000000..c154b0a
@@ -0,0 +1,39 @@
+/*
+ * DIAGNOSE 0x318 functions for reset and migration
+ *
+ * Copyright IBM, Corp. 2019
+ *
+ * Authors:
+ *  Collin Walling <[email protected]>
+ *
+ * This work is licensed under the terms of the GNU GPL, version 2 or (at your
+ * option) any later version. See the COPYING file in the top-level directory.
+ */
+
+ #ifndef DIAG318_H
+ #define DIAG318_H
+
+#include "qemu/osdep.h"
+#include "hw/qdev.h"
+
+#define TYPE_S390_DIAG318 "diag318"
+#define DIAG318(obj) \
+    OBJECT_CHECK(DIAG318State, (obj), TYPE_S390_DIAG318)
+
+typedef struct DIAG318State {
+    /*< private >*/
+    DeviceState parent_obj;
+
+    /*< public >*/
+    uint64_t cpc;
+    bool enabled;
+} DIAG318State;
+
+typedef struct DIAG318Class {
+    /*< private >*/
+    DeviceClass parent_class;
+
+    /*< public >*/
+} DIAG318Class;
+
+#endif /* DIAG318_H */
hw/s390x/s390-virtio-ccw.c +16 -0
index d11069b..44a424b 100644
@@ -36,6 +36,7 @@
 #include "cpu_models.h"
 #include "hw/nmi.h"
 #include "hw/s390x/tod.h"
+#include "hw/s390x/diag318.h"
 
 S390CPU *s390_cpu_addr2state(uint16_t cpu_addr)
 {
@@ -92,6 +93,7 @@ static const char *const reset_dev_types[] = {
     "s390-sclp-event-facility",
     "s390-flic",
     "diag288",
+    TYPE_S390_DIAG318,
 };
 
 static void subsystem_reset(void)
@@ -246,6 +248,17 @@ static void s390_create_sclpconsole(const char *type, Chardev *chardev)
     qdev_init_nofail(dev);
 }
 
+static void s390_init_diag318(void)
+{
+    Object *new = object_new(TYPE_S390_DIAG318);
+    DeviceState *dev = DEVICE(new);
+
+    object_property_add_child(qdev_get_machine(), TYPE_S390_DIAG318,
+                              new, NULL);
+    object_unref(new);
+    qdev_init_nofail(dev);
+}
+
 static void ccw_init(MachineState *machine)
 {
     int ret;
@@ -302,6 +315,8 @@ static void ccw_init(MachineState *machine)
 
     /* init the TOD clock */
     s390_init_tod();
+
+    s390_init_diag318();
 }
 
 static void s390_cpu_plug(HotplugHandler *hotplug_dev,
@@ -570,6 +585,7 @@ static void machine_set_loadparm(Object *obj, const char *val, Error **errp)
         ms->loadparm[i] = ' '; /* pad right with spaces */
     }
 }
+
 static inline void s390_machine_initfn(Object *obj)
 {
     object_property_add_bool(obj, "aes-key-wrap",

hw/s390x/s390-virtio-ccw.c static void ccw_machine_4_0_instance_options(MachineState *machine) · line 677

Quoted 10 lines
hw/s390x/s390-virtio-ccw.c +7 -0
@@ -652,6 +668,13 @@ static void ccw_machine_4_0_instance_options(MachineState *machine)
 
 static void ccw_machine_4_0_class_options(MachineClass *mc)
 {
+    /*
+     * Read Info might reveal more bytes used to detect facilities, thus
+     * reducing the number of CPU entries. Let's reduce the max CPUs by
+     * an arbitrary number in effort to anticipate future facility bytes.
+     */
+    if ((SCCB_SIZE - sizeof(ReadInfo)) / sizeof(CPUEntry) < S390_MAX_CPUS)
+        mc->max_cpus = S390_MAX_CPUS - 8;

This is too complicated, just set it always to 240.

However, I am still not sure how to best handle this scenario. One solution is

Quoted 8 lines

1. Set it statically to 240 for machine > 4.1
2. Keep the old machines unmodifed
3. Don't indicate the CPU feature for machines <= 4.0

#3 is the problematic part, as it mixes host CPU features and machines. Bad. The host CPU model should always look the same on all machines. I don't like this.

FWIW, #3 is only an issue when modeling it via the CPU model, like Christian suggested.

I suggest the following

1. Set the max #cpus for 4.1 to 240 (already done)
2. Keep it for the other machines unmodified (as suggested by Thomas)
3. Create the layout of the SCCB depending on the machine type (to be done)

If we want to model diag318 via a CPU feature (which makes sense for migration):

4. Disable diag318 with a warning if used with a machine < 4.1

Signature · 3 lines
Thanks,

David / dhildenb
#7 lore raw
Quoted 22 lines · David Hildenbrand
Quoted 8 lines

hw/s390x/s390-virtio-ccw.c static void ccw_machine_4_0_instance_options(MachineState *machine) · line 677

Quoted 2 lines
hw/s390x/s390-virtio-ccw.c +2 -0
+    if ((SCCB_SIZE - sizeof(ReadInfo)) / sizeof(CPUEntry) < S390_MAX_CPUS)
+        mc->max_cpus = S390_MAX_CPUS - 8;

This is too complicated, just set it always to 240.

However, I am still not sure how to best handle this scenario. One solution is

Quoted 8 lines

1. Set it statically to 240 for machine > 4.1
2. Keep the old machines unmodifed
3. Don't indicate the CPU feature for machines <= 4.0

#3 is the problematic part, as it mixes host CPU features and machines. Bad. The host CPU model should always look the same on all machines. I don't like this.

FWIW, #3 is only an issue when modeling it via the CPU model, like Christian suggested.

I suggest the following

Quoted 9 lines

1. Set the max #cpus for 4.1 to 240 (already done)
2. Keep it for the other machines unmodified (as suggested by Thomas)
3. Create the layout of the SCCB depending on the machine type (to be done)

If we want to model diag318 via a CPU feature (which makes sense for migration):

4. Disable diag318 with a warning if used with a machine < 4.1

I think there is a simpler solution. It is perfectly fine to fail the startup if we cannot fulfil the cpu model. So lets just allow 248 and allow this feature also for older machines. And if somebody chooses both at the same time, lets fails the startup.

#8 lore raw
Quoted 32 lines · Christian Borntraeger
Quoted 29 lines · David Hildenbrand
Quoted 8 lines

hw/s390x/s390-virtio-ccw.c static void ccw_machine_4_0_instance_options(MachineState *machine) · line 677

Quoted 2 lines
hw/s390x/s390-virtio-ccw.c +2 -0
+    if ((SCCB_SIZE - sizeof(ReadInfo)) / sizeof(CPUEntry) < S390_MAX_CPUS)
+        mc->max_cpus = S390_MAX_CPUS - 8;

This is too complicated, just set it always to 240.

However, I am still not sure how to best handle this scenario. One solution is

Quoted 8 lines

1. Set it statically to 240 for machine > 4.1
2. Keep the old machines unmodifed
3. Don't indicate the CPU feature for machines <= 4.0

#3 is the problematic part, as it mixes host CPU features and machines. Bad. The host CPU model should always look the same on all machines. I don't like this.

FWIW, #3 is only an issue when modeling it via the CPU model, like Christian suggested.

I suggest the following

1. Set the max #cpus for 4.1 to 240 (already done)
2. Keep it for the other machines unmodified (as suggested by Thomas)
3. Create the layout of the SCCB depending on the machine type (to be done)

If we want to model diag318 via a CPU feature (which makes sense for migration):

Quoted 7 lines
Quoted 2 lines

4. Disable diag318 with a warning if used with a machine < 4.1

I think there is a simpler solution. It is perfectly fine to fail the startup if we cannot fulfil the cpu model. So lets just allow 248 and allow this feature also for older machines. And if somebody chooses both at the same time, lets fails the startup.

To which knob do you want to glue the layout of the SCLP response? Like I described? Do you mean instead of warning and masking the feature off as I suggested, simply failing?

In that case, -machine ..-4.0 -cpu host will not work on new HW with new KVM. Just noting.

Signature · 3 lines
Thanks,

David / dhildenb
#9 lore raw
Quoted 36 lines · David Hildenbrand
Quoted 35 lines · Christian Borntraeger
Quoted 31 lines · David Hildenbrand
Quoted 8 lines

hw/s390x/s390-virtio-ccw.c static void ccw_machine_4_0_instance_options(MachineState *machine) · line 677

Quoted 2 lines
hw/s390x/s390-virtio-ccw.c +2 -0
+    if ((SCCB_SIZE - sizeof(ReadInfo)) / sizeof(CPUEntry) < S390_MAX_CPUS)
+        mc->max_cpus = S390_MAX_CPUS - 8;

This is too complicated, just set it always to 240.

However, I am still not sure how to best handle this scenario. One solution is

Quoted 8 lines

1. Set it statically to 240 for machine > 4.1
2. Keep the old machines unmodifed
3. Don't indicate the CPU feature for machines <= 4.0

#3 is the problematic part, as it mixes host CPU features and machines. Bad. The host CPU model should always look the same on all machines. I don't like this.

FWIW, #3 is only an issue when modeling it via the CPU model, like Christian suggested.

I suggest the following

1. Set the max #cpus for 4.1 to 240 (already done)
2. Keep it for the other machines unmodified (as suggested by Thomas)
3. Create the layout of the SCCB depending on the machine type (to be done)

If we want to model diag318 via a CPU feature (which makes sense for migration):

4. Disable diag318 with a warning if used with a machine < 4.1

Quoted 8 lines
Quoted 4 lines

I think there is a simpler solution. It is perfectly fine to fail the startup if we cannot fulfil the cpu model. So lets just allow 248 and allow this feature also for older machines. And if somebody chooses both at the same time, lets fails the startup.

To which knob do you want to glue the layout of the SCLP response? Like I described? Do you mean instead of warning and masking the feature off as I suggested, simply failing?

The sclp response will depend on the dia318 cpu model flag. If its on, the sclp response will have it, otherwise not.
- host-passthrough: not migration safe anyway
- host-model: if the target has diag318 good, otherwise we reject migration

Quoted 3 lines

In that case, -machine ..-4.0 -cpu host will not work on new HW with new KVM. Just noting.

Only if you have 248 CPUs (which is unlikely). My point was to do that for all machine levels.

#10 lore raw
Quoted 53 lines · Christian Borntraeger
Quoted 36 lines · David Hildenbrand
Quoted 35 lines · Christian Borntraeger
Quoted 31 lines · David Hildenbrand
Quoted 8 lines

hw/s390x/s390-virtio-ccw.c static void ccw_machine_4_0_instance_options(MachineState *machine) · line 677

Quoted 2 lines
hw/s390x/s390-virtio-ccw.c +2 -0
+    if ((SCCB_SIZE - sizeof(ReadInfo)) / sizeof(CPUEntry) < S390_MAX_CPUS)
+        mc->max_cpus = S390_MAX_CPUS - 8;

This is too complicated, just set it always to 240.

However, I am still not sure how to best handle this scenario. One solution is

Quoted 8 lines

1. Set it statically to 240 for machine > 4.1
2. Keep the old machines unmodifed
3. Don't indicate the CPU feature for machines <= 4.0

#3 is the problematic part, as it mixes host CPU features and machines. Bad. The host CPU model should always look the same on all machines. I don't like this.

FWIW, #3 is only an issue when modeling it via the CPU model, like Christian suggested.

I suggest the following

1. Set the max #cpus for 4.1 to 240 (already done)
2. Keep it for the other machines unmodified (as suggested by Thomas)
3. Create the layout of the SCCB depending on the machine type (to be done)

If we want to model diag318 via a CPU feature (which makes sense for migration):

4. Disable diag318 with a warning if used with a machine < 4.1

Quoted 8 lines
Quoted 4 lines

I think there is a simpler solution. It is perfectly fine to fail the startup if we cannot fulfil the cpu model. So lets just allow 248 and allow this feature also for older machines. And if somebody chooses both at the same time, lets fails the startup.

To which knob do you want to glue the layout of the SCLP response? Like I described? Do you mean instead of warning and masking the feature off as I suggested, simply failing?

The sclp response will depend on the dia318 cpu model flag. If its on, the sclp response will have it, otherwise not.
- host-passthrough: not migration safe anyway
- host-model: if the target has diag318 good, otherwise we reject migration

Quoted 1 line
Quoted 6 lines
Quoted 2 lines

In that case, -machine ..-4.0 -cpu host will not work on new HW with new KVM. Just noting.

Only if you have 248 CPUs (which is unlikely). My point was to do that for all machine levels.

The issue with this approach is that e.g. libvirt is not aware of this restriction. It could query "max_cpus" and expand the host-cpu model, but starting a guest with > 240 cpus would fail. Maybe this is acceptable.

The approach you describe is actually pretty nice. We would have to

1. Modify SCLP code to change the layout based on the feature availability
2. Check against the cpus when trying to enable the cpu model.

Signature · 3 lines
Thanks,

David / dhildenb
#11 lore raw
Quoted 57 lines · David Hildenbrand
Quoted 56 lines · Christian Borntraeger
Quoted 36 lines · David Hildenbrand
Quoted 35 lines · Christian Borntraeger
Quoted 31 lines · David Hildenbrand
Quoted 8 lines

Review

Quoted 2 lines
(quoted diff) +2 -0
+    if ((SCCB_SIZE - sizeof(ReadInfo)) / sizeof(CPUEntry) < S390_MAX_CPUS)
+        mc->max_cpus = S390_MAX_CPUS - 8;

This is too complicated, just set it always to 240.

However, I am still not sure how to best handle this scenario. One solution is

Quoted 8 lines

1. Set it statically to 240 for machine > 4.1
2. Keep the old machines unmodifed
3. Don't indicate the CPU feature for machines <= 4.0

#3 is the problematic part, as it mixes host CPU features and machines. Bad. The host CPU model should always look the same on all machines. I don't like this.

FWIW, #3 is only an issue when modeling it via the CPU model, like Christian suggested.

I suggest the following

1. Set the max #cpus for 4.1 to 240 (already done)
2. Keep it for the other machines unmodified (as suggested by Thomas)
3. Create the layout of the SCCB depending on the machine type (to be done)

If we want to model diag318 via a CPU feature (which makes sense for migration):

4. Disable diag318 with a warning if used with a machine < 4.1

Quoted 8 lines
Quoted 4 lines

I think there is a simpler solution. It is perfectly fine to fail the startup if we cannot fulfil the cpu model. So lets just allow 248 and allow this feature also for older machines. And if somebody chooses both at the same time, lets fails the startup.

To which knob do you want to glue the layout of the SCLP response? Like I described? Do you mean instead of warning and masking the feature off as I suggested, simply failing?

The sclp response will depend on the dia318 cpu model flag. If its on, the sclp response will have it, otherwise not.
- host-passthrough: not migration safe anyway
- host-model: if the target has diag318 good, otherwise we reject migration

Quoted 3 lines

In that case, -machine ..-4.0 -cpu host will not work on new HW with new KVM. Just noting.

Quoted 7 lines
Quoted 3 lines

Only if you have 248 CPUs (which is unlikely). My point was to do that for all machine levels.

The issue with this approach is that e.g. libvirt is not aware of this restriction. It could query "max_cpus" and expand the host-cpu model, but starting a guest with > 240 cpus would fail. Maybe this is acceptable.

As of today we do the cpu model check in the same way. libvirt actually tries to run QEMU and handles failures.

For a failure, the user still has still to use >240 CPUs in its XML. The only downside is that libvirt will not reject this right away.

During startup we would then print an error message like

"The diag318 cpu feature is only supported for 240 and less CPUs."

This is of similar quality as
"Selected CPU GA level is too new. Maximum supported model in the configuration: \'%s\'",

and others that we have today.

So yes, I think this would be acceptable.

Quoted 6 lines

The approach you describe is actually pretty nice. We would have to

1. Modify SCLP code to change the layout based on the feature availability
2. Check against the cpus when trying to enable the cpu model.

#12 lore raw
Quoted 74 lines · Christian Borntraeger
Quoted 57 lines · David Hildenbrand
Quoted 56 lines · Christian Borntraeger
Quoted 36 lines · David Hildenbrand
Quoted 35 lines · Christian Borntraeger
Quoted 31 lines · David Hildenbrand
Quoted 8 lines

Review

Quoted 2 lines
(quoted diff) +2 -0
+    if ((SCCB_SIZE - sizeof(ReadInfo)) / sizeof(CPUEntry) < S390_MAX_CPUS)
+        mc->max_cpus = S390_MAX_CPUS - 8;

This is too complicated, just set it always to 240.

However, I am still not sure how to best handle this scenario. One solution is

Quoted 8 lines

1. Set it statically to 240 for machine > 4.1
2. Keep the old machines unmodifed
3. Don't indicate the CPU feature for machines <= 4.0

#3 is the problematic part, as it mixes host CPU features and machines. Bad. The host CPU model should always look the same on all machines. I don't like this.

FWIW, #3 is only an issue when modeling it via the CPU model, like Christian suggested.

I suggest the following

1. Set the max #cpus for 4.1 to 240 (already done)
2. Keep it for the other machines unmodified (as suggested by Thomas)
3. Create the layout of the SCCB depending on the machine type (to be done)

If we want to model diag318 via a CPU feature (which makes sense for migration):

4. Disable diag318 with a warning if used with a machine < 4.1

Quoted 8 lines
Quoted 4 lines

I think there is a simpler solution. It is perfectly fine to fail the startup if we cannot fulfil the cpu model. So lets just allow 248 and allow this feature also for older machines. And if somebody chooses both at the same time, lets fails the startup.

To which knob do you want to glue the layout of the SCLP response? Like I described? Do you mean instead of warning and masking the feature off as I suggested, simply failing?

The sclp response will depend on the dia318 cpu model flag. If its on, the sclp response will have it, otherwise not.
- host-passthrough: not migration safe anyway
- host-model: if the target has diag318 good, otherwise we reject migration

Quoted 3 lines

In that case, -machine ..-4.0 -cpu host will not work on new HW with new KVM. Just noting.

Quoted 7 lines
Quoted 3 lines

Only if you have 248 CPUs (which is unlikely). My point was to do that for all machine levels.

The issue with this approach is that e.g. libvirt is not aware of this restriction. It could query "max_cpus" and expand the host-cpu model, but starting a guest with > 240 cpus would fail. Maybe this is acceptable.

As of today we do the cpu model check in the same way. libvirt actually tries to run QEMU and handles failures.

For a failure, the user still has still to use >240 CPUs in its XML. The only downside is that libvirt will not reject this right away.

Quoted 7 lines

During startup we would then print an error message like

"The diag318 cpu feature is only supported for 240 and less CPUs."

This is of similar quality as
"Selected CPU GA level is too new. Maximum supported model in the configuration: \'%s\'",

But that can be tested using the runability information if I am not wrong.

Quoted 3 lines

and others that we have today.

So yes, I think this would be acceptable.

I guess it is acceptable yes. I doubt anybody uses that many CPUs in production either way. But you never know.

Signature · 3 lines
Thanks,

David / dhildenb
#13 lore raw
Quoted 77 lines · On Mon, 13 May 2019 13:34:35 +0200 David Hildenbrand <[email protected]> wrote:
Quoted 76 lines · Christian Borntraeger
Quoted 57 lines · David Hildenbrand
Quoted 56 lines · Christian Borntraeger
Quoted 36 lines · David Hildenbrand
Quoted 35 lines · Christian Borntraeger
Quoted 31 lines · David Hildenbrand
Quoted 8 lines

Review

Quoted 2 lines
(quoted diff) +2 -0
+    if ((SCCB_SIZE - sizeof(ReadInfo)) / sizeof(CPUEntry) < S390_MAX_CPUS)
+        mc->max_cpus = S390_MAX_CPUS - 8;  

This is too complicated, just set it always to 240.

However, I am still not sure how to best handle this scenario. One solution is

Quoted 8 lines

1. Set it statically to 240 for machine > 4.1
2. Keep the old machines unmodifed
3. Don't indicate the CPU feature for machines <= 4.0

#3 is the problematic part, as it mixes host CPU features and machines. Bad. The host CPU model should always look the same on all machines. I don't like this.

FWIW, #3 is only an issue when modeling it via the CPU model, like Christian suggested.

I suggest the following

1. Set the max #cpus for 4.1 to 240 (already done)
2. Keep it for the other machines unmodified (as suggested by Thomas)
3. Create the layout of the SCCB depending on the machine type (to be done)

If we want to model diag318 via a CPU feature (which makes sense for migration):

4. Disable diag318 with a warning if used with a machine < 4.1

Quoted 8 lines
Quoted 4 lines

I think there is a simpler solution. It is perfectly fine to fail the startup if we cannot fulfil the cpu model. So lets just allow 248 and allow this feature also for older machines. And if somebody chooses both at the same time, lets fails the startup.

To which knob do you want to glue the layout of the SCLP response? Like I described? Do you mean instead of warning and masking the feature off as I suggested, simply failing?

The sclp response will depend on the dia318 cpu model flag. If its on, the sclp response will have it, otherwise not.
- host-passthrough: not migration safe anyway
- host-model: if the target has diag318 good, otherwise we reject migration

Quoted 3 lines

In that case, -machine ..-4.0 -cpu host will not work on new HW with new KVM. Just noting.

Quoted 7 lines
Quoted 3 lines

Only if you have 248 CPUs (which is unlikely). My point was to do that for all machine levels.

The issue with this approach is that e.g. libvirt is not aware of this restriction. It could query "max_cpus" and expand the host-cpu model, but starting a guest with > 240 cpus would fail. Maybe this is acceptable.

As of today we do the cpu model check in the same way. libvirt actually tries to run QEMU and handles failures.

For a failure, the user still has still to use >240 CPUs in its XML. The only downside is that libvirt will not reject this right away.

During startup we would then print an error message like

Quoted 7 lines
Quoted 5 lines

"The diag318 cpu feature is only supported for 240 and less CPUs."

This is of similar quality as
"Selected CPU GA level is too new. Maximum supported model in the configuration: \'%s\'",

But that can be tested using the runability information if I am not wrong.

You mean the cpu level information, right?

Quoted 7 lines
Quoted 4 lines

and others that we have today.

So yes, I think this would be acceptable.

I guess it is acceptable yes. I doubt anybody uses that many CPUs in production either way. But you never know.

I think that using that many cpus is a more uncommon setup, but I still think that having to wait for actual failure is worse than being able to find out beforehand. Any way to make this discoverable?

#14 lore raw
Quoted 93 lines · Cornelia Huck
Quoted 77 lines · On Mon, 13 May 2019 13:34:35 +0200 David Hildenbrand <[email protected]> wrote:
Quoted 76 lines · Christian Borntraeger
Quoted 57 lines · David Hildenbrand
Quoted 56 lines · Christian Borntraeger
Quoted 36 lines · David Hildenbrand
Quoted 35 lines · Christian Borntraeger
Quoted 31 lines · David Hildenbrand
Quoted 8 lines

Review

Quoted 2 lines
(quoted diff) +2 -0
+    if ((SCCB_SIZE - sizeof(ReadInfo)) / sizeof(CPUEntry) < S390_MAX_CPUS)
+        mc->max_cpus = S390_MAX_CPUS - 8;  

This is too complicated, just set it always to 240.

However, I am still not sure how to best handle this scenario. One solution is

Quoted 8 lines

1. Set it statically to 240 for machine > 4.1
2. Keep the old machines unmodifed
3. Don't indicate the CPU feature for machines <= 4.0

#3 is the problematic part, as it mixes host CPU features and machines. Bad. The host CPU model should always look the same on all machines. I don't like this.

FWIW, #3 is only an issue when modeling it via the CPU model, like Christian suggested.

I suggest the following

1. Set the max #cpus for 4.1 to 240 (already done)
2. Keep it for the other machines unmodified (as suggested by Thomas)
3. Create the layout of the SCCB depending on the machine type (to be done)

If we want to model diag318 via a CPU feature (which makes sense for migration):

4. Disable diag318 with a warning if used with a machine < 4.1

Quoted 8 lines
Quoted 4 lines

I think there is a simpler solution. It is perfectly fine to fail the startup if we cannot fulfil the cpu model. So lets just allow 248 and allow this feature also for older machines. And if somebody chooses both at the same time, lets fails the startup.

To which knob do you want to glue the layout of the SCLP response? Like I described? Do you mean instead of warning and masking the feature off as I suggested, simply failing?

The sclp response will depend on the dia318 cpu model flag. If its on, the sclp response will have it, otherwise not.
- host-passthrough: not migration safe anyway
- host-model: if the target has diag318 good, otherwise we reject migration

Quoted 3 lines

In that case, -machine ..-4.0 -cpu host will not work on new HW with new KVM. Just noting.

Quoted 7 lines
Quoted 3 lines

Only if you have 248 CPUs (which is unlikely). My point was to do that for all machine levels.

The issue with this approach is that e.g. libvirt is not aware of this restriction. It could query "max_cpus" and expand the host-cpu model, but starting a guest with > 240 cpus would fail. Maybe this is acceptable.

As of today we do the cpu model check in the same way. libvirt actually tries to run QEMU and handles failures.

For a failure, the user still has still to use >240 CPUs in its XML. The only downside is that libvirt will not reject this right away.

During startup we would then print an error message like

Quoted 7 lines
Quoted 5 lines

"The diag318 cpu feature is only supported for 240 and less CPUs."

This is of similar quality as
"Selected CPU GA level is too new. Maximum supported model in the configuration: \'%s\'",

But that can be tested using the runability information if I am not wrong.

You mean the cpu level information, right?

Quoted 3 lines
Quoted 3 lines

and others that we have today.

Quoted 7 lines
Quoted 4 lines
Quoted 1 line

So yes, I think this would be acceptable.

I guess it is acceptable yes. I doubt anybody uses that many CPUs in production either way. But you never know.

I think that using that many cpus is a more uncommon setup, but I still think that having to wait for actual failure

That can happen all the time today. You can easily say z14 in the xml when on a zEC12. Only at startup you get the error. The question is really:
do you want to error on definition of the xml or on startup. And I think startup is the better place here. This allows to create definitions that will be useful in the future (pre-planning), e.g. if you know that you will update your machine or the code soon.

Quoted 2 lines

is worse than being able
to find out beforehand. Any way to make this discoverable?

#15 lore raw
Quoted 3 lines
Quoted 3 lines
Quoted 1 line

But that can be tested using the runability information if I am not wrong.

You mean the cpu level information, right?

Yes, query-cpu-definition includes for each model runability information via "unavailable-features" (valid under the started QEMU machine).

Quoted 6 lines
Quoted 6 lines
Quoted 6 lines
Quoted 5 lines

and others that we have today.

So yes, I think this would be acceptable.

Quoted 8 lines
Quoted 5 lines
Quoted 2 lines

I guess it is acceptable yes. I doubt anybody uses that many CPUs in production either way. But you never know.

I think that using that many cpus is a more uncommon setup, but I still think that having to wait for actual failure

That can happen all the time today. You can easily say z14 in the xml when on a zEC12. Only at startup you get the error. The question is really:

"-smp 248 -cpu host" will no longer work, while e.g. "-smp 248 -cpu z12" will work. Actually, even "-smp 248" will no longer work on affected machines.

That is why wonder if it is better to disable the feature and print a warning. Similar to CMMA, where want want to tolerate when CMMA is not possible in the current environment (huge pages).

"Diag318 will not be enabled because it is not compatible with more than 240 CPUs".

However, I still think that implementing support for more than one SCLP response page is the best solution. Guests will need adaptions for > 240 CPUs with Diag318, but who cares? Existing setups will continue to work.

Implementing that SCLP thingy will avoid any warnings and any errors. It just works from the QEMU perspective.

Is implementing this realistic?

Quoted 1 line

do you want to error on definition of the xml or on startup.

I actually have no idea what the best practice on the libvirt side is. There seems to be a user for max-cpus and unavailable-features in QEMU.

And I think

Quoted 3 lines

startup is the better place here. This allows to create definitions that will be useful in the future (pre-planning), e.g. if you know that you will update your machine or the code soon.

Signature · 3 lines
Thanks,

David / dhildenb
#16 lore raw
Quoted 33 lines · David Hildenbrand
Quoted 3 lines
Quoted 3 lines
Quoted 1 line

But that can be tested using the runability information if I am not wrong.

You mean the cpu level information, right?

Yes, query-cpu-definition includes for each model runability information via "unavailable-features" (valid under the started QEMU machine).

Quoted 6 lines
Quoted 6 lines
Quoted 6 lines
Quoted 5 lines

and others that we have today.

So yes, I think this would be acceptable.

Quoted 8 lines
Quoted 5 lines
Quoted 2 lines

I guess it is acceptable yes. I doubt anybody uses that many CPUs in production either way. But you never know.

I think that using that many cpus is a more uncommon setup, but I still think that having to wait for actual failure

That can happen all the time today. You can easily say z14 in the xml when on a zEC12. Only at startup you get the error. The question is really:

"-smp 248 -cpu host" will no longer work, while e.g. "-smp 248 -cpu z12" will work. Actually, even "-smp 248" will no longer work on affected machines.

That is why wonder if it is better to disable the feature and print a warning. Similar to CMMA, where want want to tolerate when CMMA is not possible in the current environment (huge pages).

"Diag318 will not be enabled because it is not compatible with more than 240 CPUs".

Quoted 8 lines

However, I still think that implementing support for more than one SCLP response page is the best solution. Guests will need adaptions for > 240 CPUs with Diag318, but who cares? Existing setups will continue to work.

Implementing that SCLP thingy will avoid any warnings and any errors. It just works from the QEMU perspective.

Is implementing this realistic?

Yes it is but it will take time. I will try to get this rolling. To make progress on the diag318 thing, can we error on startup now and simply remove that check when when have implemented a larger sccb? If we would now do all kinds of "change the max number games" would be harder to "fix".

Quoted 13 lines
Quoted 2 lines

do you want to error on definition of the xml or on startup.

I actually have no idea what the best practice on the libvirt side is. There seems to be a user for max-cpus and unavailable-features in QEMU.

And I think

Quoted 3 lines

startup is the better place here. This allows to create definitions that will be useful in the future (pre-planning), e.g. if you know that you will update your machine or the code soon.

#17 lore raw
Quoted 41 lines · On Tue, 14 May 2019 10:37:32 +0200 Christian Borntraeger…
Quoted 40 lines · David Hildenbrand
Quoted 3 lines
Quoted 3 lines
Quoted 1 line

But that can be tested using the runability information if I am not wrong.

You mean the cpu level information, right?

Yes, query-cpu-definition includes for each model runability information via "unavailable-features" (valid under the started QEMU machine).

Quoted 6 lines
Quoted 6 lines
Quoted 6 lines
Quoted 5 lines

and others that we have today.

So yes, I think this would be acceptable.

Quoted 8 lines
Quoted 5 lines
Quoted 2 lines

I guess it is acceptable yes. I doubt anybody uses that many CPUs in production either way. But you never know.

I think that using that many cpus is a more uncommon setup, but I still think that having to wait for actual failure

That can happen all the time today. You can easily say z14 in the xml when on a zEC12. Only at startup you get the error. The question is really:

"-smp 248 -cpu host" will no longer work, while e.g. "-smp 248 -cpu z12" will work. Actually, even "-smp 248" will no longer work on affected machines.

That is why wonder if it is better to disable the feature and print a warning. Similar to CMMA, where want want to tolerate when CMMA is not possible in the current environment (huge pages).

"Diag318 will not be enabled because it is not compatible with more than 240 CPUs".

However, I still think that implementing support for more than one SCLP response page is the best solution. Guests will need adaptions for > 240 CPUs with Diag318, but who cares? Existing setups will continue to work.

Implementing that SCLP thingy will avoid any warnings and any errors. It just works from the QEMU perspective.

Quoted 6 lines
Quoted 1 line

Is implementing this realistic?

Yes it is but it will take time. I will try to get this rolling. To make progress on the diag318 thing, can we error on startup now and simply remove that check when when have implemented a larger sccb? If we would now do all kinds of "change the max number games" would be harder to "fix".

So, the idea right now is:

- fail to start if you try to specify a diag318 device and more than 240 cpus (do we need a knob to turn off the device?)
- in the future, support more than one SCLP response page

I'm getting a bit lost in the discussion; but the above sounds reasonable to me.

#18 lore raw
Quoted 51 lines · Cornelia Huck
Quoted 41 lines · On Tue, 14 May 2019 10:37:32 +0200 Christian Borntraeger…
Quoted 40 lines · David Hildenbrand
Quoted 3 lines
Quoted 3 lines
Quoted 1 line

But that can be tested using the runability information if I am not wrong.

You mean the cpu level information, right?

Yes, query-cpu-definition includes for each model runability information via "unavailable-features" (valid under the started QEMU machine).

Quoted 6 lines
Quoted 6 lines
Quoted 6 lines
Quoted 5 lines

and others that we have today.

So yes, I think this would be acceptable.

Quoted 8 lines
Quoted 5 lines
Quoted 2 lines

I guess it is acceptable yes. I doubt anybody uses that many CPUs in production either way. But you never know.

I think that using that many cpus is a more uncommon setup, but I still think that having to wait for actual failure

That can happen all the time today. You can easily say z14 in the xml when on a zEC12. Only at startup you get the error. The question is really:

"-smp 248 -cpu host" will no longer work, while e.g. "-smp 248 -cpu z12" will work. Actually, even "-smp 248" will no longer work on affected machines.

That is why wonder if it is better to disable the feature and print a warning. Similar to CMMA, where want want to tolerate when CMMA is not possible in the current environment (huge pages).

"Diag318 will not be enabled because it is not compatible with more than 240 CPUs".

However, I still think that implementing support for more than one SCLP response page is the best solution. Guests will need adaptions for > 240 CPUs with Diag318, but who cares? Existing setups will continue to work.

Implementing that SCLP thingy will avoid any warnings and any errors. It just works from the QEMU perspective.

Quoted 6 lines
Quoted 1 line

Is implementing this realistic?

Yes it is but it will take time. I will try to get this rolling. To make progress on the diag318 thing, can we error on startup now and simply remove that check when when have implemented a larger sccb? If we would now do all kinds of "change the max number games" would be harder to "fix".

Quoted 4 lines

So, the idea right now is:

- fail to start if you try to specify a diag318 device and more than 240 cpus (do we need a knob to turn off the device?)

The know will be the cpu-model e.g. -cpu z14,diag318=off or something like that

Quoted 5 lines

- in the future, support more than one SCLP response page

I'm getting a bit lost in the discussion; but the above sounds reasonable to me.

#19 lore raw
In reply to Cornelia Huck
Quoted 51 lines · Cornelia Huck
Quoted 41 lines · On Tue, 14 May 2019 10:37:32 +0200 Christian Borntraeger…
Quoted 40 lines · David Hildenbrand
Quoted 3 lines
Quoted 3 lines
Quoted 1 line

But that can be tested using the runability information if I am not wrong.

You mean the cpu level information, right?

Yes, query-cpu-definition includes for each model runability information via "unavailable-features" (valid under the started QEMU machine).

Quoted 6 lines
Quoted 6 lines
Quoted 6 lines
Quoted 5 lines

and others that we have today.

So yes, I think this would be acceptable.

Quoted 8 lines
Quoted 5 lines
Quoted 2 lines

I guess it is acceptable yes. I doubt anybody uses that many CPUs in production either way. But you never know.

I think that using that many cpus is a more uncommon setup, but I still think that having to wait for actual failure

That can happen all the time today. You can easily say z14 in the xml when on a zEC12. Only at startup you get the error. The question is really:

"-smp 248 -cpu host" will no longer work, while e.g. "-smp 248 -cpu z12" will work. Actually, even "-smp 248" will no longer work on affected machines.

That is why wonder if it is better to disable the feature and print a warning. Similar to CMMA, where want want to tolerate when CMMA is not possible in the current environment (huge pages).

"Diag318 will not be enabled because it is not compatible with more than 240 CPUs".

However, I still think that implementing support for more than one SCLP response page is the best solution. Guests will need adaptions for > 240 CPUs with Diag318, but who cares? Existing setups will continue to work.

Implementing that SCLP thingy will avoid any warnings and any errors. It just works from the QEMU perspective.

Quoted 6 lines
Quoted 1 line

Is implementing this realistic?

Yes it is but it will take time. I will try to get this rolling. To make progress on the diag318 thing, can we error on startup now and simply remove that check when when have implemented a larger sccb? If we would now do all kinds of "change the max number games" would be harder to "fix".

Quoted 9 lines

So, the idea right now is:

- fail to start if you try to specify a diag318 device and more than 240 cpus (do we need a knob to turn off the device?)
- in the future, support more than one SCLP response page

I'm getting a bit lost in the discussion; but the above sounds reasonable to me.

We can

1. Fail to start with #cpus > 240 when diag318=on
2. Remove the error once we support more than one SCLP response page

Or

1. Allow to start with #cpus > 240 when diag318=on, but indicate only 240 CPUs via SCLP
2. Print a warning
3. Remove the restriction and the warning once we support more than one SCLP response page

While I prefer the second approach (similar to defining zPCI devices without zpci=on), I could also live with the first approach.

Signature · 3 lines
Thanks,

David / dhildenb
#20 lore raw
Quoted 69 lines · David Hildenbrand
Quoted 51 lines · Cornelia Huck
Quoted 41 lines · On Tue, 14 May 2019 10:37:32 +0200 Christian Borntraeger…
Quoted 40 lines · David Hildenbrand
Quoted 3 lines
Quoted 3 lines
Quoted 1 line

But that can be tested using the runability information if I am not wrong.

You mean the cpu level information, right?

Yes, query-cpu-definition includes for each model runability information via "unavailable-features" (valid under the started QEMU machine).

Quoted 6 lines
Quoted 6 lines
Quoted 6 lines
Quoted 5 lines

and others that we have today.

So yes, I think this would be acceptable.

Quoted 8 lines
Quoted 5 lines
Quoted 2 lines

I guess it is acceptable yes. I doubt anybody uses that many CPUs in production either way. But you never know.

I think that using that many cpus is a more uncommon setup, but I still think that having to wait for actual failure

That can happen all the time today. You can easily say z14 in the xml when on a zEC12. Only at startup you get the error. The question is really:

"-smp 248 -cpu host" will no longer work, while e.g. "-smp 248 -cpu z12" will work. Actually, even "-smp 248" will no longer work on affected machines.

That is why wonder if it is better to disable the feature and print a warning. Similar to CMMA, where want want to tolerate when CMMA is not possible in the current environment (huge pages).

"Diag318 will not be enabled because it is not compatible with more than 240 CPUs".

However, I still think that implementing support for more than one SCLP response page is the best solution. Guests will need adaptions for > 240 CPUs with Diag318, but who cares? Existing setups will continue to work.

Implementing that SCLP thingy will avoid any warnings and any errors. It just works from the QEMU perspective.

Quoted 6 lines
Quoted 1 line

Is implementing this realistic?

Yes it is but it will take time. I will try to get this rolling. To make progress on the diag318 thing, can we error on startup now and simply remove that check when when have implemented a larger sccb? If we would now do all kinds of "change the max number games" would be harder to "fix".

Quoted 9 lines

So, the idea right now is:

- fail to start if you try to specify a diag318 device and more than 240 cpus (do we need a knob to turn off the device?)
- in the future, support more than one SCLP response page

I'm getting a bit lost in the discussion; but the above sounds reasonable to me.

We can

1. Fail to start with #cpus > 240 when diag318=on
2. Remove the error once we support more than one SCLP response page

Or

Quoted 8 lines

1. Allow to start with #cpus > 240 when diag318=on, but indicate only 240 CPUs via SCLP
2. Print a warning
3. Remove the restriction and the warning once we support more than one SCLP response page

While I prefer the second approach (similar to defining zPCI devices without zpci=on), I could also live with the first approach.

Lets just continue with your other suggestion to simply limit the sclp response and do not do any failure or machine change. That seems like the easiest solution.

#21 lore raw
Quoted 7 lines · On Tue, 14 May 2019 11:07:41 +0200 Christian Borntraeger…
Quoted 5 lines · David Hildenbrand

We can

1. Fail to start with #cpus > 240 when diag318=on
2. Remove the error once we support more than one SCLP response page

Quoted 7 lines
Quoted 7 lines

Or

1. Allow to start with #cpus > 240 when diag318=on, but indicate only 240 CPUs via SCLP
2. Print a warning
3. Remove the restriction and the warning once we support more than one SCLP response page

We'd need compat handling for step 3., then?

Quoted 7 lines
Quoted 3 lines

While I prefer the second approach (similar to defining zPCI devices without zpci=on), I could also live with the first approach.

Lets just continue with your other suggestion to simply limit the sclp response and do not do any failure or machine change. That seems like the easiest solution.

That's the second option, right? Should be reasonable.

#22 lore raw
In reply to David Hildenbrand
Quoted 69 lines · David Hildenbrand
Quoted 51 lines · Cornelia Huck
Quoted 41 lines · On Tue, 14 May 2019 10:37:32 +0200 Christian Borntraeger…
Quoted 40 lines · David Hildenbrand
Quoted 3 lines
Quoted 3 lines
Quoted 1 line

But that can be tested using the runability information if I am not wrong.

You mean the cpu level information, right?

Yes, query-cpu-definition includes for each model runability information via "unavailable-features" (valid under the started QEMU machine).

Quoted 6 lines
Quoted 6 lines
Quoted 6 lines
Quoted 5 lines

and others that we have today.

So yes, I think this would be acceptable.

Quoted 8 lines
Quoted 5 lines
Quoted 2 lines

I guess it is acceptable yes. I doubt anybody uses that many CPUs in production either way. But you never know.

I think that using that many cpus is a more uncommon setup, but I still think that having to wait for actual failure

That can happen all the time today. You can easily say z14 in the xml when on a zEC12. Only at startup you get the error. The question is really:

"-smp 248 -cpu host" will no longer work, while e.g. "-smp 248 -cpu z12" will work. Actually, even "-smp 248" will no longer work on affected machines.

That is why wonder if it is better to disable the feature and print a warning. Similar to CMMA, where want want to tolerate when CMMA is not possible in the current environment (huge pages).

"Diag318 will not be enabled because it is not compatible with more than 240 CPUs".

However, I still think that implementing support for more than one SCLP response page is the best solution. Guests will need adaptions for > 240 CPUs with Diag318, but who cares? Existing setups will continue to work.

Implementing that SCLP thingy will avoid any warnings and any errors. It just works from the QEMU perspective.

Quoted 6 lines
Quoted 1 line

Is implementing this realistic?

Yes it is but it will take time. I will try to get this rolling. To make progress on the diag318 thing, can we error on startup now and simply remove that check when when have implemented a larger sccb? If we would now do all kinds of "change the max number games" would be harder to "fix".

Quoted 9 lines

So, the idea right now is:

- fail to start if you try to specify a diag318 device and more than 240 cpus (do we need a knob to turn off the device?)
- in the future, support more than one SCLP response page

I'm getting a bit lost in the discussion; but the above sounds reasonable to me.

We can

1. Fail to start with #cpus > 240 when diag318=on
2. Remove the error once we support more than one SCLP response page

Or

Quoted 8 lines

1. Allow to start with #cpus > 240 when diag318=on, but indicate only 240 CPUs via SCLP
2. Print a warning
3. Remove the restriction and the warning once we support more than one SCLP response page

While I prefer the second approach (similar to defining zPCI devices without zpci=on), I could also live with the first approach.

I prefer approach 1.

#23 lore raw
Quoted 78 lines · Christian Borntraeger
Quoted 75 lines · David Hildenbrand
Quoted 51 lines · Cornelia Huck
Quoted 41 lines · On Tue, 14 May 2019 10:37:32 +0200 Christian Borntraeger…
Quoted 40 lines · David Hildenbrand
Quoted 3 lines
Quoted 3 lines
Quoted 1 line

But that can be tested using the runability information if I am not wrong.

You mean the cpu level information, right?

Yes, query-cpu-definition includes for each model runability information via "unavailable-features" (valid under the started QEMU machine).

Quoted 6 lines
Quoted 6 lines
Quoted 6 lines
Quoted 5 lines

and others that we have today.

So yes, I think this would be acceptable.

Quoted 8 lines
Quoted 5 lines
Quoted 2 lines

I guess it is acceptable yes. I doubt anybody uses that many CPUs in production either way. But you never know.

I think that using that many cpus is a more uncommon setup, but I still think that having to wait for actual failure

That can happen all the time today. You can easily say z14 in the xml when on a zEC12. Only at startup you get the error. The question is really:

"-smp 248 -cpu host" will no longer work, while e.g. "-smp 248 -cpu z12" will work. Actually, even "-smp 248" will no longer work on affected machines.

That is why wonder if it is better to disable the feature and print a warning. Similar to CMMA, where want want to tolerate when CMMA is not possible in the current environment (huge pages).

"Diag318 will not be enabled because it is not compatible with more than 240 CPUs".

However, I still think that implementing support for more than one SCLP response page is the best solution. Guests will need adaptions for > 240 CPUs with Diag318, but who cares? Existing setups will continue to work.

Implementing that SCLP thingy will avoid any warnings and any errors. It just works from the QEMU perspective.

Quoted 6 lines
Quoted 1 line

Is implementing this realistic?

Yes it is but it will take time. I will try to get this rolling. To make progress on the diag318 thing, can we error on startup now and simply remove that check when when have implemented a larger sccb? If we would now do all kinds of "change the max number games" would be harder to "fix".

Quoted 9 lines

So, the idea right now is:

- fail to start if you try to specify a diag318 device and more than 240 cpus (do we need a knob to turn off the device?)
- in the future, support more than one SCLP response page

I'm getting a bit lost in the discussion; but the above sounds reasonable to me.

We can

1. Fail to start with #cpus > 240 when diag318=on
2. Remove the error once we support more than one SCLP response page

Or

1. Allow to start with #cpus > 240 when diag318=on, but indicate only 240 CPUs via SCLP
2. Print a warning
3. Remove the restriction and the warning once we support more than one SCLP response page

Quoted 5 lines
Quoted 2 lines

While I prefer the second approach (similar to defining zPCI devices without zpci=on), I could also live with the first approach.

I prefer approach 1.

Isn't approach #2 what we discussed (limiting sclp, but of course to 247 CPUs), but with an additional warning? I'm confused.

Signature · 3 lines
Thanks,

David / dhildenb
#24 lore raw
Quoted 79 lines · David Hildenbrand
Quoted 78 lines · Christian Borntraeger
Quoted 75 lines · David Hildenbrand
Quoted 51 lines · Cornelia Huck
Quoted 41 lines · On Tue, 14 May 2019 10:37:32 +0200 Christian Borntraeger…
Quoted 40 lines · David Hildenbrand
Quoted 3 lines
Quoted 3 lines
Quoted 1 line

But that can be tested using the runability information if I am not wrong.

You mean the cpu level information, right?

Yes, query-cpu-definition includes for each model runability information via "unavailable-features" (valid under the started QEMU machine).

Quoted 6 lines
Quoted 6 lines
Quoted 6 lines
Quoted 5 lines

and others that we have today.

So yes, I think this would be acceptable.

Quoted 8 lines
Quoted 5 lines
Quoted 2 lines

I guess it is acceptable yes. I doubt anybody uses that many CPUs in production either way. But you never know.

I think that using that many cpus is a more uncommon setup, but I still think that having to wait for actual failure

That can happen all the time today. You can easily say z14 in the xml when on a zEC12. Only at startup you get the error. The question is really:

"-smp 248 -cpu host" will no longer work, while e.g. "-smp 248 -cpu z12" will work. Actually, even "-smp 248" will no longer work on affected machines.

That is why wonder if it is better to disable the feature and print a warning. Similar to CMMA, where want want to tolerate when CMMA is not possible in the current environment (huge pages).

"Diag318 will not be enabled because it is not compatible with more than 240 CPUs".

However, I still think that implementing support for more than one SCLP response page is the best solution. Guests will need adaptions for > 240 CPUs with Diag318, but who cares? Existing setups will continue to work.

Implementing that SCLP thingy will avoid any warnings and any errors. It just works from the QEMU perspective.

Quoted 6 lines
Quoted 1 line

Is implementing this realistic?

Yes it is but it will take time. I will try to get this rolling. To make progress on the diag318 thing, can we error on startup now and simply remove that check when when have implemented a larger sccb? If we would now do all kinds of "change the max number games" would be harder to "fix".

Quoted 9 lines

So, the idea right now is:

- fail to start if you try to specify a diag318 device and more than 240 cpus (do we need a knob to turn off the device?)
- in the future, support more than one SCLP response page

I'm getting a bit lost in the discussion; but the above sounds reasonable to me.

We can

1. Fail to start with #cpus > 240 when diag318=on
2. Remove the error once we support more than one SCLP response page

Or

1. Allow to start with #cpus > 240 when diag318=on, but indicate only 240 CPUs via SCLP
2. Print a warning
3. Remove the restriction and the warning once we support more than one SCLP response page

Quoted 8 lines
Quoted 5 lines
Quoted 2 lines

While I prefer the second approach (similar to defining zPCI devices without zpci=on), I could also live with the first approach.

I prefer approach 1.

Isn't approach #2 what we discussed (limiting sclp, but of course to 247 CPUs), but with an additional warning? I'm confused.

Different numbering interpretion. I was talking about 1 = "Allow to start with #cpus > 240 when diag318=on, but indicate only 240 CPUs via SCLP"

#25 lore raw
Quoted 85 lines · Christian Borntraeger
Quoted 82 lines · David Hildenbrand
Quoted 81 lines · Christian Borntraeger
Quoted 77 lines · David Hildenbrand
Quoted 51 lines · Cornelia Huck
Quoted 41 lines · On Tue, 14 May 2019 10:37:32 +0200 Christian Borntraeger…
Quoted 40 lines · David Hildenbrand
Quoted 3 lines
Quoted 3 lines
Quoted 1 line

But that can be tested using the runability information if I am not wrong.

You mean the cpu level information, right?

Yes, query-cpu-definition includes for each model runability information via "unavailable-features" (valid under the started QEMU machine).

Quoted 6 lines
Quoted 6 lines
Quoted 6 lines
Quoted 5 lines

and others that we have today.

So yes, I think this would be acceptable.

Quoted 8 lines
Quoted 5 lines
Quoted 2 lines

I guess it is acceptable yes. I doubt anybody uses that many CPUs in production either way. But you never know.

I think that using that many cpus is a more uncommon setup, but I still think that having to wait for actual failure

That can happen all the time today. You can easily say z14 in the xml when on a zEC12. Only at startup you get the error. The question is really:

"-smp 248 -cpu host" will no longer work, while e.g. "-smp 248 -cpu z12" will work. Actually, even "-smp 248" will no longer work on affected machines.

That is why wonder if it is better to disable the feature and print a warning. Similar to CMMA, where want want to tolerate when CMMA is not possible in the current environment (huge pages).

"Diag318 will not be enabled because it is not compatible with more than 240 CPUs".

However, I still think that implementing support for more than one SCLP response page is the best solution. Guests will need adaptions for > 240 CPUs with Diag318, but who cares? Existing setups will continue to work.

Implementing that SCLP thingy will avoid any warnings and any errors. It just works from the QEMU perspective.

Quoted 6 lines
Quoted 1 line

Is implementing this realistic?

Yes it is but it will take time. I will try to get this rolling. To make progress on the diag318 thing, can we error on startup now and simply remove that check when when have implemented a larger sccb? If we would now do all kinds of "change the max number games" would be harder to "fix".

Quoted 9 lines

So, the idea right now is:

- fail to start if you try to specify a diag318 device and more than 240 cpus (do we need a knob to turn off the device?)
- in the future, support more than one SCLP response page

I'm getting a bit lost in the discussion; but the above sounds reasonable to me.

We can

1. Fail to start with #cpus > 240 when diag318=on
2. Remove the error once we support more than one SCLP response page

Or

1. Allow to start with #cpus > 240 when diag318=on, but indicate only 240 CPUs via SCLP
2. Print a warning
3. Remove the restriction and the warning once we support more than one SCLP response page

While I prefer the second approach (similar to defining zPCI devices without zpci=on), I could also live with the first approach.

Quoted 8 lines
Quoted 5 lines
Quoted 2 lines

I prefer approach 1.

Isn't approach #2 what we discussed (limiting sclp, but of course to 247 CPUs), but with an additional warning? I'm confused.

Different numbering interpretion. I was talking about 1 = "Allow to start with #cpus > 240 when diag318=on, but indicate only 240 CPUs via SCLP"

So yes, variant 2 when I use your numbering. The only question is: do we need a warning? It probably does not hurt.

#26 lore raw
Quoted 91 lines · Christian Borntraeger
Quoted 88 lines · Christian Borntraeger
Quoted 85 lines · David Hildenbrand
Quoted 83 lines · Christian Borntraeger
Quoted 69 lines · David Hildenbrand
Quoted 51 lines · Cornelia Huck
Quoted 41 lines · On Tue, 14 May 2019 10:37:32 +0200 Christian Borntraeger…
Quoted 40 lines · David Hildenbrand
Quoted 3 lines
Quoted 3 lines
Quoted 1 line

But that can be tested using the runability information if I am not wrong.

You mean the cpu level information, right?

Yes, query-cpu-definition includes for each model runability information via "unavailable-features" (valid under the started QEMU machine).

Quoted 6 lines
Quoted 6 lines
Quoted 6 lines
Quoted 5 lines

and others that we have today.

So yes, I think this would be acceptable.

Quoted 8 lines
Quoted 5 lines
Quoted 2 lines

I guess it is acceptable yes. I doubt anybody uses that many CPUs in production either way. But you never know.

I think that using that many cpus is a more uncommon setup, but I still think that having to wait for actual failure

That can happen all the time today. You can easily say z14 in the xml when on a zEC12. Only at startup you get the error. The question is really:

"-smp 248 -cpu host" will no longer work, while e.g. "-smp 248 -cpu z12" will work. Actually, even "-smp 248" will no longer work on affected machines.

That is why wonder if it is better to disable the feature and print a warning. Similar to CMMA, where want want to tolerate when CMMA is not possible in the current environment (huge pages).

"Diag318 will not be enabled because it is not compatible with more than 240 CPUs".

However, I still think that implementing support for more than one SCLP response page is the best solution. Guests will need adaptions for > 240 CPUs with Diag318, but who cares? Existing setups will continue to work.

Implementing that SCLP thingy will avoid any warnings and any errors. It just works from the QEMU perspective.

Quoted 6 lines
Quoted 1 line

Is implementing this realistic?

Yes it is but it will take time. I will try to get this rolling. To make progress on the diag318 thing, can we error on startup now and simply remove that check when when have implemented a larger sccb? If we would now do all kinds of "change the max number games" would be harder to "fix".

Quoted 9 lines

So, the idea right now is:

- fail to start if you try to specify a diag318 device and more than 240 cpus (do we need a knob to turn off the device?)
- in the future, support more than one SCLP response page

I'm getting a bit lost in the discussion; but the above sounds reasonable to me.

We can

1. Fail to start with #cpus > 240 when diag318=on
2. Remove the error once we support more than one SCLP response page

Or

Quoted 8 lines

1. Allow to start with #cpus > 240 when diag318=on, but indicate only 240 CPUs via SCLP
2. Print a warning
3. Remove the restriction and the warning once we support more than one SCLP response page

While I prefer the second approach (similar to defining zPCI devices without zpci=on), I could also live with the first approach.

I prefer approach 1.

Quoted 8 lines
Quoted 5 lines
Quoted 2 lines

Isn't approach #2 what we discussed (limiting sclp, but of course to 247 CPUs), but with an additional warning? I'm confused.

Different numbering interpretion. I was talking about 1 = "Allow to start with #cpus > 240 when diag318=on, but indicate only 240 CPUs via SCLP"

So yes, variant 2 when I use your numbering. The only question is: do we need a warning? It probably does not hurt.

After all, we are talking about 1 VCPU that the guest can only use by indirect probing ... I leave that up to Collin :)

Signature · 3 lines
Thanks,

David / dhildenb
#27 lore raw
Quoted 34 lines · On Tue, 14 May 2019 11:27:32 +0200 David Hildenbrand <[email protected]> wrote:
Quoted 33 lines · Christian Borntraeger
Quoted 30 lines · Christian Borntraeger
Quoted 26 lines · David Hildenbrand
Quoted 17 lines · Christian Borntraeger
Quoted 13 lines · David Hildenbrand

We can

1. Fail to start with #cpus > 240 when diag318=on
2. Remove the error once we support more than one SCLP response page

Or

1. Allow to start with #cpus > 240 when diag318=on, but indicate only 240 CPUs via SCLP
2. Print a warning
3. Remove the restriction and the warning once we support more than one SCLP response page

Quoted 5 lines
Quoted 2 lines

While I prefer the second approach (similar to defining zPCI devices without zpci=on), I could also live with the first approach.

I prefer approach 1.

Isn't approach #2 what we discussed (limiting sclp, but of course to 247 CPUs), but with an additional warning? I'm confused.

Quoted 8 lines
Quoted 5 lines
Quoted 2 lines

Different numbering interpretion. I was talking about 1 = "Allow to start with #cpus > 240 when diag318=on, but indicate only 240 CPUs via SCLP"

So yes, variant 2 when I use your numbering. The only question is: do we need a warning? It probably does not hurt.

After all, we are talking about 1 VCPU that the guest can only use by indirect probing ... I leave that up to Collin :)

I'd prefer a warning... even if it is a corner case, I think it's better to be explicit instead of silent.

#28 lore raw
Quoted 40 lines · Cornelia Huck
Quoted 37 lines · On Tue, 14 May 2019 11:27:32 +0200 David Hildenbrand <[email protected]> wrote:
Quoted 36 lines · Christian Borntraeger
Quoted 32 lines · Christian Borntraeger
Quoted 18 lines · David Hildenbrand
Quoted 17 lines · Christian Borntraeger
Quoted 13 lines · David Hildenbrand

We can

1. Fail to start with #cpus > 240 when diag318=on
2. Remove the error once we support more than one SCLP response page

Or
b
1. Allow to start with #cpus > 240 when diag318=on, but indicate only 240 CPUs via SCLP
2. Print a warning
3. Remove the restriction and the warning once we support more than one SCLP response page

Quoted 8 lines
Quoted 5 lines
Quoted 2 lines

While I prefer the second approach (similar to defining zPCI devices without zpci=on), I could also live with the first approach.

I prefer approach 1.

Isn't approach #2 what we discussed (limiting sclp, but of course to 247 CPUs), but with an additional warning? I'm confused.

Different numbering interpretion. I was talking about 1 = "Allow to start with #cpus > 240 when diag318=on, but indicate only 240 CPUs via SCLP"

Quoted 9 lines
Quoted 5 lines
Quoted 2 lines

So yes, variant 2 when I use your numbering. The only question is: do we need a warning? It probably does not hurt.

After all, we are talking about 1 VCPU that the guest can only use by indirect probing ... I leave that up to Collin :)

I'd prefer a warning... even if it is a corner case, I think it's better to be explicit instead of silent.

Thanks for the discussion. I'll implement diag318 as a CPU feature again and adjust the max_cpu handling (to 247 now instead of 240).

Question: should diag318 be enabled by default for current CPU models? What should we do in the case where diag318=on but KVM does not support handling?

I am not familiar with how close QEMU and KVM versions stay in sync.

#29 lore raw
Quoted 47 lines · Collin Walling
Quoted 46 lines · Cornelia Huck
Quoted 42 lines · On Tue, 14 May 2019 11:27:32 +0200 David Hildenbrand <[email protected]> wrote:
Quoted 30 lines · Christian Borntraeger
Quoted 27 lines · Christian Borntraeger
Quoted 24 lines · David Hildenbrand
Quoted 22 lines · Christian Borntraeger
Quoted 13 lines · David Hildenbrand

We can

1. Fail to start with #cpus > 240 when diag318=on
2. Remove the error once we support more than one SCLP response page

Or
b
1. Allow to start with #cpus > 240 when diag318=on, but indicate only 240 CPUs via SCLP
2. Print a warning
3. Remove the restriction and the warning once we support more than one SCLP response page

Quoted 2 lines

While I prefer the second approach (similar to defining zPCI devices without zpci=on), I could also live with the first approach.

I prefer approach 1.

Quoted 8 lines
Quoted 5 lines
Quoted 2 lines

Isn't approach #2 what we discussed (limiting sclp, but of course to 247 CPUs), but with an additional warning? I'm confused.

Different numbering interpretion. I was talking about 1 = "Allow to start with #cpus > 240 when diag318=on, but indicate only 240 CPUs via SCLP"

So yes, variant 2 when I use your numbering. The only question is: do we need a warning? It probably does not hurt.

After all, we are talking about 1 VCPU that the guest can only use by indirect probing ... I leave that up to Collin :)

Quoted 6 lines
Quoted 3 lines

I'd prefer a warning... even if it is a corner case, I think it's better to be explicit instead of silent.

Thanks for the discussion. I'll implement diag318 as a CPU feature again and adjust the max_cpu handling (to 247 now instead of 240).

Well, max cpu is 248, we decided to just not report cpus > 247 to the guest via sclp.

Quoted 4 lines

Question: should diag318 be enabled by default for current CPU models? What should we do in the case where diag318=on but KVM does not support handling?

Lets keep it in the full model for now. we can change that later on.

Quoted 2 lines

I am not familiar with how close QEMU and KVM versions stay in sync.

#30 lore raw
In reply to Christian Borntraeger
Quoted 43 lines · Christian Borntraeger
Quoted 40 lines · David Hildenbrand
Quoted 3 lines
Quoted 3 lines
Quoted 1 line

But that can be tested using the runability information if I am not wrong.

You mean the cpu level information, right?

Yes, query-cpu-definition includes for each model runability information via "unavailable-features" (valid under the started QEMU machine).

Quoted 6 lines
Quoted 6 lines
Quoted 6 lines
Quoted 5 lines

and others that we have today.

So yes, I think this would be acceptable.

Quoted 8 lines
Quoted 5 lines
Quoted 2 lines

I guess it is acceptable yes. I doubt anybody uses that many CPUs in production either way. But you never know.

I think that using that many cpus is a more uncommon setup, but I still think that having to wait for actual failure

That can happen all the time today. You can easily say z14 in the xml when on a zEC12. Only at startup you get the error. The question is really:

"-smp 248 -cpu host" will no longer work, while e.g. "-smp 248 -cpu z12" will work. Actually, even "-smp 248" will no longer work on affected machines.

That is why wonder if it is better to disable the feature and print a warning. Similar to CMMA, where want want to tolerate when CMMA is not possible in the current environment (huge pages).

"Diag318 will not be enabled because it is not compatible with more than 240 CPUs".

However, I still think that implementing support for more than one SCLP response page is the best solution. Guests will need adaptions for > 240 CPUs with Diag318, but who cares? Existing setups will continue to work.

Implementing that SCLP thingy will avoid any warnings and any errors. It just works from the QEMU perspective.

Quoted 6 lines
Quoted 1 line

Is implementing this realistic?

Yes it is but it will take time. I will try to get this rolling. To make progress on the diag318 thing, can we error on startup now and simply remove that check when when have implemented a larger sccb? If we would now do all kinds of "change the max number games" would be harder to "fix".

Another idea for temporary handling: Simply only indicate 240 CPUs to the guest if the response does not fit into a page. Once we have that SCLP thingy, this will be fixed. Guest migration back and forth should work, as the VCPUs are fully functional (and initially always stopped), the guest will simply not be able to detect them via SCLP when booting up, and therefore not use them.

Signature · 3 lines
Thanks,

David / dhildenb
#31 lore raw
Quoted 52 lines · David Hildenbrand
Quoted 43 lines · Christian Borntraeger
Quoted 40 lines · David Hildenbrand
Quoted 3 lines
Quoted 3 lines
Quoted 1 line

But that can be tested using the runability information if I am not wrong.

You mean the cpu level information, right?

Yes, query-cpu-definition includes for each model runability information via "unavailable-features" (valid under the started QEMU machine).

Quoted 6 lines
Quoted 6 lines
Quoted 6 lines
Quoted 5 lines

and others that we have today.

So yes, I think this would be acceptable.

Quoted 8 lines
Quoted 5 lines
Quoted 2 lines

I guess it is acceptable yes. I doubt anybody uses that many CPUs in production either way. But you never know.

I think that using that many cpus is a more uncommon setup, but I still think that having to wait for actual failure

That can happen all the time today. You can easily say z14 in the xml when on a zEC12. Only at startup you get the error. The question is really:

"-smp 248 -cpu host" will no longer work, while e.g. "-smp 248 -cpu z12" will work. Actually, even "-smp 248" will no longer work on affected machines.

That is why wonder if it is better to disable the feature and print a warning. Similar to CMMA, where want want to tolerate when CMMA is not possible in the current environment (huge pages).

"Diag318 will not be enabled because it is not compatible with more than 240 CPUs".

However, I still think that implementing support for more than one SCLP response page is the best solution. Guests will need adaptions for > 240 CPUs with Diag318, but who cares? Existing setups will continue to work.

Implementing that SCLP thingy will avoid any warnings and any errors. It just works from the QEMU perspective.

Quoted 6 lines
Quoted 1 line

Is implementing this realistic?

Yes it is but it will take time. I will try to get this rolling. To make progress on the diag318 thing, can we error on startup now and simply remove that check when when have implemented a larger sccb? If we would now do all kinds of "change the max number games" would be harder to "fix".

Quoted 6 lines

Another idea for temporary handling: Simply only indicate 240 CPUs to the guest if the response does not fit into a page. Once we have that SCLP thingy, this will be fixed. Guest migration back and forth should work, as the VCPUs are fully functional (and initially always stopped), the guest will simply not be able to detect them via SCLP when booting up, and therefore not use them.

Yes, that looks like a good temporary solution. In fact if the guest relies on simply probing it could even make use of the additional CPUs. Its just the sclp response that is limited to 240 (or make it 247?)

#32 lore raw
Quoted 9 lines · On Tue, 14 May 2019 10:56:43 +0200 Christian Borntraeger…
Quoted 6 lines · David Hildenbrand

Another idea for temporary handling: Simply only indicate 240 CPUs to the guest if the response does not fit into a page. Once we have that SCLP thingy, this will be fixed. Guest migration back and forth should work, as the VCPUs are fully functional (and initially always stopped), the guest will simply not be able to detect them via SCLP when booting up, and therefore not use them.

Quoted 3 lines

Yes, that looks like a good temporary solution. In fact if the guest relies on simply probing it could even make use of the additional CPUs. Its just the sclp response that is limited to 240 (or make it 247?)

Where did the 240 come from - extra spare room? If so, 247 would probably be all right?

#33 lore raw
Quoted 12 lines · Cornelia Huck
Quoted 9 lines · On Tue, 14 May 2019 10:56:43 +0200 Christian Borntraeger…
Quoted 6 lines · David Hildenbrand

Another idea for temporary handling: Simply only indicate 240 CPUs to the guest if the response does not fit into a page. Once we have that SCLP thingy, this will be fixed. Guest migration back and forth should work, as the VCPUs are fully functional (and initially always stopped), the guest will simply not be able to detect them via SCLP when booting up, and therefore not use them.

Quoted 7 lines
Quoted 3 lines

Yes, that looks like a good temporary solution. In fact if the guest relies on simply probing it could even make use of the additional CPUs. Its just the sclp response that is limited to 240 (or make it 247?)

Where did the 240 come from - extra spare room? If so, 247 would probably be all right?

+++ b/include/hw/s390x/sclp.h

@@ -133,6 +133,8 @@ typedef struct ReadInfo {
     uint16_t highest_cpu;
     uint8_t  _reserved5[124 - 122];     /* 122-123 */
     uint32_t hmfai;
+    uint8_t  _reserved7[134 - 128];     /* 128-133 */
+    uint8_t  fac134;
     struct CPUEntry entries[0];

} QEMU_PACKED ReadInfo;

So we have "4096 - 135 + 1" memory. Each element is 16 bytes wide. -> 246 CPUs fit.

Signature · 3 lines
Thanks,

David / dhildenb
#34 lore raw
Quoted 32 lines · David Hildenbrand
Quoted 12 lines · Cornelia Huck
Quoted 9 lines · On Tue, 14 May 2019 10:56:43 +0200 Christian Borntraeger…
Quoted 6 lines · David Hildenbrand

Another idea for temporary handling: Simply only indicate 240 CPUs to the guest if the response does not fit into a page. Once we have that SCLP thingy, this will be fixed. Guest migration back and forth should work, as the VCPUs are fully functional (and initially always stopped), the guest will simply not be able to detect them via SCLP when booting up, and therefore not use them.

Quoted 7 lines
Quoted 3 lines

Yes, that looks like a good temporary solution. In fact if the guest relies on simply probing it could even make use of the additional CPUs. Its just the sclp response that is limited to 240 (or make it 247?)

Where did the 240 come from - extra spare room? If so, 247 would probably be all right?

+++ b/include/hw/s390x/sclp.h

(quoted diff) +2 -0
@@ -133,6 +133,8 @@ typedef struct ReadInfo {
     uint16_t highest_cpu;
     uint8_t  _reserved5[124 - 122];     /* 122-123 */
     uint32_t hmfai;
+    uint8_t  _reserved7[134 - 128];     /* 128-133 */
+    uint8_t  fac134;
     struct CPUEntry entries[0];
 } QEMU_PACKED ReadInfo;
 
Quoted 2 lines

So we have "4096 - 135 + 1" memory. Each element is 16 bytes wide. -> 246 CPUs fit.

(I meant 247 :( )

Signature · 3 lines
Thanks,

David / dhildenb
#35 lore raw
In reply to Christian Borntraeger
Quoted 62 lines · Christian Borntraeger
Quoted 52 lines · David Hildenbrand
Quoted 43 lines · Christian Borntraeger
Quoted 40 lines · David Hildenbrand
Quoted 3 lines
Quoted 3 lines
Quoted 1 line

But that can be tested using the runability information if I am not wrong.

You mean the cpu level information, right?

Yes, query-cpu-definition includes for each model runability information via "unavailable-features" (valid under the started QEMU machine).

Quoted 6 lines
Quoted 6 lines
Quoted 6 lines
Quoted 5 lines

and others that we have today.

So yes, I think this would be acceptable.

Quoted 8 lines
Quoted 5 lines
Quoted 2 lines

I guess it is acceptable yes. I doubt anybody uses that many CPUs in production either way. But you never know.

I think that using that many cpus is a more uncommon setup, but I still think that having to wait for actual failure

That can happen all the time today. You can easily say z14 in the xml when on a zEC12. Only at startup you get the error. The question is really:

"-smp 248 -cpu host" will no longer work, while e.g. "-smp 248 -cpu z12" will work. Actually, even "-smp 248" will no longer work on affected machines.

That is why wonder if it is better to disable the feature and print a warning. Similar to CMMA, where want want to tolerate when CMMA is not possible in the current environment (huge pages).

"Diag318 will not be enabled because it is not compatible with more than 240 CPUs".

However, I still think that implementing support for more than one SCLP response page is the best solution. Guests will need adaptions for > 240 CPUs with Diag318, but who cares? Existing setups will continue to work.

Implementing that SCLP thingy will avoid any warnings and any errors. It just works from the QEMU perspective.

Quoted 6 lines
Quoted 1 line

Is implementing this realistic?

Yes it is but it will take time. I will try to get this rolling. To make progress on the diag318 thing, can we error on startup now and simply remove that check when when have implemented a larger sccb? If we would now do all kinds of "change the max number games" would be harder to "fix".

Quoted 6 lines

Another idea for temporary handling: Simply only indicate 240 CPUs to the guest if the response does not fit into a page. Once we have that SCLP thingy, this will be fixed. Guest migration back and forth should work, as the VCPUs are fully functional (and initially always stopped), the guest will simply not be able to detect them via SCLP when booting up, and therefore not use them.

Quoted 3 lines

Yes, that looks like a good temporary solution. In fact if the guest relies on simply probing it could even make use of the additional CPUs. Its just the sclp response that is limited to 240 (or make it 247?)

I think the limiting factor was more than a single CPU, but I don't recall. We can do the math again and come up with the right number.

Signature · 3 lines
Thanks,

David / dhildenb
#36 lore raw
Quoted 63 lines · David Hildenbrand
Quoted 62 lines · Christian Borntraeger
Quoted 58 lines · David Hildenbrand
Quoted 43 lines · Christian Borntraeger
Quoted 40 lines · David Hildenbrand
Quoted 3 lines
Quoted 3 lines
Quoted 1 line

But that can be tested using the runability information if I am not wrong.

You mean the cpu level information, right?

Yes, query-cpu-definition includes for each model runability information via "unavailable-features" (valid under the started QEMU machine).

Quoted 6 lines
Quoted 6 lines
Quoted 6 lines
Quoted 5 lines

and others that we have today.

So yes, I think this would be acceptable.

Quoted 8 lines
Quoted 5 lines
Quoted 2 lines

I guess it is acceptable yes. I doubt anybody uses that many CPUs in production either way. But you never know.

I think that using that many cpus is a more uncommon setup, but I still think that having to wait for actual failure

That can happen all the time today. You can easily say z14 in the xml when on a zEC12. Only at startup you get the error. The question is really:

"-smp 248 -cpu host" will no longer work, while e.g. "-smp 248 -cpu z12" will work. Actually, even "-smp 248" will no longer work on affected machines.

That is why wonder if it is better to disable the feature and print a warning. Similar to CMMA, where want want to tolerate when CMMA is not possible in the current environment (huge pages).

"Diag318 will not be enabled because it is not compatible with more than 240 CPUs".

However, I still think that implementing support for more than one SCLP response page is the best solution. Guests will need adaptions for > 240 CPUs with Diag318, but who cares? Existing setups will continue to work.

Implementing that SCLP thingy will avoid any warnings and any errors. It just works from the QEMU perspective.

Quoted 6 lines
Quoted 1 line

Is implementing this realistic?

Yes it is but it will take time. I will try to get this rolling. To make progress on the diag318 thing, can we error on startup now and simply remove that check when when have implemented a larger sccb? If we would now do all kinds of "change the max number games" would be harder to "fix".

Another idea for temporary handling: Simply only indicate 240 CPUs to the guest if the response does not fit into a page. Once we have that SCLP thingy, this will be fixed. Guest migration back and forth should work, as the VCPUs are fully functional (and initially always stopped), the guest will simply not be able to detect them via SCLP when booting up, and therefore not use them.

Quoted 6 lines
Quoted 3 lines

Yes, that looks like a good temporary solution. In fact if the guest relies on simply probing it could even make use of the additional CPUs. Its just the sclp response that is limited to 240 (or make it 247?)

I think the limiting factor was more than a single CPU, but I don't recall. We can do the math again and come up with the right number.

I think We need 8 byte per CPU. With byte 134 we should still be ok with
247. Collin can do the math in the patch description.

#37 lore raw
Quoted 70 lines · Christian Borntraeger
Quoted 67 lines · David Hildenbrand
Quoted 65 lines · Christian Borntraeger
Quoted 52 lines · David Hildenbrand
Quoted 43 lines · Christian Borntraeger
Quoted 40 lines · David Hildenbrand
Quoted 3 lines
Quoted 3 lines
Quoted 1 line

But that can be tested using the runability information if I am not wrong.

You mean the cpu level information, right?

Yes, query-cpu-definition includes for each model runability information via "unavailable-features" (valid under the started QEMU machine).

Quoted 6 lines
Quoted 6 lines
Quoted 6 lines
Quoted 5 lines

and others that we have today.

So yes, I think this would be acceptable.

Quoted 8 lines
Quoted 5 lines
Quoted 2 lines

I guess it is acceptable yes. I doubt anybody uses that many CPUs in production either way. But you never know.

I think that using that many cpus is a more uncommon setup, but I still think that having to wait for actual failure

That can happen all the time today. You can easily say z14 in the xml when on a zEC12. Only at startup you get the error. The question is really:

"-smp 248 -cpu host" will no longer work, while e.g. "-smp 248 -cpu z12" will work. Actually, even "-smp 248" will no longer work on affected machines.

That is why wonder if it is better to disable the feature and print a warning. Similar to CMMA, where want want to tolerate when CMMA is not possible in the current environment (huge pages).

"Diag318 will not be enabled because it is not compatible with more than 240 CPUs".

However, I still think that implementing support for more than one SCLP response page is the best solution. Guests will need adaptions for > 240 CPUs with Diag318, but who cares? Existing setups will continue to work.

Implementing that SCLP thingy will avoid any warnings and any errors. It just works from the QEMU perspective.

Quoted 6 lines
Quoted 1 line

Is implementing this realistic?

Yes it is but it will take time. I will try to get this rolling. To make progress on the diag318 thing, can we error on startup now and simply remove that check when when have implemented a larger sccb? If we would now do all kinds of "change the max number games" would be harder to "fix".

Quoted 6 lines

Another idea for temporary handling: Simply only indicate 240 CPUs to the guest if the response does not fit into a page. Once we have that SCLP thingy, this will be fixed. Guest migration back and forth should work, as the VCPUs are fully functional (and initially always stopped), the guest will simply not be able to detect them via SCLP when booting up, and therefore not use them.

Yes, that looks like a good temporary solution. In fact if the guest relies on simply probing it could even make use of the additional CPUs. Its just the sclp response that is limited to 240 (or make it 247?)

Quoted 7 lines
Quoted 2 lines

I think the limiting factor was more than a single CPU, but I don't recall. We can do the math again and come up with the right number.

I think We need 8 byte per CPU. With byte 134 we should still be ok with
247. Collin can do the math in the patch description.

Yes 247 fits just fine. The 240 came up as extra space in case we expand the rscpi even more in the future.

I used the

The SCCB_SIZE - sizeof(ReadInfo)) / sizeof(CPUEntry) < S390_MAX_CPUS

calculation as an example of what we could do if the SCCB_SIZE ever increases (it would allow older machines to retroactively allow diag318 if they can also support a larger SCCB... but thinking out loud, there might be many more moving parts that would make this preemptive approach too tricky to implement today).