Skip to contentSkip to conversation

pseries: avoid unplug the last online CPU core + assorted fixes

PATCH Applied Daniel Henrique Barboza 3 participants 23 messages Jan 14, 2021 – Jan 18, 2021

lore
Share & tools
lore
#0 lore raw

cover pseries: avoid unplug the last online CPU core + assorted fixes

Hello,

The last 2 patches of this series contains a fix for a hotunplug situation that we have in the pseries machine, where hotunpluging the last online CPU of the guest causes the machine to behave not that great (https://bugzilla.redhat.com/show_bug.cgi?id=1911414).

First 5 patches are somewhat trivial changes that I ended up doing while investigating.

Daniel Henrique Barboza (7):
spapr.h: fix trailing whitespace in phb_placement spapr_hcall.c: make do_client_architecture_support static spapr_rtas.c: fix identation in rtas_ibm_nmi_interlock() string spapr_rtas.c: fix identation of rtas_ibm_suspend_me() args spapr_cpu_core.c: use g_auto* in spapr_create_vcpu() spapr.c: introduce spapr_core_unplug_possible() spapr.c: consider CPU core online state before allowing unplug

 hw/ppc/spapr.c          | 66 ++++++++++++++++++++++++++++++++++++-----
 hw/ppc/spapr_cpu_core.c | 12 ++------
 hw/ppc/spapr_hcall.c    |  1 +
 hw/ppc/spapr_rtas.c     |  9 +++---
 include/hw/ppc/spapr.h  |  7 +----
 5 files changed, 68 insertions(+), 27 deletions(-)
Signature · 1 line
2.26.2
#1 lore raw

PATCH 1/7 spapr.h: fix trailing whitespace in phb_placement

This whitespace was messing with lots of diffs if you happen to use an editor that eliminates trailing whitespaces on file save.

---
 include/hw/ppc/spapr.h | 2 +-
 1 file changed, 1 insertion(+), 1 deletion(-)
include/hw/ppc/spapr.h +1 -1
index 1cc19575f5..4d37c2a93b 100644
@@ -145,7 +145,7 @@ struct SpaprMachineClass {
     bool pre_6_0_memory_unplug;
 
     bool (*phb_placement)(SpaprMachineState *spapr, uint32_t index,
-                          uint64_t *buid, hwaddr *pio, 
+                          uint64_t *buid, hwaddr *pio,
                           hwaddr *mmio32, hwaddr *mmio64,
                           unsigned n_dma, uint32_t *liobns, hwaddr *nv2gpa,
                           hwaddr *nv2atsd, Error **errp);
Signature · 1 line
2.26.2
Applied
#2 lore raw

Re: [PATCH v1 1/7] spapr.h: fix trailing whitespace in phb_placement

Quoted 5 lines · Daniel Henrique Barboza

This whitespace was messing with lots of diffs if you happen to use an editor that eliminates trailing whitespaces on file save.

Applied to ppc-for-6.0, thanks.

Quoted 17 lines
---
 include/hw/ppc/spapr.h | 2 +-
 1 file changed, 1 insertion(+), 1 deletion(-)
include/hw/ppc/spapr.h +1 -1
index 1cc19575f5..4d37c2a93b 100644
@@ -145,7 +145,7 @@ struct SpaprMachineClass {
     bool pre_6_0_memory_unplug;
 
     bool (*phb_placement)(SpaprMachineState *spapr, uint32_t index,
-                          uint64_t *buid, hwaddr *pio, 
+                          uint64_t *buid, hwaddr *pio,
                           hwaddr *mmio32, hwaddr *mmio64,
                           unsigned n_dma, uint32_t *liobns, hwaddr *nv2gpa,
                           hwaddr *nv2atsd, Error **errp);
Signature · 4 lines
David Gibson			| I'll have my music baroque, and my code
david AT gibson.dropbear.id.au	| minimalist, thank you.  NOT _the_ _other_
				| _way_ _around_!
http://www.ozlabs.org/~dgibson
#3 lore raw

PATCH 2/7 spapr_hcall.c: make do_client_architecture_support static

In reply to Daniel Henrique Barboza

The function is called only inside spapr_hcall.c.

---
 hw/ppc/spapr_hcall.c   | 1 +
 include/hw/ppc/spapr.h | 5 -----
 2 files changed, 1 insertion(+), 5 deletions(-)
hw/ppc/spapr_hcall.c +1 -0
index e5dfc1ba7a..7b5cd3553c 100644
@@ -1632,6 +1632,7 @@ static uint32_t cas_check_pvr(PowerPCCPU *cpu, uint32_t max_compat,
     return best_compat;
 }
 
+static
 target_ulong do_client_architecture_support(PowerPCCPU *cpu,
                                             SpaprMachineState *spapr,
                                             target_ulong vec,
include/hw/ppc/spapr.h +0 -5
index 4d37c2a93b..a2d1fb21c6 100644
@@ -583,11 +583,6 @@ void spapr_register_hypercall(target_ulong opcode, spapr_hcall_fn fn);
 target_ulong spapr_hypercall(PowerPCCPU *cpu, target_ulong opcode,
                              target_ulong *args);
 
-target_ulong do_client_architecture_support(PowerPCCPU *cpu,
-                                            SpaprMachineState *spapr,
-                                            target_ulong addr,
-                                            target_ulong fdt_bufsize);
-
 /* Virtual Processor Area structure constants */
 #define VPA_MIN_SIZE           640
 #define VPA_SIZE_OFFSET        0x4
Signature · 1 line
2.26.2
#4 lore raw

Re: [PATCH v1 2/7] spapr_hcall.c: make do_client_architecture_support static

Quoted 3 lines · Daniel Henrique Barboza

The function is called only inside spapr_hcall.c.

Nice catch - that's a hangover from an earlier version where it was called elswhere. Applied to ppc-for-6.0.

Quoted 33 lines
---
 hw/ppc/spapr_hcall.c   | 1 +
 include/hw/ppc/spapr.h | 5 -----
 2 files changed, 1 insertion(+), 5 deletions(-)
hw/ppc/spapr_hcall.c +1 -0
index e5dfc1ba7a..7b5cd3553c 100644
@@ -1632,6 +1632,7 @@ static uint32_t cas_check_pvr(PowerPCCPU *cpu, uint32_t max_compat,
     return best_compat;
 }
 
+static
 target_ulong do_client_architecture_support(PowerPCCPU *cpu,
                                             SpaprMachineState *spapr,
                                             target_ulong vec,
include/hw/ppc/spapr.h +0 -5
index 4d37c2a93b..a2d1fb21c6 100644
@@ -583,11 +583,6 @@ void spapr_register_hypercall(target_ulong opcode, spapr_hcall_fn fn);
 target_ulong spapr_hypercall(PowerPCCPU *cpu, target_ulong opcode,
                              target_ulong *args);
 
-target_ulong do_client_architecture_support(PowerPCCPU *cpu,
-                                            SpaprMachineState *spapr,
-                                            target_ulong addr,
-                                            target_ulong fdt_bufsize);
-
 /* Virtual Processor Area structure constants */
 #define VPA_MIN_SIZE           640
 #define VPA_SIZE_OFFSET        0x4
Signature · 4 lines
David Gibson			| I'll have my music baroque, and my code
david AT gibson.dropbear.id.au	| minimalist, thank you.  NOT _the_ _other_
				| _way_ _around_!
http://www.ozlabs.org/~dgibson
#5 lore raw

PATCH 3/7 spapr_rtas.c: fix identation in rtas_ibm_nmi_interlock() string

In reply to Daniel Henrique Barboza
---
 hw/ppc/spapr_rtas.c | 3 ++-
 1 file changed, 2 insertions(+), 1 deletion(-)
hw/ppc/spapr_rtas.c +2 -1
index 513c7a8435..d014684d60 100644
@@ -463,7 +463,8 @@ static void rtas_ibm_nmi_interlock(PowerPCCPU *cpu,
 
     if (spapr->fwnmi_machine_check_addr == -1) {
         qemu_log_mask(LOG_GUEST_ERROR,
-"FWNMI: ibm,nmi-interlock RTAS called with FWNMI not registered.\n");
+                      "FWNMI: ibm,nmi-interlock RTAS called with "
+                      "FWNMI not registered.\n");
 
         /* NMI register not called */
         rtas_st(rets, 0, RTAS_OUT_PARAM_ERROR);
Signature · 1 line
2.26.2
#6 lore raw

Re: [PATCH v1 3/7] spapr_rtas.c: fix identation in rtas_ibm_nmi_interlock() string

Quoted 1 line · Daniel Henrique Barboza

Nack. I believe the general consensus is that greppability for long error message strings is sufficient reason to break the normal coding style rules.

Quoted 18 lines
---
 hw/ppc/spapr_rtas.c | 3 ++-
 1 file changed, 2 insertions(+), 1 deletion(-)
hw/ppc/spapr_rtas.c +2 -1
index 513c7a8435..d014684d60 100644
@@ -463,7 +463,8 @@ static void rtas_ibm_nmi_interlock(PowerPCCPU *cpu,
 
     if (spapr->fwnmi_machine_check_addr == -1) {
         qemu_log_mask(LOG_GUEST_ERROR,
-"FWNMI: ibm,nmi-interlock RTAS called with FWNMI not registered.\n");
+                      "FWNMI: ibm,nmi-interlock RTAS called with "
+                      "FWNMI not registered.\n");
 
         /* NMI register not called */
         rtas_st(rets, 0, RTAS_OUT_PARAM_ERROR);
Signature · 4 lines
David Gibson			| I'll have my music baroque, and my code
david AT gibson.dropbear.id.au	| minimalist, thank you.  NOT _the_ _other_
				| _way_ _around_!
http://www.ozlabs.org/~dgibson
#7 lore raw

PATCH 4/7 spapr_rtas.c: fix identation of rtas_ibm_suspend_me() args

In reply to Daniel Henrique Barboza
---
 hw/ppc/spapr_rtas.c | 6 +++---
 1 file changed, 3 insertions(+), 3 deletions(-)
hw/ppc/spapr_rtas.c +3 -3
index d014684d60..fbefd5d3cb 100644
@@ -219,9 +219,9 @@ static void rtas_stop_self(PowerPCCPU *cpu, SpaprMachineState *spapr,
 }
 
 static void rtas_ibm_suspend_me(PowerPCCPU *cpu, SpaprMachineState *spapr,
-                           uint32_t token, uint32_t nargs,
-                           target_ulong args,
-                           uint32_t nret, target_ulong rets)
+                                uint32_t token, uint32_t nargs,
+                                target_ulong args,
+                                uint32_t nret, target_ulong rets)
 {
     CPUState *cs;
 
Signature · 1 line
2.26.2
Applied
#8 lore raw

Re: [PATCH v1 4/7] spapr_rtas.c: fix identation of rtas_ibm_suspend_me() args

Quoted 1 line · Daniel Henrique Barboza

Applied to ppc-for-6.0, thanks.

Quoted 21 lines
---
 hw/ppc/spapr_rtas.c | 6 +++---
 1 file changed, 3 insertions(+), 3 deletions(-)
hw/ppc/spapr_rtas.c +3 -3
index d014684d60..fbefd5d3cb 100644
@@ -219,9 +219,9 @@ static void rtas_stop_self(PowerPCCPU *cpu, SpaprMachineState *spapr,
 }
 
 static void rtas_ibm_suspend_me(PowerPCCPU *cpu, SpaprMachineState *spapr,
-                           uint32_t token, uint32_t nargs,
-                           target_ulong args,
-                           uint32_t nret, target_ulong rets)
+                                uint32_t token, uint32_t nargs,
+                                target_ulong args,
+                                uint32_t nret, target_ulong rets)
 {
     CPUState *cs;
 
Signature · 4 lines
David Gibson			| I'll have my music baroque, and my code
david AT gibson.dropbear.id.au	| minimalist, thank you.  NOT _the_ _other_
				| _way_ _around_!
http://www.ozlabs.org/~dgibson
#9 lore raw

PATCH 5/7 spapr_cpu_core.c: use g_auto* in spapr_create_vcpu()

In reply to Daniel Henrique Barboza

Use g_autoptr() with Object and g_autofree with the string to avoid the need of a cleanup path.

---
 hw/ppc/spapr_cpu_core.c | 12 +++---------
 1 file changed, 3 insertions(+), 9 deletions(-)
hw/ppc/spapr_cpu_core.c +3 -9
index 2f7dc3c23d..4f316a6f9d 100644
@@ -277,8 +277,8 @@ static PowerPCCPU *spapr_create_vcpu(SpaprCpuCore *sc, int i, Error **errp)
 {
     SpaprCpuCoreClass *scc = SPAPR_CPU_CORE_GET_CLASS(sc);
     CPUCore *cc = CPU_CORE(sc);
-    Object *obj;
-    char *id;
+    g_autoptr(Object) obj = NULL;
+    g_autofree char *id = NULL;
     CPUState *cs;
     PowerPCCPU *cpu;
 
@@ -293,23 +293,17 @@ static PowerPCCPU *spapr_create_vcpu(SpaprCpuCore *sc, int i, Error **errp)
     cs->start_powered_off = true;
     cs->cpu_index = cc->core_id + i;
     if (!spapr_set_vcpu_id(cpu, cs->cpu_index, errp)) {
-        goto err;
+        return NULL;
     }
 
     cpu->node_id = sc->node_id;
 
     id = g_strdup_printf("thread[%d]", i);
     object_property_add_child(OBJECT(sc), id, obj);
-    g_free(id);
 
     cpu->machine_data = g_new0(SpaprCpuState, 1);
 
-    object_unref(obj);
     return cpu;
-
-err:
-    object_unref(obj);
-    return NULL;
 }
 
 static void spapr_cpu_core_realize(DeviceState *dev, Error **errp)
Signature · 1 line
2.26.2
Applied
#10 lore raw

Re: [PATCH v1 5/7] spapr_cpu_core.c: use g_auto* in spapr_create_vcpu()

Quoted 4 lines · Daniel Henrique Barboza

Use g_autoptr() with Object and g_autofree with the string to avoid the need of a cleanup path.

Applied to ppc-for-6.0, thanks.

Quoted 44 lines
---
 hw/ppc/spapr_cpu_core.c | 12 +++---------
 1 file changed, 3 insertions(+), 9 deletions(-)
hw/ppc/spapr_cpu_core.c +3 -9
index 2f7dc3c23d..4f316a6f9d 100644
@@ -277,8 +277,8 @@ static PowerPCCPU *spapr_create_vcpu(SpaprCpuCore *sc, int i, Error **errp)
 {
     SpaprCpuCoreClass *scc = SPAPR_CPU_CORE_GET_CLASS(sc);
     CPUCore *cc = CPU_CORE(sc);
-    Object *obj;
-    char *id;
+    g_autoptr(Object) obj = NULL;
+    g_autofree char *id = NULL;
     CPUState *cs;
     PowerPCCPU *cpu;
 
@@ -293,23 +293,17 @@ static PowerPCCPU *spapr_create_vcpu(SpaprCpuCore *sc, int i, Error **errp)
     cs->start_powered_off = true;
     cs->cpu_index = cc->core_id + i;
     if (!spapr_set_vcpu_id(cpu, cs->cpu_index, errp)) {
-        goto err;
+        return NULL;
     }
 
     cpu->node_id = sc->node_id;
 
     id = g_strdup_printf("thread[%d]", i);
     object_property_add_child(OBJECT(sc), id, obj);
-    g_free(id);
 
     cpu->machine_data = g_new0(SpaprCpuState, 1);
 
-    object_unref(obj);
     return cpu;
-
-err:
-    object_unref(obj);
-    return NULL;
 }
 
 static void spapr_cpu_core_realize(DeviceState *dev, Error **errp)
Signature · 4 lines
David Gibson			| I'll have my music baroque, and my code
david AT gibson.dropbear.id.au	| minimalist, thank you.  NOT _the_ _other_
				| _way_ _around_!
http://www.ozlabs.org/~dgibson
#11 lore raw

PATCH 6/7 spapr.c: introduce spapr_core_unplug_possible()

In reply to Daniel Henrique Barboza

Next patch is going to add more conditions to allow a CPU core hotunplug. Let's put it into a separated function to avoid crowding the body of spapr_core_unplug_request().

---
 hw/ppc/spapr.c | 27 ++++++++++++++++++++-------
 1 file changed, 20 insertions(+), 7 deletions(-)
hw/ppc/spapr.c +20 -7
index 2c403b574e..a2f01c21aa 100644
@@ -3706,22 +3706,35 @@ static void spapr_core_unplug(HotplugHandler *hotplug_dev, DeviceState *dev)
     qdev_unrealize(dev);
 }
 
-static
-void spapr_core_unplug_request(HotplugHandler *hotplug_dev, DeviceState *dev,
-                               Error **errp)
+static int spapr_core_unplug_possible(HotplugHandler *hotplug_dev, CPUCore *cc,
+                                      Error **errp)
 {
-    SpaprMachineState *spapr = SPAPR_MACHINE(OBJECT(hotplug_dev));
     int index;
-    SpaprDrc *drc;
-    CPUCore *cc = CPU_CORE(dev);
 
     if (!spapr_find_cpu_slot(MACHINE(hotplug_dev), cc->core_id, &index)) {
         error_setg(errp, "Unable to find CPU core with core-id: %d",
                    cc->core_id);
-        return;
+        return -1;
     }
+
     if (index == 0) {
         error_setg(errp, "Boot CPU core may not be unplugged");
+        return -1;
+    }
+
+    return 0;
+}
+
+static
+void spapr_core_unplug_request(HotplugHandler *hotplug_dev, DeviceState *dev,
+                               Error **errp)
+{
+    ERRP_GUARD();
+    SpaprMachineState *spapr = SPAPR_MACHINE(OBJECT(hotplug_dev));
+    SpaprDrc *drc;
+    CPUCore *cc = CPU_CORE(dev);
+
+    if (spapr_core_unplug_possible(hotplug_dev, cc, errp) < 0) {
         return;
     }
 
Signature · 1 line
2.26.2
#12 lore raw

Re: [PATCH v1 6/7] spapr.c: introduce spapr_core_unplug_possible()

Quoted 5 lines · Daniel Henrique Barboza

Next patch is going to add more conditions to allow a CPU core hotunplug. Let's put it into a separated function to avoid crowding the body of spapr_core_unplug_request().

LGTM, except for naming. I'd expect a function named *_possible() to return a boolean, where "true" means it *is* possible, rather than a 0-or-negative-error value.

So you could either change the type and sense of the function, or change the name to, say spapr_core_unplug_check(), which I would expect to return an error code.

Quoted 51 lines
---
 hw/ppc/spapr.c | 27 ++++++++++++++++++++-------
 1 file changed, 20 insertions(+), 7 deletions(-)
hw/ppc/spapr.c +20 -7
index 2c403b574e..a2f01c21aa 100644
@@ -3706,22 +3706,35 @@ static void spapr_core_unplug(HotplugHandler *hotplug_dev, DeviceState *dev)
     qdev_unrealize(dev);
 }
 
-static
-void spapr_core_unplug_request(HotplugHandler *hotplug_dev, DeviceState *dev,
-                               Error **errp)
+static int spapr_core_unplug_possible(HotplugHandler *hotplug_dev, CPUCore *cc,
+                                      Error **errp)
 {
-    SpaprMachineState *spapr = SPAPR_MACHINE(OBJECT(hotplug_dev));
     int index;
-    SpaprDrc *drc;
-    CPUCore *cc = CPU_CORE(dev);
 
     if (!spapr_find_cpu_slot(MACHINE(hotplug_dev), cc->core_id, &index)) {
         error_setg(errp, "Unable to find CPU core with core-id: %d",
                    cc->core_id);
-        return;
+        return -1;
     }
+
     if (index == 0) {
         error_setg(errp, "Boot CPU core may not be unplugged");
+        return -1;
+    }
+
+    return 0;
+}
+
+static
+void spapr_core_unplug_request(HotplugHandler *hotplug_dev, DeviceState *dev,
+                               Error **errp)
+{
+    ERRP_GUARD();
+    SpaprMachineState *spapr = SPAPR_MACHINE(OBJECT(hotplug_dev));
+    SpaprDrc *drc;
+    CPUCore *cc = CPU_CORE(dev);
+
+    if (spapr_core_unplug_possible(hotplug_dev, cc, errp) < 0) {
         return;
     }
 
Signature · 4 lines
David Gibson			| I'll have my music baroque, and my code
david AT gibson.dropbear.id.au	| minimalist, thank you.  NOT _the_ _other_
				| _way_ _around_!
http://www.ozlabs.org/~dgibson
#13 lore raw

PATCH 7/7 spapr.c: consider CPU core online state before allowing unplug

In reply to Daniel Henrique Barboza

The only restriction we have when unplugging CPUs is to forbid unplug of the boot cpu core. spapr_core_unplug_possible() does not contemplate the possibility of some cores being offlined by the guest, meaning that we're rolling the dice regarding on whether we're unplugging the last online CPU core the guest has.

If we hit the jackpot, we're going to detach the core DRC and pulse the hotplug IRQ, but the guest OS will refuse to release the CPU. Our spapr_core_unplug() DRC release callback will never be called and the CPU core object will keep existing in QEMU. No error message will be sent to the user, but the CPU core wasn't unplugged from the guest.

If the guest OS onlines the CPU core again we won't be able to hotunplug it either. 'dmesg' inside the guest will report a failed attempt to offline an unknown CPU:

[  923.003994] pseries-hotplug-cpu: Failed to offline CPU <NULL>, rc: -16

This is the result of stopping the DRC state transition in the middle in the first failed attempt.

We can avoid this, and potentially other bad things from happening, if we avoid to attempt the unplug altogether in this scenario. Let's check for the online/offline state of the CPU cores in the guest before allowing the hotunplug, and forbid removing a CPU core if it's the last one online in the guest.

---
 hw/ppc/spapr.c | 39 ++++++++++++++++++++++++++++++++++++++-
 1 file changed, 38 insertions(+), 1 deletion(-)
hw/ppc/spapr.c +38 -1
index a2f01c21aa..d269dcd102 100644
@@ -3709,9 +3709,16 @@ static void spapr_core_unplug(HotplugHandler *hotplug_dev, DeviceState *dev)
 static int spapr_core_unplug_possible(HotplugHandler *hotplug_dev, CPUCore *cc,
                                       Error **errp)
 {
+    CPUArchId *core_slot;
+    SpaprCpuCore *core;
+    PowerPCCPU *cpu;
+    CPUState *cs;
+    bool last_cpu_online = true;
     int index;
 
-    if (!spapr_find_cpu_slot(MACHINE(hotplug_dev), cc->core_id, &index)) {
+    core_slot = spapr_find_cpu_slot(MACHINE(hotplug_dev), cc->core_id,
+                                    &index);
+    if (!core_slot) {
         error_setg(errp, "Unable to find CPU core with core-id: %d",
                    cc->core_id);
         return -1;
@@ -3722,6 +3729,36 @@ static int spapr_core_unplug_possible(HotplugHandler *hotplug_dev, CPUCore *cc,
         return -1;
     }
 
+    /* Allow for any non-boot CPU core to be unplugged if already offline */
+    core = SPAPR_CPU_CORE(core_slot->cpu);
+    cs = CPU(core->threads[0]);
+    if (cs->halted) {
+        return 0;
+    }
+
+    /*
+     * Do not allow core unplug if it's the last core online.
+     */
+    cpu = POWERPC_CPU(cs);
+    CPU_FOREACH(cs) {
+        PowerPCCPU *c = POWERPC_CPU(cs);
+
+        if (c == cpu) {
+            continue;
+        }
+
+        if (!cs->halted) {
+            last_cpu_online = false;
+            break;
+        }
+    }
+
+    if (last_cpu_online) {
+        error_setg(errp, "Unable to unplug CPU core with core-id %d: it is "
+                   "the only CPU core online in the guest", cc->core_id);
+        return -1;
+    }
+
     return 0;
 }
 
Signature · 1 line
2.26.2
#14 lore raw

Re: [PATCH v1 7/7] spapr.c: consider CPU core online state before allowing unplug

Quoted 18 lines · Daniel Henrique Barboza

The only restriction we have when unplugging CPUs is to forbid unplug of the boot cpu core. spapr_core_unplug_possible() does not contemplate the possibility of some cores being offlined by the guest, meaning that we're rolling the dice regarding on whether we're unplugging the last online CPU core the guest has.

If we hit the jackpot, we're going to detach the core DRC and pulse the hotplug IRQ, but the guest OS will refuse to release the CPU. Our spapr_core_unplug() DRC release callback will never be called and the CPU core object will keep existing in QEMU. No error message will be sent to the user, but the CPU core wasn't unplugged from the guest.

If the guest OS onlines the CPU core again we won't be able to hotunplug it either. 'dmesg' inside the guest will report a failed attempt to offline an unknown CPU:

[  923.003994] pseries-hotplug-cpu: Failed to offline CPU <NULL>, rc: -16
Quoted 8 lines

This is the result of stopping the DRC state transition in the middle in the first failed attempt.

We can avoid this, and potentially other bad things from happening, if we avoid to attempt the unplug altogether in this scenario. Let's check for the online/offline state of the CPU cores in the guest before allowing the hotunplug, and forbid removing a CPU core if it's the last one online in the guest.

Good explanation overall, but I think it would be a bit clearer and more direct if you remove the "roll the dice" / "hit the jackpot" metaphor.

Quoted 12 lines
---
 hw/ppc/spapr.c | 39 ++++++++++++++++++++++++++++++++++++++-
 1 file changed, 38 insertions(+), 1 deletion(-)
hw/ppc/spapr.c +0 -0
index a2f01c21aa..d269dcd102 100644

hw/ppc/spapr.c static void spapr_core_unplug(HotplugHandler *hotplug_dev, DeviceState *dev) · line 3710

Quoted 2 lines
hw/ppc/spapr.c +0 -0
@@ -3709,9 +3709,16 @@ static void spapr_core_unplug(HotplugHandler *hotplug_dev, DeviceState *dev)
 static int spapr_core_unplug_possible(HotplugHandler *hotplug_dev, CPUCore *cc,
                                       Error **errp)

This will need a small rework w.r.t. my suggestions for the previous patch, obviously.

Quoted 16 lines
hw/ppc/spapr.c +8 -1
 {
+    CPUArchId *core_slot;
+    SpaprCpuCore *core;
+    PowerPCCPU *cpu;
+    CPUState *cs;
+    bool last_cpu_online = true;
     int index;
 
-    if (!spapr_find_cpu_slot(MACHINE(hotplug_dev), cc->core_id, &index)) {
+    core_slot = spapr_find_cpu_slot(MACHINE(hotplug_dev), cc->core_id,
+                                    &index);
+    if (!core_slot) {
         error_setg(errp, "Unable to find CPU core with core-id: %d",
                    cc->core_id);
         return -1;

hw/ppc/spapr.c static int spapr_core_unplug_possible(HotplugHandler *hotplug_dev, CPUCore *cc, · line 3737

Quoted 9 lines
hw/ppc/spapr.c +6 -0
@@ -3722,6 +3729,36 @@ static int spapr_core_unplug_possible(HotplugHandler *hotplug_dev, CPUCore *cc,
         return -1;
     }
 
+    /* Allow for any non-boot CPU core to be unplugged if already offline */
+    core = SPAPR_CPU_CORE(core_slot->cpu);
+    cs = CPU(core->threads[0]);
+    if (cs->halted) {
+        return 0;
+    }

I think you need to check that *all* the cpu's threads are offline, not just thread 0 for this to be correct.

Quoted 7 lines
hw/ppc/spapr.c +7 -0
+
+    /*
+     * Do not allow core unplug if it's the last core online.
+     */
+    cpu = POWERPC_CPU(cs);
+    CPU_FOREACH(cs) {
+        PowerPCCPU *c = POWERPC_CPU(cs);

hw/ppc/spapr.c static int spapr_core_unplug_possible(HotplugHandler *hotplug_dev, CPUCore *cc, · line 3754

Quoted 10 lines
hw/ppc/spapr.c +10 -0
+
+        if (c == cpu) {
+            continue;
+        }
+
+        if (!cs->halted) {
+            last_cpu_online = false;
+            break;
+        }
+    }

Likewise here I think your logic needs more careful handling of threads - you need to disallow the hotplug if all of the currently online threads are on the core slated for removal.

I'm also a little bit worried about whether just checking cs->halted is sufficient. That's a qemu/tcg core concept that I think that may be set in some circumstances when the CPU is *not* offline. The logic of the suspend-me RTAS call is specifically to both set halted *and* to block interrupts so there's nothing that can pull the vcpu out of halted state. It's possible that handling this correctly will require adding some qemu internal state to explicitly track the "online" state of a vcpu as managed by "suspend-me" and "start-cpu" RTAS calls.

Quoted 10 lines
hw/ppc/spapr.c +7 -0
+
+    if (last_cpu_online) {
+        error_setg(errp, "Unable to unplug CPU core with core-id %d: it is "
+                   "the only CPU core online in the guest", cc->core_id);
+        return -1;
+    }
+
     return 0;
 }
 
Signature · 4 lines
David Gibson			| I'll have my music baroque, and my code
david AT gibson.dropbear.id.au	| minimalist, thank you.  NOT _the_ _other_
				| _way_ _around_!
http://www.ozlabs.org/~dgibson
#15 lore raw

Re: [PATCH v1 7/7] spapr.c: consider CPU core online state before allowing unplug

In reply to Daniel Henrique Barboza
Quoted 2 lines · On Thu, 14 Jan 2021 15:06:28 -0300 Daniel Henrique Barboza…

The only restriction we have when unplugging CPUs is to forbid unplug of the boot cpu core. spapr_core_unplug_possible() does not contemplate the

I can't remember why this restriction was introduced in the first place... This should be investigated and documented if the limitation still stands.

Quoted 4 lines

possibility of some cores being offlined by the guest, meaning that we're rolling the dice regarding on whether we're unplugging the last online CPU core the guest has.

Trying to unplug the last CPU is obviously something that deserves special care. LoPAPR is quite explicit on the outcome : this should terminate the partition.

13.7.4.1.1. Isolation of CPUs

The isolation of a CPU, in all cases, is preceded by the stop-self RTAS function for all processor threads, and the OS insures that all the CPU’s threads are in the RTAS stopped state prior to isolating the CPU. Isolation of a processor that is not stopped produces unpredictable results. The stopping of the last processor thread of a LPAR partition effectively kills the partition, and at that point, ownership of all partition resources reverts to the platform firmware.

R1-13.7.4.1.1-1. For the LRDR option: Prior to issuing the RTAS set-indicator specifying isolate isolation-state of a CPU DR connector type, all the CPU threads must be in the RTAS stopped state.

R1-13.7.4.1.1-2. For the LRDR option: Stopping of the last processor thread of a LPAR partition with the stop-self RTAS function, must kill the partition, with ownership of all partition resources reverting to the platform firmware.

This is clearly not how things work today : linux doesn't call "stop-self" on the last vCPU and even if it did, QEMU doesn't terminate the VM.

If there's a valid reason to not implement this PAPR behavior, I'd like it to be documented.

Quoted 6 lines

If we hit the jackpot, we're going to detach the core DRC and pulse the hotplug IRQ, but the guest OS will refuse to release the CPU. Our spapr_core_unplug() DRC release callback will never be called and the CPU core object will keep existing in QEMU. No error message will be sent to the user, but the CPU core wasn't unplugged from the guest.

Quoted 9 lines

If the guest OS onlines the CPU core again we won't be able to hotunplug it either. 'dmesg' inside the guest will report a failed attempt to offline an unknown CPU:

[  923.003994] pseries-hotplug-cpu: Failed to offline CPU <NULL>, rc: -16

This is the result of stopping the DRC state transition in the middle in the first failed attempt.

Yes, at this point only a machine reset can fix things up.

Given this is linux's choice not to call "stop-self" as it should do, I'm not super fan of hardcoding this logic in QEMU, unless there are really good reasons to do so.

Quoted 72 lines

We can avoid this, and potentially other bad things from happening, if we avoid to attempt the unplug altogether in this scenario. Let's check for the online/offline state of the CPU cores in the guest before allowing the hotunplug, and forbid removing a CPU core if it's the last one online in the guest.

---
 hw/ppc/spapr.c | 39 ++++++++++++++++++++++++++++++++++++++-
 1 file changed, 38 insertions(+), 1 deletion(-)
hw/ppc/spapr.c +38 -1
index a2f01c21aa..d269dcd102 100644
@@ -3709,9 +3709,16 @@ static void spapr_core_unplug(HotplugHandler *hotplug_dev, DeviceState *dev)
 static int spapr_core_unplug_possible(HotplugHandler *hotplug_dev, CPUCore *cc,
                                       Error **errp)
 {
+    CPUArchId *core_slot;
+    SpaprCpuCore *core;
+    PowerPCCPU *cpu;
+    CPUState *cs;
+    bool last_cpu_online = true;
     int index;
 
-    if (!spapr_find_cpu_slot(MACHINE(hotplug_dev), cc->core_id, &index)) {
+    core_slot = spapr_find_cpu_slot(MACHINE(hotplug_dev), cc->core_id,
+                                    &index);
+    if (!core_slot) {
         error_setg(errp, "Unable to find CPU core with core-id: %d",
                    cc->core_id);
         return -1;
@@ -3722,6 +3729,36 @@ static int spapr_core_unplug_possible(HotplugHandler *hotplug_dev, CPUCore *cc,
         return -1;
     }
 
+    /* Allow for any non-boot CPU core to be unplugged if already offline */
+    core = SPAPR_CPU_CORE(core_slot->cpu);
+    cs = CPU(core->threads[0]);
+    if (cs->halted) {
+        return 0;
+    }
+
+    /*
+     * Do not allow core unplug if it's the last core online.
+     */
+    cpu = POWERPC_CPU(cs);
+    CPU_FOREACH(cs) {
+        PowerPCCPU *c = POWERPC_CPU(cs);
+
+        if (c == cpu) {
+            continue;
+        }
+
+        if (!cs->halted) {
+            last_cpu_online = false;
+            break;
+        }
+    }
+
+    if (last_cpu_online) {
+        error_setg(errp, "Unable to unplug CPU core with core-id %d: it is "
+                   "the only CPU core online in the guest", cc->core_id);
+        return -1;
+    }
+
     return 0;
 }
 
#16 lore raw

Re: [PATCH v1 7/7] spapr.c: consider CPU core online state before allowing unplug

Quoted 5 lines · Greg Kurz
Quoted 2 lines · On Thu, 14 Jan 2021 15:06:28 -0300 Daniel Henrique Barboza…

The only restriction we have when unplugging CPUs is to forbid unplug of the boot cpu core. spapr_core_unplug_possible() does not contemplate the

I'll look into it.

Quoted 15 lines

I can't remember why this restriction was introduced in the first place... This should be investigated and documented if the limitation still stands.

Quoted 4 lines

possibility of some cores being offlined by the guest, meaning that we're rolling the dice regarding on whether we're unplugging the last online CPU core the guest has.

Trying to unplug the last CPU is obviously something that deserves special care. LoPAPR is quite explicit on the outcome : this should terminate the partition.

13.7.4.1.1. Isolation of CPUs

Quoted 7 lines

The isolation of a CPU, in all cases, is preceded by the stop-self RTAS function for all processor threads, and the OS insures that all the CPU’s threads are in the RTAS stopped state prior to isolating the CPU. Isolation of a processor that is not stopped produces unpredictable results. The stopping of the last processor thread of a LPAR partition effectively kills the partition, and at that point, ownership of all partition resources reverts to the platform firmware.

I was just investigating a reason why we should check for all thread states before unplugging the core, like David suggested in his reply. rtas_stop_self() was setting 'cs->halted = 1' without a thread activity check like ibm,suspend-me() does and I was wondering why. This text you sent explains it, quoting:

"> The isolation of a CPU, in all cases, is preceded by the stop-self
RTAS function for all processor threads, and the OS insures that all the CPU’s threads are in the RTAS stopped state prior to isolating the CPU."

This seems to be corroborated by arch/powerpc/platform/pseries/hotplug-cpu.c:

static void pseries_cpu_offline_self(void)

{
	unsigned int hwcpu = hard_smp_processor_id();

	local_irq_disable();
	idle_task_exit();
	if (xive_enabled())
		xive_teardown_cpu();
	else
		xics_teardown_cpu();

	unregister_slb_shadow(hwcpu);
	rtas_stop_self();

	/* Should never get here... */
	BUG();
	for(;;);
}

IIUC this means that we can rely on cs->halted = 1 since it's coming right after a rtas_stop_self() call. This is still a bit confusing though and I wouldn't mind standardizing the 'CPU core is offline' condition with what ibm,suspend-me is already doing.

David, what do you think?

Quoted 11 lines

R1-13.7.4.1.1-1. For the LRDR option: Prior to issuing the RTAS set-indicator specifying isolate isolation-state of a CPU DR connector type, all the CPU threads must be in the RTAS stopped state.

R1-13.7.4.1.1-2. For the LRDR option: Stopping of the last processor thread of a LPAR partition with the stop-self RTAS function, must kill the partition, with ownership of all partition resources reverting to the platform firmware.

Quoted 6 lines

This is clearly not how things work today : linux doesn't call "stop-self" on the last vCPU and even if it did, QEMU doesn't terminate the VM.

If there's a valid reason to not implement this PAPR behavior, I'd like it to be documented.

I can only speculate. This would create a unorthodox way of shutting down the guest, when the user can just shutdown the whole thing gracefully.

Unless we're considering borderline cases, like the security risk mentioned in the kernel docs (Documentation/core-api/cpu_hotplug.rst):

"Such advances require CPUs available to a kernel to be removed either for provisioning reasons, or for RAS purposes to keep an offending CPU off system execution path. Hence the need for CPU hotplug support in the Linux kernel."

In this extreme scenario I can see a reason to kill the partition/guest by offlining the last online CPU - if it's compromising the host we'd rather terminate immediately instead of waiting for graceful shutdown.

Quoted 17 lines
Quoted 7 lines

If we hit the jackpot, we're going to detach the core DRC and pulse the hotplug IRQ, but the guest OS will refuse to release the CPU. Our spapr_core_unplug() DRC release callback will never be called and the CPU core object will keep existing in QEMU. No error message will be sent to the user, but the CPU core wasn't unplugged from the guest.

Quoted 9 lines

If the guest OS onlines the CPU core again we won't be able to hotunplug it either. 'dmesg' inside the guest will report a failed attempt to offline an unknown CPU:

[  923.003994] pseries-hotplug-cpu: Failed to offline CPU <NULL>, rc: -16

This is the result of stopping the DRC state transition in the middle in the first failed attempt.

Quoted 5 lines

Yes, at this point only a machine reset can fix things up.

Given this is linux's choice not to call "stop-self" as it should do, I'm not super fan of hardcoding this logic in QEMU, unless there are really good reasons to do so.

I understand where are you coming from and I sympathize. Not sure about how users would feel about that though. They expect a somewhat compatible behavior of multi-arch features like hotplug/hotunplug, and x86 will neither hotunplug nor offline the last CPU as well.

There is a very high chance that, even if we pull this design off, I'll need to go to Libvirt and disable it because we broke compatibility with how vcpu unplug operated earlier.

Thanks,

DHB

Quoted 74 lines
Quoted 73 lines

We can avoid this, and potentially other bad things from happening, if we avoid to attempt the unplug altogether in this scenario. Let's check for the online/offline state of the CPU cores in the guest before allowing the hotunplug, and forbid removing a CPU core if it's the last one online in the guest.

---
  hw/ppc/spapr.c | 39 ++++++++++++++++++++++++++++++++++++++-
  1 file changed, 38 insertions(+), 1 deletion(-)
hw/ppc/spapr.c +38 -1
index a2f01c21aa..d269dcd102 100644
@@ -3709,9 +3709,16 @@ static void spapr_core_unplug(HotplugHandler *hotplug_dev, DeviceState *dev)
  static int spapr_core_unplug_possible(HotplugHandler *hotplug_dev, CPUCore *cc,
                                        Error **errp)
  {
+    CPUArchId *core_slot;
+    SpaprCpuCore *core;
+    PowerPCCPU *cpu;
+    CPUState *cs;
+    bool last_cpu_online = true;
      int index;
 
-    if (!spapr_find_cpu_slot(MACHINE(hotplug_dev), cc->core_id, &index)) {
+    core_slot = spapr_find_cpu_slot(MACHINE(hotplug_dev), cc->core_id,
+                                    &index);
+    if (!core_slot) {
          error_setg(errp, "Unable to find CPU core with core-id: %d",
                     cc->core_id);
          return -1;
@@ -3722,6 +3729,36 @@ static int spapr_core_unplug_possible(HotplugHandler *hotplug_dev, CPUCore *cc,
          return -1;
      }
 
+    /* Allow for any non-boot CPU core to be unplugged if already offline */
+    core = SPAPR_CPU_CORE(core_slot->cpu);
+    cs = CPU(core->threads[0]);
+    if (cs->halted) {
+        return 0;
+    }
+
+    /*
+     * Do not allow core unplug if it's the last core online.
+     */
+    cpu = POWERPC_CPU(cs);
+    CPU_FOREACH(cs) {
+        PowerPCCPU *c = POWERPC_CPU(cs);
+
+        if (c == cpu) {
+            continue;
+        }
+
+        if (!cs->halted) {
+            last_cpu_online = false;
+            break;
+        }
+    }
+
+    if (last_cpu_online) {
+        error_setg(errp, "Unable to unplug CPU core with core-id %d: it is "
+                   "the only CPU core online in the guest", cc->core_id);
+        return -1;
+    }
+
      return 0;
  }
 
#17 lore raw

Re: [PATCH v1 7/7] spapr.c: consider CPU core online state before allowing unplug

Quoted 41 lines · Daniel Henrique Barboza
Quoted 5 lines · Greg Kurz
Quoted 2 lines · On Thu, 14 Jan 2021 15:06:28 -0300 Daniel Henrique Barboza…

The only restriction we have when unplugging CPUs is to forbid unplug of the boot cpu core. spapr_core_unplug_possible() does not contemplate the

I'll look into it.

Quoted 15 lines

I can't remember why this restriction was introduced in the first place... This should be investigated and documented if the limitation still stands.

Quoted 4 lines

possibility of some cores being offlined by the guest, meaning that we're rolling the dice regarding on whether we're unplugging the last online CPU core the guest has.

Trying to unplug the last CPU is obviously something that deserves special care. LoPAPR is quite explicit on the outcome : this should terminate the partition.

13.7.4.1.1. Isolation of CPUs

Quoted 7 lines

The isolation of a CPU, in all cases, is preceded by the stop-self RTAS function for all processor threads, and the OS insures that all the CPU’s threads are in the RTAS stopped state prior to isolating the CPU. Isolation of a processor that is not stopped produces unpredictable results. The stopping of the last processor thread of a LPAR partition effectively kills the partition, and at that point, ownership of all partition resources reverts to the platform firmware.

I was just investigating a reason why we should check for all thread states before unplugging the core, like David suggested in his reply. rtas_stop_self() was setting 'cs->halted = 1' without a thread activity check like ibm,suspend-me() does and I was wondering why. This text you sent explains it, quoting:

Quoted 7 lines

"> The isolation of a CPU, in all cases, is preceded by the stop-self
RTAS function for all processor threads, and the OS insures that all the CPU’s threads are in the RTAS stopped state prior to isolating the CPU."

This seems to be corroborated by arch/powerpc/platform/pseries/hotplug-cpu.c:

Um... this seems like you're overcomplicating this. The crucial point here is that 'start-cpu' and 'stop-self' operate on individual threads, where as dynamic reconfiguration hotplug and unplug works on whole cores.

Quoted 20 lines

static void pseries_cpu_offline_self(void)

{
	unsigned int hwcpu = hard_smp_processor_id();

	local_irq_disable();
	idle_task_exit();
	if (xive_enabled())
		xive_teardown_cpu();
	else
		xics_teardown_cpu();

	unregister_slb_shadow(hwcpu);
	rtas_stop_self();

	/* Should never get here... */
	BUG();
	for(;;);
}
Quoted 4 lines

IIUC this means that we can rely on cs->halted = 1 since it's coming right after a rtas_stop_self() call. This is still a bit confusing though and I wouldn't mind standardizing the 'CPU core is offline' condition with what ibm,suspend-me is already doing.

At the moment we have no tracking of whether a core is online. We kinda-sorta track whether a *thread* is online through stop-self / start-cpu.

Quoted 1 line

David, what do you think?

I think we can rely on cs->halted = 1 when the thread is offline. What I'm much less certain about is whether we can count on the thread being offline when cs->halted = 1.

Quoted 29 lines
Quoted 10 lines

R1-13.7.4.1.1-1. For the LRDR option: Prior to issuing the RTAS set-indicator specifying isolate isolation-state of a CPU DR connector type, all the CPU threads must be in the RTAS stopped state.

R1-13.7.4.1.1-2. For the LRDR option: Stopping of the last processor thread of a LPAR partition with the stop-self RTAS function, must kill the partition, with ownership of all partition resources reverting to the platform firmware.

Quoted 6 lines

This is clearly not how things work today : linux doesn't call "stop-self" on the last vCPU and even if it did, QEMU doesn't terminate the VM.

If there's a valid reason to not implement this PAPR behavior, I'd like it to be documented.

I can only speculate. This would create a unorthodox way of shutting down the guest, when the user can just shutdown the whole thing gracefully.

Unless we're considering borderline cases, like the security risk mentioned in the kernel docs (Documentation/core-api/cpu_hotplug.rst):

"Such advances require CPUs available to a kernel to be removed either for provisioning reasons, or for RAS purposes to keep an offending CPU off system execution path. Hence the need for CPU hotplug support in the Linux kernel."

Quoted 4 lines

In this extreme scenario I can see a reason to kill the partition/guest by offlining the last online CPU - if it's compromising the host we'd rather terminate immediately instead of waiting for graceful shutdown.

I'm a bit confused by this comment. You seem to be conflating online/offline operations (start-cpu/stop-self) with hot plug/unplug operations - they're obviously related, but they're not the same thing.

Quoted 30 lines
Quoted 16 lines
Quoted 6 lines

If we hit the jackpot, we're going to detach the core DRC and pulse the hotplug IRQ, but the guest OS will refuse to release the CPU. Our spapr_core_unplug() DRC release callback will never be called and the CPU core object will keep existing in QEMU. No error message will be sent to the user, but the CPU core wasn't unplugged from the guest.

Quoted 9 lines

If the guest OS onlines the CPU core again we won't be able to hotunplug it either. 'dmesg' inside the guest will report a failed attempt to offline an unknown CPU:

[  923.003994] pseries-hotplug-cpu: Failed to offline CPU <NULL>, rc: -16

This is the result of stopping the DRC state transition in the middle in the first failed attempt.

Quoted 5 lines

Yes, at this point only a machine reset can fix things up.

Given this is linux's choice not to call "stop-self" as it should do, I'm not super fan of hardcoding this logic in QEMU, unless there are really good reasons to do so.

I understand where are you coming from and I sympathize. Not sure about how users would feel about that though. They expect a somewhat compatible behavior of multi-arch features like hotplug/hotunplug, and x86 will neither hotunplug nor offline the last CPU as well.

There is a very high chance that, even if we pull this design off, I'll need to go to Libvirt and disable it because we broke compatibility with how vcpu unplug operated earlier.

Signature · 4 lines
David Gibson			| I'll have my music baroque, and my code
david AT gibson.dropbear.id.au	| minimalist, thank you.  NOT _the_ _other_
				| _way_ _around_!
http://www.ozlabs.org/~dgibson
#18 lore raw

Re: [PATCH v1 7/7] spapr.c: consider CPU core online state before allowing unplug

Quoted 84 lines · David Gibson
Quoted 41 lines · Daniel Henrique Barboza
Quoted 5 lines · Greg Kurz
Quoted 2 lines · On Thu, 14 Jan 2021 15:06:28 -0300 Daniel Henrique Barboza…

The only restriction we have when unplugging CPUs is to forbid unplug of the boot cpu core. spapr_core_unplug_possible() does not contemplate the

I'll look into it.

Quoted 15 lines

I can't remember why this restriction was introduced in the first place... This should be investigated and documented if the limitation still stands.

Quoted 4 lines

possibility of some cores being offlined by the guest, meaning that we're rolling the dice regarding on whether we're unplugging the last online CPU core the guest has.

Trying to unplug the last CPU is obviously something that deserves special care. LoPAPR is quite explicit on the outcome : this should terminate the partition.

13.7.4.1.1. Isolation of CPUs

Quoted 7 lines

The isolation of a CPU, in all cases, is preceded by the stop-self RTAS function for all processor threads, and the OS insures that all the CPU’s threads are in the RTAS stopped state prior to isolating the CPU. Isolation of a processor that is not stopped produces unpredictable results. The stopping of the last processor thread of a LPAR partition effectively kills the partition, and at that point, ownership of all partition resources reverts to the platform firmware.

I was just investigating a reason why we should check for all thread states before unplugging the core, like David suggested in his reply. rtas_stop_self() was setting 'cs->halted = 1' without a thread activity check like ibm,suspend-me() does and I was wondering why. This text you sent explains it, quoting:

Quoted 7 lines

"> The isolation of a CPU, in all cases, is preceded by the stop-self
RTAS function for all processor threads, and the OS insures that all the CPU’s threads are in the RTAS stopped state prior to isolating the CPU."

This seems to be corroborated by arch/powerpc/platform/pseries/hotplug-cpu.c:

Um... this seems like you're overcomplicating this. The crucial point here is that 'start-cpu' and 'stop-self' operate on individual threads, where as dynamic reconfiguration hotplug and unplug works on whole cores.

Quoted 20 lines

static void pseries_cpu_offline_self(void)

{
	unsigned int hwcpu = hard_smp_processor_id();

	local_irq_disable();
	idle_task_exit();
	if (xive_enabled())
		xive_teardown_cpu();
	else
		xics_teardown_cpu();

	unregister_slb_shadow(hwcpu);
	rtas_stop_self();

	/* Should never get here... */
	BUG();
	for(;;);
}
Quoted 4 lines

IIUC this means that we can rely on cs->halted = 1 since it's coming right after a rtas_stop_self() call. This is still a bit confusing though and I wouldn't mind standardizing the 'CPU core is offline' condition with what ibm,suspend-me is already doing.

At the moment we have no tracking of whether a core is online. We kinda-sorta track whether a *thread* is online through stop-self / start-cpu.

Quoted 5 lines
Quoted 1 line

David, what do you think?

I think we can rely on cs->halted = 1 when the thread is offline. What I'm much less certain about is whether we can count on the thread being offline when cs->halted = 1.

I guess we should just stick with your first suggestion then. I'll check for both cs->halted and msr to assert whether a whole core if offline, like ibm,suspend-me is doing:

        if (!cs->halted || (e->msr & (1ULL << MSR_EE))) {
            rtas_st(rets, 0, H_MULTI_THREADS_ACTIVE);
            return;
        }

Another question not necessarily related to this fix: we do a similar check in the start of do_client_architecture_support(), returning the same H_MULTI_THREADS_ACTIVE error, but we're not checking e->msr:

    /* CAS is supposed to be called early when only the boot vCPU is active. */
    CPU_FOREACH(cs) {
        if (cs == CPU(cpu)) {
            continue;
        }
        if (!cs->halted) {
            warn_report("guest has multiple active vCPUs at CAS, which is not allowed");
            return H_MULTI_THREADS_ACTIVE;
        }
    }

Should we bother changing this logic to check for e->msr as well? If there is no possible harm done I'd rather have the same check returning H_MULTI_THREADS_ACTIVE in both places. If there is a special condition in early boot where this check in do_client_architecture_support() is enough, I would like to put a comment in there to make it clear why.

Thanks,

DHB

Quoted 71 lines
Quoted 30 lines
Quoted 11 lines

R1-13.7.4.1.1-1. For the LRDR option: Prior to issuing the RTAS set-indicator specifying isolate isolation-state of a CPU DR connector type, all the CPU threads must be in the RTAS stopped state.

R1-13.7.4.1.1-2. For the LRDR option: Stopping of the last processor thread of a LPAR partition with the stop-self RTAS function, must kill the partition, with ownership of all partition resources reverting to the platform firmware.

Quoted 6 lines

This is clearly not how things work today : linux doesn't call "stop-self" on the last vCPU and even if it did, QEMU doesn't terminate the VM.

If there's a valid reason to not implement this PAPR behavior, I'd like it to be documented.

I can only speculate. This would create a unorthodox way of shutting down the guest, when the user can just shutdown the whole thing gracefully.

Unless we're considering borderline cases, like the security risk mentioned in the kernel docs (Documentation/core-api/cpu_hotplug.rst):

"Such advances require CPUs available to a kernel to be removed either for provisioning reasons, or for RAS purposes to keep an offending CPU off system execution path. Hence the need for CPU hotplug support in the Linux kernel."

Quoted 4 lines

In this extreme scenario I can see a reason to kill the partition/guest by offlining the last online CPU - if it's compromising the host we'd rather terminate immediately instead of waiting for graceful shutdown.

I'm a bit confused by this comment. You seem to be conflating online/offline operations (start-cpu/stop-self) with hot plug/unplug operations - they're obviously related, but they're not the same thing.

Quoted 30 lines
Quoted 16 lines
Quoted 6 lines

If we hit the jackpot, we're going to detach the core DRC and pulse the hotplug IRQ, but the guest OS will refuse to release the CPU. Our spapr_core_unplug() DRC release callback will never be called and the CPU core object will keep existing in QEMU. No error message will be sent to the user, but the CPU core wasn't unplugged from the guest.

Quoted 9 lines

If the guest OS onlines the CPU core again we won't be able to hotunplug it either. 'dmesg' inside the guest will report a failed attempt to offline an unknown CPU:

[  923.003994] pseries-hotplug-cpu: Failed to offline CPU <NULL>, rc: -16

This is the result of stopping the DRC state transition in the middle in the first failed attempt.

Quoted 5 lines

Yes, at this point only a machine reset can fix things up.

Given this is linux's choice not to call "stop-self" as it should do, I'm not super fan of hardcoding this logic in QEMU, unless there are really good reasons to do so.

I understand where are you coming from and I sympathize. Not sure about how users would feel about that though. They expect a somewhat compatible behavior of multi-arch features like hotplug/hotunplug, and x86 will neither hotunplug nor offline the last CPU as well.

There is a very high chance that, even if we pull this design off, I'll need to go to Libvirt and disable it because we broke compatibility with how vcpu unplug operated earlier.

#19 lore raw

Re: [PATCH v1 7/7] spapr.c: consider CPU core online state before allowing unplug

In reply to Greg Kurz
Quoted 8 lines · Greg Kurz
Quoted 2 lines · On Thu, 14 Jan 2021 15:06:28 -0300 Daniel Henrique Barboza…

The only restriction we have when unplugging CPUs is to forbid unplug of the boot cpu core. spapr_core_unplug_possible() does not contemplate the

I can't remember why this restriction was introduced in the first place... This should be investigated and documented if the limitation still stands.

I looked it up and found out that restriction was added by this commit:

commit 62be8b044adf47327ebefdefb25f28a40316ebd0

Author: Bharata B Rao <[email protected]>
Date:   Wed Jul 27 10:44:42 2016 +0530

spapr: Prevent boot CPU core removal

Boot CPU is assumed to be always present in QEMU code. So until that assumptions are gone, deny removal request. In another words, QEMU won't support boot CPU core hot-unplug.

I don't think it necessarily has to do with pSeries code though. I was unable to offline the CPU0 of my x86 notebook:

# lscpu | grep -i 'on-line'
On-line CPU(s) list:             0-7
# echo 0 > /sys/devices/system/cpu/cpu0/online
bash: /sys/devices/system/cpu/cpu0/online: Permission denied
#
# echo 0 > /sys/devices/system/cpu/cpu1/online
#
# lscpu | grep -i 'on-line'
On-line CPU(s) list:             0,2-7
# echo 0 > /sys/devices/system/cpu/cpu0/online
bash: /sys/devices/system/cpu/cpu0/online: Permission denied
#

The pseries kernel does not have this restriction (offlining CPU0).

Maybe we're limiting CPU0 unplug in pseries because it would break common QEMU code that has this restriction due to x86/ACPI mechanics because, apparently, x86 can't hotunplug CPU0.

If a good x86 soul reads this and confirm/deny my assumption I appreciate :)

Thanks,

DHB

Quoted 132 lines
Quoted 5 lines

possibility of some cores being offlined by the guest, meaning that we're rolling the dice regarding on whether we're unplugging the last online CPU core the guest has.

Trying to unplug the last CPU is obviously something that deserves special care. LoPAPR is quite explicit on the outcome : this should terminate the partition.

13.7.4.1.1. Isolation of CPUs

The isolation of a CPU, in all cases, is preceded by the stop-self RTAS function for all processor threads, and the OS insures that all the CPU’s threads are in the RTAS stopped state prior to isolating the CPU. Isolation of a processor that is not stopped produces unpredictable results. The stopping of the last processor thread of a LPAR partition effectively kills the partition, and at that point, ownership of all partition resources reverts to the platform firmware.

R1-13.7.4.1.1-1. For the LRDR option: Prior to issuing the RTAS set-indicator specifying isolate isolation-state of a CPU DR connector type, all the CPU threads must be in the RTAS stopped state.

R1-13.7.4.1.1-2. For the LRDR option: Stopping of the last processor thread of a LPAR partition with the stop-self RTAS function, must kill the partition, with ownership of all partition resources reverting to the platform firmware.

This is clearly not how things work today : linux doesn't call "stop-self" on the last vCPU and even if it did, QEMU doesn't terminate the VM.

If there's a valid reason to not implement this PAPR behavior, I'd like it to be documented.

Quoted 6 lines

If we hit the jackpot, we're going to detach the core DRC and pulse the hotplug IRQ, but the guest OS will refuse to release the CPU. Our spapr_core_unplug() DRC release callback will never be called and the CPU core object will keep existing in QEMU. No error message will be sent to the user, but the CPU core wasn't unplugged from the guest.

Quoted 9 lines

If the guest OS onlines the CPU core again we won't be able to hotunplug it either. 'dmesg' inside the guest will report a failed attempt to offline an unknown CPU:

[  923.003994] pseries-hotplug-cpu: Failed to offline CPU <NULL>, rc: -16

This is the result of stopping the DRC state transition in the middle in the first failed attempt.

Yes, at this point only a machine reset can fix things up.

Given this is linux's choice not to call "stop-self" as it should do, I'm not super fan of hardcoding this logic in QEMU, unless there are really good reasons to do so.

Quoted 72 lines

We can avoid this, and potentially other bad things from happening, if we avoid to attempt the unplug altogether in this scenario. Let's check for the online/offline state of the CPU cores in the guest before allowing the hotunplug, and forbid removing a CPU core if it's the last one online in the guest.

---
  hw/ppc/spapr.c | 39 ++++++++++++++++++++++++++++++++++++++-
  1 file changed, 38 insertions(+), 1 deletion(-)
hw/ppc/spapr.c +38 -1
index a2f01c21aa..d269dcd102 100644
@@ -3709,9 +3709,16 @@ static void spapr_core_unplug(HotplugHandler *hotplug_dev, DeviceState *dev)
  static int spapr_core_unplug_possible(HotplugHandler *hotplug_dev, CPUCore *cc,
                                        Error **errp)
  {
+    CPUArchId *core_slot;
+    SpaprCpuCore *core;
+    PowerPCCPU *cpu;
+    CPUState *cs;
+    bool last_cpu_online = true;
      int index;
 
-    if (!spapr_find_cpu_slot(MACHINE(hotplug_dev), cc->core_id, &index)) {
+    core_slot = spapr_find_cpu_slot(MACHINE(hotplug_dev), cc->core_id,
+                                    &index);
+    if (!core_slot) {
          error_setg(errp, "Unable to find CPU core with core-id: %d",
                     cc->core_id);
          return -1;
@@ -3722,6 +3729,36 @@ static int spapr_core_unplug_possible(HotplugHandler *hotplug_dev, CPUCore *cc,
          return -1;
      }
 
+    /* Allow for any non-boot CPU core to be unplugged if already offline */
+    core = SPAPR_CPU_CORE(core_slot->cpu);
+    cs = CPU(core->threads[0]);
+    if (cs->halted) {
+        return 0;
+    }
+
+    /*
+     * Do not allow core unplug if it's the last core online.
+     */
+    cpu = POWERPC_CPU(cs);
+    CPU_FOREACH(cs) {
+        PowerPCCPU *c = POWERPC_CPU(cs);
+
+        if (c == cpu) {
+            continue;
+        }
+
+        if (!cs->halted) {
+            last_cpu_online = false;
+            break;
+        }
+    }
+
+    if (last_cpu_online) {
+        error_setg(errp, "Unable to unplug CPU core with core-id %d: it is "
+                   "the only CPU core online in the guest", cc->core_id);
+        return -1;
+    }
+
      return 0;
  }
 
#20 lore raw

Re: [PATCH v1 7/7] spapr.c: consider CPU core online state before allowing unplug

In reply to Greg Kurz
Quoted 8 lines · Greg Kurz
Quoted 2 lines · On Thu, 14 Jan 2021 15:06:28 -0300 Daniel Henrique Barboza…

The only restriction we have when unplugging CPUs is to forbid unplug of the boot cpu core. spapr_core_unplug_possible() does not contemplate the

I can't remember why this restriction was introduced in the first place... This should be investigated and documented if the limitation still stands.

I looked it up and found out that restriction was added by this commit:

commit 62be8b044adf47327ebefdefb25f28a40316ebd0

Author: Bharata B Rao <[email protected]>
Date:   Wed Jul 27 10:44:42 2016 +0530

spapr: Prevent boot CPU core removal

Boot CPU is assumed to be always present in QEMU code. So until that assumptions are gone, deny removal request. In another words, QEMU won't support boot CPU core hot-unplug.

I don't think it necessarily has to do with pSeries code though. I was unable to offline the CPU0 of my x86 notebook:

# lscpu | grep -i 'on-line'
On-line CPU(s) list:             0-7
# echo 0 > /sys/devices/system/cpu/cpu0/online
bash: /sys/devices/system/cpu/cpu0/online: Permission denied
#
# echo 0 > /sys/devices/system/cpu/cpu1/online
#
# lscpu | grep -i 'on-line'
On-line CPU(s) list:             0,2-7
# echo 0 > /sys/devices/system/cpu/cpu0/online
bash: /sys/devices/system/cpu/cpu0/online: Permission denied
#

The pseries kernel does not have this restriction (offlining CPU0).

Maybe we're limiting CPU0 unplug in pseries because it would break common QEMU code that has this restriction due to x86/ACPI mechanics because, apparently, x86 can't hotunplug CPU0.

If a good x86 soul reads this and confirm/deny my assumption I appreciate :)

Thanks,

DHB

Quoted 132 lines
Quoted 5 lines

possibility of some cores being offlined by the guest, meaning that we're rolling the dice regarding on whether we're unplugging the last online CPU core the guest has.

Trying to unplug the last CPU is obviously something that deserves special care. LoPAPR is quite explicit on the outcome : this should terminate the partition.

13.7.4.1.1. Isolation of CPUs

The isolation of a CPU, in all cases, is preceded by the stop-self RTAS function for all processor threads, and the OS insures that all the CPU’s threads are in the RTAS stopped state prior to isolating the CPU. Isolation of a processor that is not stopped produces unpredictable results. The stopping of the last processor thread of a LPAR partition effectively kills the partition, and at that point, ownership of all partition resources reverts to the platform firmware.

R1-13.7.4.1.1-1. For the LRDR option: Prior to issuing the RTAS set-indicator specifying isolate isolation-state of a CPU DR connector type, all the CPU threads must be in the RTAS stopped state.

R1-13.7.4.1.1-2. For the LRDR option: Stopping of the last processor thread of a LPAR partition with the stop-self RTAS function, must kill the partition, with ownership of all partition resources reverting to the platform firmware.

This is clearly not how things work today : linux doesn't call "stop-self" on the last vCPU and even if it did, QEMU doesn't terminate the VM.

If there's a valid reason to not implement this PAPR behavior, I'd like it to be documented.

Quoted 6 lines

If we hit the jackpot, we're going to detach the core DRC and pulse the hotplug IRQ, but the guest OS will refuse to release the CPU. Our spapr_core_unplug() DRC release callback will never be called and the CPU core object will keep existing in QEMU. No error message will be sent to the user, but the CPU core wasn't unplugged from the guest.

Quoted 9 lines

If the guest OS onlines the CPU core again we won't be able to hotunplug it either. 'dmesg' inside the guest will report a failed attempt to offline an unknown CPU:

[  923.003994] pseries-hotplug-cpu: Failed to offline CPU <NULL>, rc: -16

This is the result of stopping the DRC state transition in the middle in the first failed attempt.

Yes, at this point only a machine reset can fix things up.

Given this is linux's choice not to call "stop-self" as it should do, I'm not super fan of hardcoding this logic in QEMU, unless there are really good reasons to do so.

Quoted 72 lines

We can avoid this, and potentially other bad things from happening, if we avoid to attempt the unplug altogether in this scenario. Let's check for the online/offline state of the CPU cores in the guest before allowing the hotunplug, and forbid removing a CPU core if it's the last one online in the guest.

---
  hw/ppc/spapr.c | 39 ++++++++++++++++++++++++++++++++++++++-
  1 file changed, 38 insertions(+), 1 deletion(-)
hw/ppc/spapr.c +38 -1
index a2f01c21aa..d269dcd102 100644
@@ -3709,9 +3709,16 @@ static void spapr_core_unplug(HotplugHandler *hotplug_dev, DeviceState *dev)
  static int spapr_core_unplug_possible(HotplugHandler *hotplug_dev, CPUCore *cc,
                                        Error **errp)
  {
+    CPUArchId *core_slot;
+    SpaprCpuCore *core;
+    PowerPCCPU *cpu;
+    CPUState *cs;
+    bool last_cpu_online = true;
      int index;
 
-    if (!spapr_find_cpu_slot(MACHINE(hotplug_dev), cc->core_id, &index)) {
+    core_slot = spapr_find_cpu_slot(MACHINE(hotplug_dev), cc->core_id,
+                                    &index);
+    if (!core_slot) {
          error_setg(errp, "Unable to find CPU core with core-id: %d",
                     cc->core_id);
          return -1;
@@ -3722,6 +3729,36 @@ static int spapr_core_unplug_possible(HotplugHandler *hotplug_dev, CPUCore *cc,
          return -1;
      }
 
+    /* Allow for any non-boot CPU core to be unplugged if already offline */
+    core = SPAPR_CPU_CORE(core_slot->cpu);
+    cs = CPU(core->threads[0]);
+    if (cs->halted) {
+        return 0;
+    }
+
+    /*
+     * Do not allow core unplug if it's the last core online.
+     */
+    cpu = POWERPC_CPU(cs);
+    CPU_FOREACH(cs) {
+        PowerPCCPU *c = POWERPC_CPU(cs);
+
+        if (c == cpu) {
+            continue;
+        }
+
+        if (!cs->halted) {
+            last_cpu_online = false;
+            break;
+        }
+    }
+
+    if (last_cpu_online) {
+        error_setg(errp, "Unable to unplug CPU core with core-id %d: it is "
+                   "the only CPU core online in the guest", cc->core_id);
+        return -1;
+    }
+
      return 0;
  }
 
#21 lore raw

Re: [PATCH v1 7/7] spapr.c: consider CPU core online state before allowing unplug

In reply to Greg Kurz
Quoted 38 lines · Greg Kurz
Quoted 2 lines · On Thu, 14 Jan 2021 15:06:28 -0300 Daniel Henrique Barboza…

The only restriction we have when unplugging CPUs is to forbid unplug of the boot cpu core. spapr_core_unplug_possible() does not contemplate the

I can't remember why this restriction was introduced in the first place... This should be investigated and documented if the limitation still stands.

Quoted 4 lines

possibility of some cores being offlined by the guest, meaning that we're rolling the dice regarding on whether we're unplugging the last online CPU core the guest has.

Trying to unplug the last CPU is obviously something that deserves special care. LoPAPR is quite explicit on the outcome : this should terminate the partition.

13.7.4.1.1. Isolation of CPUs

The isolation of a CPU, in all cases, is preceded by the stop-self RTAS function for all processor threads, and the OS insures that all the CPU’s threads are in the RTAS stopped state prior to isolating the CPU. Isolation of a processor that is not stopped produces unpredictable results. The stopping of the last processor thread of a LPAR partition effectively kills the partition, and at that point, ownership of all partition resources reverts to the platform firmware.

R1-13.7.4.1.1-1. For the LRDR option: Prior to issuing the RTAS set-indicator specifying isolate isolation-state of a CPU DR connector type, all the CPU threads must be in the RTAS stopped state.

R1-13.7.4.1.1-2. For the LRDR option: Stopping of the last processor thread of a LPAR partition with the stop-self RTAS function, must kill the partition, with ownership of all partition resources reverting to the platform firmware.

Quoted 6 lines

This is clearly not how things work today : linux doesn't call "stop-self" on the last vCPU and even if it did, QEMU doesn't terminate the VM.

If there's a valid reason to not implement this PAPR behavior, I'd like it to be documented.

So, we should implement this in QEMU - if you stop-self the last thread, it should be the equivalent of a power off. Linux not ever doing that probably makes sense - it wants you to encourage you to shut down properly for data safety.

Quoted 16 lines
Quoted 6 lines

If we hit the jackpot, we're going to detach the core DRC and pulse the hotplug IRQ, but the guest OS will refuse to release the CPU. Our spapr_core_unplug() DRC release callback will never be called and the CPU core object will keep existing in QEMU. No error message will be sent to the user, but the CPU core wasn't unplugged from the guest.

Quoted 9 lines

If the guest OS onlines the CPU core again we won't be able to hotunplug it either. 'dmesg' inside the guest will report a failed attempt to offline an unknown CPU:

[  923.003994] pseries-hotplug-cpu: Failed to offline CPU <NULL>, rc: -16

This is the result of stopping the DRC state transition in the middle in the first failed attempt.

Quoted 5 lines

Yes, at this point only a machine reset can fix things up.

Given this is linux's choice not to call "stop-self" as it should do, I'm not super fan of hardcoding this logic in QEMU, unless there are really good reasons to do so.

Uh.. sorry I don't follow how linux is doing something wrong here.

Quoted 73 lines
Quoted 72 lines

We can avoid this, and potentially other bad things from happening, if we avoid to attempt the unplug altogether in this scenario. Let's check for the online/offline state of the CPU cores in the guest before allowing the hotunplug, and forbid removing a CPU core if it's the last one online in the guest.

---
 hw/ppc/spapr.c | 39 ++++++++++++++++++++++++++++++++++++++-
 1 file changed, 38 insertions(+), 1 deletion(-)
hw/ppc/spapr.c +38 -1
index a2f01c21aa..d269dcd102 100644
@@ -3709,9 +3709,16 @@ static void spapr_core_unplug(HotplugHandler *hotplug_dev, DeviceState *dev)
 static int spapr_core_unplug_possible(HotplugHandler *hotplug_dev, CPUCore *cc,
                                       Error **errp)
 {
+    CPUArchId *core_slot;
+    SpaprCpuCore *core;
+    PowerPCCPU *cpu;
+    CPUState *cs;
+    bool last_cpu_online = true;
     int index;
 
-    if (!spapr_find_cpu_slot(MACHINE(hotplug_dev), cc->core_id, &index)) {
+    core_slot = spapr_find_cpu_slot(MACHINE(hotplug_dev), cc->core_id,
+                                    &index);
+    if (!core_slot) {
         error_setg(errp, "Unable to find CPU core with core-id: %d",
                    cc->core_id);
         return -1;
@@ -3722,6 +3729,36 @@ static int spapr_core_unplug_possible(HotplugHandler *hotplug_dev, CPUCore *cc,
         return -1;
     }
 
+    /* Allow for any non-boot CPU core to be unplugged if already offline */
+    core = SPAPR_CPU_CORE(core_slot->cpu);
+    cs = CPU(core->threads[0]);
+    if (cs->halted) {
+        return 0;
+    }
+
+    /*
+     * Do not allow core unplug if it's the last core online.
+     */
+    cpu = POWERPC_CPU(cs);
+    CPU_FOREACH(cs) {
+        PowerPCCPU *c = POWERPC_CPU(cs);
+
+        if (c == cpu) {
+            continue;
+        }
+
+        if (!cs->halted) {
+            last_cpu_online = false;
+            break;
+        }
+    }
+
+    if (last_cpu_online) {
+        error_setg(errp, "Unable to unplug CPU core with core-id %d: it is "
+                   "the only CPU core online in the guest", cc->core_id);
+        return -1;
+    }
+
     return 0;
 }
 
Signature · 4 lines
David Gibson			| I'll have my music baroque, and my code
david AT gibson.dropbear.id.au	| minimalist, thank you.  NOT _the_ _other_
				| _way_ _around_!
http://www.ozlabs.org/~dgibson
#22 lore raw

Re: [PATCH v1 7/7] spapr.c: consider CPU core online state before allowing unplug

Quoted 43 lines · On Mon, 18 Jan 2021 12:12:03 +1100 David Gibson <[email protected]>…
Quoted 41 lines · Greg Kurz
Quoted 2 lines · On Thu, 14 Jan 2021 15:06:28 -0300 Daniel Henrique Barboza…

The only restriction we have when unplugging CPUs is to forbid unplug of the boot cpu core. spapr_core_unplug_possible() does not contemplate the

I can't remember why this restriction was introduced in the first place... This should be investigated and documented if the limitation still stands.

Quoted 4 lines

possibility of some cores being offlined by the guest, meaning that we're rolling the dice regarding on whether we're unplugging the last online CPU core the guest has.

Trying to unplug the last CPU is obviously something that deserves special care. LoPAPR is quite explicit on the outcome : this should terminate the partition.

13.7.4.1.1. Isolation of CPUs

The isolation of a CPU, in all cases, is preceded by the stop-self RTAS function for all processor threads, and the OS insures that all the CPU’s threads are in the RTAS stopped state prior to isolating the CPU. Isolation of a processor that is not stopped produces unpredictable results. The stopping of the last processor thread of a LPAR partition effectively kills the partition, and at that point, ownership of all partition resources reverts to the platform firmware.

R1-13.7.4.1.1-1. For the LRDR option: Prior to issuing the RTAS set-indicator specifying isolate isolation-state of a CPU DR connector type, all the CPU threads must be in the RTAS stopped state.

R1-13.7.4.1.1-2. For the LRDR option: Stopping of the last processor thread of a LPAR partition with the stop-self RTAS function, must kill the partition, with ownership of all partition resources reverting to the platform firmware.

This is clearly not how things work today : linux doesn't call "stop-self" on the last vCPU and even if it did, QEMU doesn't terminate the VM.

Quoted 8 lines
Quoted 2 lines

If there's a valid reason to not implement this PAPR behavior, I'd like it to be documented.

So, we should implement this in QEMU - if you stop-self the last thread, it should be the equivalent of a power off. Linux not ever doing that probably makes sense - it wants you to encourage you to shut down properly for data safety.

Yes I agree it's fine if linux enforces some safeguard to prevent a brutal shutdown when writing 0 to /sys/devices/system/cpu/cpu?/online within the guest. But in this case, off-lining is part of the usual CPU unplug sequence, which was requested by the host : I don't think the safeguard is relevant in this case. This PAPR _feature_ is just another way of uncleanly shutting down the guest.

Quoted 16 lines
Quoted 16 lines
Quoted 15 lines

If we hit the jackpot, we're going to detach the core DRC and pulse the hotplug IRQ, but the guest OS will refuse to release the CPU. Our spapr_core_unplug() DRC release callback will never be called and the CPU core object will keep existing in QEMU. No error message will be sent to the user, but the CPU core wasn't unplugged from the guest.

If the guest OS onlines the CPU core again we won't be able to hotunplug it either. 'dmesg' inside the guest will report a failed attempt to offline an unknown CPU:

[  923.003994] pseries-hotplug-cpu: Failed to offline CPU <NULL>, rc: -16

This is the result of stopping the DRC state transition in the middle in the first failed attempt.

Quoted 8 lines
Quoted 5 lines

Yes, at this point only a machine reset can fix things up.

Given this is linux's choice not to call "stop-self" as it should do, I'm not super fan of hardcoding this logic in QEMU, unless there are really good reasons to do so.

Uh.. sorry I don't follow how linux is doing something wrong here.

Well... it doesn't finalize the hot-unplug sequence, and we have no way to cope with that except a machine reset. So I would nearly say this is working as expected : CPU hot unplug was requested and we wait for the guest to release the CPU. Linux not wanting to release the CPU until next reboot for some reason isn't really our concern.

Quoted 6 lines
Quoted 6 lines
Quoted 6 lines

We can avoid this, and potentially other bad things from happening, if we avoid to attempt the unplug altogether in this scenario. Let's check for the online/offline state of the CPU cores in the guest before allowing the hotunplug, and forbid removing a CPU core if it's the last one online in the guest.

An unplug request can be accepted but its handling can still race with some manual off-lining in the guest, which would leave us in the very same situation. So I don't think this patch fixes anything actually (TOCTOU).

I tend to think that mixing manual CPU off-lining and CPU hot-unplug is probably not the best thing to do in the first place, unless one really knows what they're doing. Maybe we should rather document the caveats instead of adding workarounds for what remains a corner case ?

Quoted 68 lines
Quoted 67 lines
Quoted 66 lines
---
 hw/ppc/spapr.c | 39 ++++++++++++++++++++++++++++++++++++++-
 1 file changed, 38 insertions(+), 1 deletion(-)
hw/ppc/spapr.c +38 -1
index a2f01c21aa..d269dcd102 100644
@@ -3709,9 +3709,16 @@ static void spapr_core_unplug(HotplugHandler *hotplug_dev, DeviceState *dev)
 static int spapr_core_unplug_possible(HotplugHandler *hotplug_dev, CPUCore *cc,
                                       Error **errp)
 {
+    CPUArchId *core_slot;
+    SpaprCpuCore *core;
+    PowerPCCPU *cpu;
+    CPUState *cs;
+    bool last_cpu_online = true;
     int index;
 
-    if (!spapr_find_cpu_slot(MACHINE(hotplug_dev), cc->core_id, &index)) {
+    core_slot = spapr_find_cpu_slot(MACHINE(hotplug_dev), cc->core_id,
+                                    &index);
+    if (!core_slot) {
         error_setg(errp, "Unable to find CPU core with core-id: %d",
                    cc->core_id);
         return -1;
@@ -3722,6 +3729,36 @@ static int spapr_core_unplug_possible(HotplugHandler *hotplug_dev, CPUCore *cc,
         return -1;
     }
 
+    /* Allow for any non-boot CPU core to be unplugged if already offline */
+    core = SPAPR_CPU_CORE(core_slot->cpu);
+    cs = CPU(core->threads[0]);
+    if (cs->halted) {
+        return 0;
+    }
+
+    /*
+     * Do not allow core unplug if it's the last core online.
+     */
+    cpu = POWERPC_CPU(cs);
+    CPU_FOREACH(cs) {
+        PowerPCCPU *c = POWERPC_CPU(cs);
+
+        if (c == cpu) {
+            continue;
+        }
+
+        if (!cs->halted) {
+            last_cpu_online = false;
+            break;
+        }
+    }
+
+    if (last_cpu_online) {
+        error_setg(errp, "Unable to unplug CPU core with core-id %d: it is "
+                   "the only CPU core online in the guest", cc->core_id);
+        return -1;
+    }
+
     return 0;
 }