Implement CPU hotplug on Arm

Move IRQs from dying CPU to the online ones when a CPU is getting offlined. When onlining, rebalance all IRQs in a round-robin fashion. Guest-bound IRQs are already handled by scheduler in the process of moving vCPUs to active pCPUs, so we only need to handle IRQs used by Xen itself.

Signed-off-by Mykyta Poturai <[email protected]>

---
v8->v9:
* replace CONFIG_CPU_HOTPLUG with CONFIG_CPU_ONLINE_OFFLINE

v7->v8:
* check only existings ESPIs

v6->v7:
* replace ifdef with IS_ENABLED

v5->v6:
* don't do any balancing on boot
* only do balancing when cpu hotplug is enabled

v4->v5:
* handle CPU onlining as well
* more comments
* fix crash when ESPI is disabled
* don't assume CPU 0 is a boot CPU
* use insigned int for irq number
* remove assumption that all irqs a bound to CPU 0 by default from the commit message

v3->v4:
* patch introduced

---
 xen/arch/arm/include/asm/irq.h |  6 ++++
 xen/arch/arm/irq.c             | 60 ++++++++++++++++++++++++++++++++++
 xen/arch/arm/smpboot.c         |  7 ++++
 3 files changed, 73 insertions(+)

xen/arch/arm/include/asm/irq.h +6 -0

index 09788dbfeb..d043360135 100644
@@ -126,6 +126,12 @@ bool irq_type_set_by_domain(const struct domain *d);
 void irq_end_none(struct irq_desc *irq);
 #define irq_end_none irq_end_none
 
+#ifdef CONFIG_CPU_ONLINE_OFFLINE
+void rebalance_irqs(unsigned int from, bool up);
+#else
+static inline void rebalance_irqs(unsigned int from, bool up) {}
+#endif
+
 #endif /* _ASM_HW_IRQ_H */
 /*
  * Local variables:

xen/arch/arm/irq.c +60 -0

index 7204bc2b68..6445183f5c 100644
@@ -158,6 +158,61 @@ static int init_local_irq_data(unsigned int cpu)
     return 0;
 }
 
+#ifdef CONFIG_CPU_ONLINE_OFFLINE
+static int cpu_next;
+
+static void balance_irq(int irq, unsigned int from, bool up)
+{
+    struct irq_desc *desc = irq_to_desc(irq);
+    unsigned long flags;
+
+    ASSERT(!cpumask_empty(&cpu_online_map));
+
+    spin_lock_irqsave(&desc->lock, flags);
+    if ( likely(!desc->action) )
+        goto out;
+
+    if ( likely(test_bit(_IRQ_GUEST, &desc->status) ||
+                test_bit(_IRQ_MOVE_PENDING, &desc->status)) )
+        goto out;
+
+    /*
+     * Setting affinity to a mask of multiple CPUs causes the GIC drivers to
+     * select one CPU from that mask. If the dying CPU was included in the IRQ's
+     * affinity mask, we cannot determine exactly which CPU the interrupt is
+     * currently routed to, as GIC drivers lack a concrete get_affinity API. So
+     * to be safe we must reroute it to a new, definitely online, CPU. In the
+     * case of CPU going down, we move only the interrupt that could reside on
+     * it. Otherwise, we rearrange all interrupts in a round-robin fashion.
+     */
+    if ( !up && !cpumask_test_cpu(from, desc->affinity) )
+        goto out;
+
+    cpu_next = cpumask_cycle(cpu_next, &cpu_online_map);
+    irq_set_affinity(desc, cpumask_of(cpu_next));
+
+out:
+    spin_unlock_irqrestore(&desc->lock, flags);
+}
+
+void rebalance_irqs(unsigned int from, bool up)
+{
+    int irq;
+
+    if ( cpumask_empty(&cpu_online_map) )
+        return;
+
+    for ( irq = NR_LOCAL_IRQS; irq < NR_IRQS; irq++ )
+        balance_irq(irq, from, up);
+
+#ifdef CONFIG_GICV3_ESPI
+    for ( irq = ESPI_BASE_INTID; irq < ESPI_BASE_INTID + gic_number_espis();
+          irq++ )
+        balance_irq(irq, from, up);
+#endif
+}
+#endif /* CONFIG_CPU_ONLINE_OFFLINE */
+
 static int cpu_callback(struct notifier_block *nfb, unsigned long action,
                         void *hcpu)
 {

@@ -172,6 +227,11 @@ static int cpu_callback(struct notifier_block *nfb, unsigned long action,
             printk(XENLOG_ERR "Unable to allocate local IRQ for CPU%u\n",
                    cpu);
         break;
+    case CPU_ONLINE:
+        if ( IS_ENABLED(CONFIG_CPU_ONLINE_OFFLINE) &&
+             system_state >= SYS_STATE_active )
+            rebalance_irqs(cpu, true);
+        break;
     }
 
     return notifier_from_errno(rc);

xen/arch/arm/smpboot.c +0 -0

index 1806c47a08..239d4a7a25 100644

xen/arch/arm/smpboot.c +3 -0

+    if ( IS_ENABLED(CONFIG_CPU_ONLINE_OFFLINE) )
+        rebalance_irqs(cpu, false);
+
     /* Return to caller; eventually the IPI mechanism will unwind and the 
      * scheduler will drop to the idle loop, which will call stop_cpu(). */
 }

Move IRQs from dying CPU to the online ones when a CPU is getting offlined. When onlining, rebalance all IRQs in a round-robin fashion. Guest-bound IRQs are already handled by scheduler in the process of moving vCPUs to active pCPUs, so we only need to handle IRQs used by Xen itself.

Signed-off-by Mykyta Poturai <[email protected]>

---
v8->v9:
* replace CONFIG_CPU_HOTPLUG with CONFIG_CPU_ONLINE_OFFLINE

v7->v8:
* check only existings ESPIs

v6->v7:
* replace ifdef with IS_ENABLED

v5->v6:
* don't do any balancing on boot
* only do balancing when cpu hotplug is enabled

v4->v5:
* handle CPU onlining as well
* more comments
* fix crash when ESPI is disabled
* don't assume CPU 0 is a boot CPU
* use insigned int for irq number
* remove assumption that all irqs a bound to CPU 0 by default from the commit message

v3->v4:
* patch introduced

---
 xen/arch/arm/include/asm/irq.h |  6 ++++
 xen/arch/arm/irq.c             | 60 ++++++++++++++++++++++++++++++++++
 xen/arch/arm/smpboot.c         |  7 ++++
 3 files changed, 73 insertions(+)

xen/arch/arm/include/asm/irq.h +6 -0

index 09788dbfeb..d043360135 100644
@@ -126,6 +126,12 @@ bool irq_type_set_by_domain(const struct domain *d);
 void irq_end_none(struct irq_desc *irq);
 #define irq_end_none irq_end_none
 
+#ifdef CONFIG_CPU_ONLINE_OFFLINE
+void rebalance_irqs(unsigned int from, bool up);
+#else
+static inline void rebalance_irqs(unsigned int from, bool up) {}
+#endif
+
 #endif /* _ASM_HW_IRQ_H */
 /*
  * Local variables:

xen/arch/arm/irq.c +0 -0

index 7204bc2b68..6445183f5c 100644

xen/arch/arm/irq.c +58 -0

+
+static void balance_irq(int irq, unsigned int from, bool up)
+{
+    struct irq_desc *desc = irq_to_desc(irq);
+    unsigned long flags;
+
+    ASSERT(!cpumask_empty(&cpu_online_map));
+
+    spin_lock_irqsave(&desc->lock, flags);
+    if ( likely(!desc->action) )
+        goto out;
+
+    if ( likely(test_bit(_IRQ_GUEST, &desc->status) ||
+                test_bit(_IRQ_MOVE_PENDING, &desc->status)) )
+        goto out;
+
+    /*
+     * Setting affinity to a mask of multiple CPUs causes the GIC drivers to
+     * select one CPU from that mask. If the dying CPU was included in the IRQ's
+     * affinity mask, we cannot determine exactly which CPU the interrupt is
+     * currently routed to, as GIC drivers lack a concrete get_affinity API. So
+     * to be safe we must reroute it to a new, definitely online, CPU. In the
+     * case of CPU going down, we move only the interrupt that could reside on
+     * it. Otherwise, we rearrange all interrupts in a round-robin fashion.
+     */
+    if ( !up && !cpumask_test_cpu(from, desc->affinity) )
+        goto out;
+
+    cpu_next = cpumask_cycle(cpu_next, &cpu_online_map);
+    irq_set_affinity(desc, cpumask_of(cpu_next));
+
+out:
+    spin_unlock_irqrestore(&desc->lock, flags);
+}
+
+void rebalance_irqs(unsigned int from, bool up)
+{
+    int irq;
+
+    if ( cpumask_empty(&cpu_online_map) )
+        return;
+
+    for ( irq = NR_LOCAL_IRQS; irq < NR_IRQS; irq++ )
+        balance_irq(irq, from, up);
+
+#ifdef CONFIG_GICV3_ESPI
+    for ( irq = ESPI_BASE_INTID; irq < ESPI_BASE_INTID + gic_number_espis();
+          irq++ )
+        balance_irq(irq, from, up);
+#endif
+}
+#endif /* CONFIG_CPU_ONLINE_OFFLINE */
+
 static int cpu_callback(struct notifier_block *nfb, unsigned long action,
                         void *hcpu)
 {

@@ -172,6 +227,11 @@ static int cpu_callback(struct notifier_block *nfb, unsigned long action,
             printk(XENLOG_ERR "Unable to allocate local IRQ for CPU%u\n",
                    cpu);
         break;
+    case CPU_ONLINE:
+        if ( IS_ENABLED(CONFIG_CPU_ONLINE_OFFLINE) &&
+             system_state >= SYS_STATE_active )
+            rebalance_irqs(cpu, true);
+        break;
     }
 
     return notifier_from_errno(rc);

xen/arch/arm/smpboot.c +7 -0

index 1806c47a08..239d4a7a25 100644
@@ -433,6 +433,13 @@ void __cpu_disable(void)
 
     smp_mb();
 
+    /*
+     * Now that the interrupts are cleared and the CPU marked as offline,
+     * move interrupts out of it
+     */
+    if ( IS_ENABLED(CONFIG_CPU_ONLINE_OFFLINE) )
+        rebalance_irqs(cpu, false);
+
     /* Return to caller; eventually the IPI mechanism will unwind and the 
      * scheduler will drop to the idle loop, which will call stop_cpu(). */
 }

SMT-disable enforcement check is moved into a separate architecture-specific function.

For now this operations only support Arm64. For proper Arm32 support, there needs to be a mechanism to free per-cpu page tables, allocated in init_domheap_mappings. Also, hotplug is not supported if ITS enabled, and partially supported FFA, or TEE is enabled, as they use non-static IRQ actions.

Remove ifdef guards for x86 in flask, as cpu hotplug is now supported on more architectures.

Signed-off-by Mykyta Poturai <[email protected]>

---
v8->v9:
* simplify dependencies of config CPU_ONLINE_OFFLINE again

xen/common/smp.c +1 -0

@@ -104,6 +105,40 @@ void smp_call_function_interrupt(void)
     irq_exit();
 }
 
+#ifdef CONFIG_CPU_ONLINE_OFFLINE

xen/common/sysctl.c +7 -0

+        {
+        case XEN_SYSCTL_CPU_HOTPLUG_ONLINE:
+            fn = cpu_up_helper;
+            break;
+
+        case XEN_SYSCTL_CPU_HOTPLUG_OFFLINE:
+            fn = cpu_down_helper;

With CPU hotplug sysctls implemented on Arm it becomes useful to have a tool for calling them.

According to the commit history it seems that putting hptool under config MIGRATE was a measure to fix IA64 build. As IA64 is no longer supported it can now be brought back. So build it unconditionally.

Operations specific to x86 architecture are moved into a separate file and only built on x86.

Signed-off-by Mykyta Poturai <[email protected]>

Signed-off-by Mykyta Poturai <[email protected]>

---
v8->v9:
* fix typos

v7->v8:
* remove support status update
* update config option name

v6->v7:
* add testing and limitations

v5->v6:
* no changes

v4->v5:
* s/supported/implemented/
* update SUPPORT.md

v3->v4:
* update configuration section

v2->v3:
* patch introduced

---
 docs/misc/cpu-hotplug.txt | 98 +++++++++++++++++++++++++++++++++++++++
 1 file changed, 98 insertions(+)
 create mode 100644 docs/misc/cpu-hotplug.txt

docs/misc/cpu-hotplug.txt new file +26 -0

new file mode 100644
index 0000000000..4f6068db30
@@ -0,0 +1,98 @@
+CPU Hotplug
+===========
+
+CPU hotplug is a feature that allows pCPU cores to be added to or removed from a
+running system without requiring a reboot. It is implemented on x86 and Arm64
+architectures.
+
+Implementation Details
+----------------------
+
+CPU hotplug is implemented through the `XEN_SYSCTL_CPU_HOTPLUG_*` sysctl calls.
+The specific calls are:
+
+- `XEN_SYSCTL_CPU_HOTPLUG_ONLINE`: Brings a pCPU online
+- `XEN_SYSCTL_CPU_HOTPLUG_OFFLINE`: Takes a pCPU offline
+- `XEN_SYSCTL_CPU_HOTPLUG_SMT_ENABLE`: Enables SMT threads (x86 only)
+- `XEN_SYSCTL_CPU_HOTPLUG_SMT_DISABLE`: Disables SMT threads (x86 only)
+
+All cores can be disabled, assuming hardware support, except for the boot core.
+Sysctl calls are routed to the boot core before doing any actual up/down
+operations on other cores.
+
+If there are Xen-bound interrupts pinned to the pCPU being offlined, they will
+be automatically migrated to other online pCPUs. Interrupts used by guest
+domains are handled by the scheduler when it reschedules the vCPUs to a new,
+online, pCPU. When a pCPU is being onlined, some Xen-bound interrupts will get

docs/misc/cpu-hotplug.txt new file +59 -0

+
+Configuration
+-------------
+
+The presence of the feature is controlled by CONFIG_CPU_ONLINE_OFFLINE option.
+It is enabled by default on x86 architecture. On Arm64, the option is disabled
+by default and marked as EXPERT. xen-hptool userspace tool is built
+unconditionally.
+
+Usage
+-----
+
+Disable core:
+
+$ xen-hptool cpu-offline 2
+Prepare to offline CPU 2
+(XEN) Removing cpu 2 from runqueue 0
+CPU 2 offlined successfully
+
+Enable core:
+
+$ xen-hptool cpu-online 2
+Prepare to online CPU 2
+(XEN) Bringing up CPU2
+(XEN) GICv3: CPU2: Found redistributor in region 0 @00000a004005c000
+(XEN) CPU2: Guest atomics will try 1 times before pausing the domain
+(XEN) CPU 2 booted.
+(XEN) Adding cpu 2 to runqueue 0
+CPU 2 onlined successfully
+
+Disabling a core with pinned vCPUs:
+
+$ xl vcpu-pin 0 3 3 3
+$ xl vcpu-pin 0 2 3 3
+$ xl vcpu-pin 0 1 3 3
+$ xl vcpu-pin 0 0 3 3
+$ xen-hptool cpu-offline 3
+Prepare to offline CPU 3
+(XEN) Breaking affinity for d0v0
+(XEN) Breaking affinity for d0v1
+(XEN) Breaking affinity for d0v2
+(XEN) Breaking affinity for d0v3
+(XEN) Removing cpu 3 from runqueue 0
+CPU 3 offlined successfully
+
+Testing
+-------
+
+The CPU hotplug feature has been tested on both x86 and Arm64 QEMU setups and on
+R-Car Gen5 (Arm64) hardware.
+
+The tests included:
+- Offlining and onlining cores with no pinned vCPUs
+- Offlining cores with pinned vCPUs
+- Offlining cores with Xen-bound interrupts
+- Offlining all cores except the boot core
+- Offlining the boot core (expected to fail)
+- Enabling and disabling SMT threads (x86 only)
+- Offlining cores to which guests with passthrough devices are pinned

Hi,

Mykyta Poturai <[email protected]> writes:

I have a couple of questions, please see below

Quoted 61 lines

Signed-off-by Mykyta Poturai <[email protected]>

---
v8->v9:
* fix typos

v7->v8:
* remove support status update
* update config option name

v6->v7:
* add testing and limitations

v5->v6:
* no changes

v4->v5:
* s/supported/implemented/
* update SUPPORT.md

v3->v4:
* update configuration section

v2->v3:
* patch introduced

---
  docs/misc/cpu-hotplug.txt | 98 +++++++++++++++++++++++++++++++++++++++
  1 file changed, 98 insertions(+)

create mode 100644 docs/misc/cpu-hotplug.txt

docs/misc/cpu-hotplug.txt new file +26 -0

new file mode 100644
index 0000000000..4f6068db30
@@ -0,0 +1,98 @@
+CPU Hotplug
+===========
+
+CPU hotplug is a feature that allows pCPU cores to be added to or removed from a
+running system without requiring a reboot. It is implemented on x86 and Arm64
+architectures.
+
+Implementation Details
+----------------------
+
+CPU hotplug is implemented through the `XEN_SYSCTL_CPU_HOTPLUG_*` sysctl calls.
+The specific calls are:
+
+- `XEN_SYSCTL_CPU_HOTPLUG_ONLINE`: Brings a pCPU online
+- `XEN_SYSCTL_CPU_HOTPLUG_OFFLINE`: Takes a pCPU offline
+- `XEN_SYSCTL_CPU_HOTPLUG_SMT_ENABLE`: Enables SMT threads (x86 only)
+- `XEN_SYSCTL_CPU_HOTPLUG_SMT_DISABLE`: Disables SMT threads (x86 only)
+
+All cores can be disabled, assuming hardware support, except for the boot core.
+Sysctl calls are routed to the boot core before doing any actual up/down
+operations on other cores.
+
+If there are Xen-bound interrupts pinned to the pCPU being offlined, they will
+be automatically migrated to other online pCPUs. Interrupts used by guest
+domains are handled by the scheduler when it reschedules the vCPUs to a new,
+online, pCPU. When a pCPU is being onlined, some Xen-bound interrupts will get

docs/misc/cpu-hotplug.txt line 36

Quoted 10 lines

docs/misc/cpu-hotplug.txt new file +10 -0

+redistributed to the newly onlined pCPU to prevent imbalance.
+
+If pCPU being offlined has some vCPUs pinned to it, they will be automatically
+unpinned and migrated to other online pCPUs.
+
+Limitations
+-----------
+
+On Arm64 cpu hotplug is currently not compatible with ITS, due to an issue with
+the redistributor assignment.

In this case you probably want to make sure that CONFIG_CPU_ONLINE_OFFLINE is incompatible with CONFIG_HAS_ITS

docs/misc/cpu-hotplug.txt line 39

Quoted 3 lines

docs/misc/cpu-hotplug.txt new file +3 -0

+
+On Arm64 there can be problems with FFA if secure FW supports the notification
+ABI, and with TEE, as both use non-static IRQ actions.

Could you please elaborate on this? AFAIK, OP-TEE has own cpu_up/cpu_down callbacks, so it should know which CPUs are online.

Quoted 81 lines · Volodymyr Babchuk

Hi,

Mykyta Poturai <[email protected]> writes:

I have a couple of questions, please see below

Quoted 61 lines

Signed-off-by Mykyta Poturai <[email protected]>

---
v8->v9:
* fix typos

v7->v8:
* remove support status update
* update config option name

v6->v7:
* add testing and limitations

v5->v6:
* no changes

v4->v5:
* s/supported/implemented/
* update SUPPORT.md

v3->v4:
* update configuration section

v2->v3:
* patch introduced

---
  docs/misc/cpu-hotplug.txt | 98 +++++++++++++++++++++++++++++++++++++++
  1 file changed, 98 insertions(+)

create mode 100644 docs/misc/cpu-hotplug.txt

docs/misc/cpu-hotplug.txt new file +26 -0

new file mode 100644
index 0000000000..4f6068db30
@@ -0,0 +1,98 @@
+CPU Hotplug
+===========
+
+CPU hotplug is a feature that allows pCPU cores to be added to or removed from a
+running system without requiring a reboot. It is implemented on x86 and Arm64
+architectures.
+
+Implementation Details
+----------------------
+
+CPU hotplug is implemented through the `XEN_SYSCTL_CPU_HOTPLUG_*` sysctl calls.
+The specific calls are:
+
+- `XEN_SYSCTL_CPU_HOTPLUG_ONLINE`: Brings a pCPU online
+- `XEN_SYSCTL_CPU_HOTPLUG_OFFLINE`: Takes a pCPU offline
+- `XEN_SYSCTL_CPU_HOTPLUG_SMT_ENABLE`: Enables SMT threads (x86 only)
+- `XEN_SYSCTL_CPU_HOTPLUG_SMT_DISABLE`: Disables SMT threads (x86 only)
+
+All cores can be disabled, assuming hardware support, except for the boot core.
+Sysctl calls are routed to the boot core before doing any actual up/down
+operations on other cores.
+
+If there are Xen-bound interrupts pinned to the pCPU being offlined, they will
+be automatically migrated to other online pCPUs. Interrupts used by guest
+domains are handled by the scheduler when it reschedules the vCPUs to a new,
+online, pCPU. When a pCPU is being onlined, some Xen-bound interrupts will get

docs/misc/cpu-hotplug.txt line 36

Quoted 10 lines

docs/misc/cpu-hotplug.txt new file +10 -0

+redistributed to the newly onlined pCPU to prevent imbalance.
+
+If pCPU being offlined has some vCPUs pinned to it, they will be automatically
+unpinned and migrated to other online pCPUs.
+
+Limitations
+-----------
+
+On Arm64 cpu hotplug is currently not compatible with ITS, due to an issue with
+the redistributor assignment.

In this case you probably want to make sure that CONFIG_CPU_ONLINE_OFFLINE is incompatible with CONFIG_HAS_ITS

Quoted 6 lines

docs/misc/cpu-hotplug.txt line 39

Quoted 3 lines

docs/misc/cpu-hotplug.txt new file +3 -0

+
+On Arm64 there can be problems with FFA if secure FW supports the notification
+ABI, and with TEE, as both use non-static IRQ actions.

Could you please elaborate on this? AFAIK, OP-TEE has own cpu_up/cpu_down callbacks, so it should know which CPUs are online.

It's more of a problem with the way interrupt handlers are set up in Xen. Mykola already has fixes for it in a separate series, but I'm not how to best sync up our efforts.