Implement CPU hotplug on Arm

index fae80e6cd2..28ad4903b4 100644
@@ -232,7 +232,9 @@ static void gic_update_one_lr(struct vcpu *v, int i)
             if ( test_bit(GIC_IRQ_GUEST_MIGRATING, &p->status) )
             {
                 struct vcpu *v_target = vgic_get_target_vcpu(v, irq);
+                spin_lock(&p->desc->lock);
                 irq_set_affinity(p->desc, cpumask_of(v_target->processor));
+                spin_unlock(&p->desc->lock);
                 clear_bit(GIC_IRQ_GUEST_MIGRATING, &p->status);
             }
         }

index 8d3517e761..594aaedd22 100644
@@ -224,10 +224,15 @@ static inline struct domain *irq_get_domain(struct irq_desc *desc)
     return irq_get_guest_info(desc)->d;
 }
 
+/* Must be called with desc->lock held */
 void irq_set_affinity(struct irq_desc *desc, const cpumask_t *mask)
 {
-    if ( desc != NULL )
-        desc->handler->set_affinity(desc, mask);
+    if ( desc == NULL )
+        return;
+
+    ASSERT(spin_is_locked(&desc->lock));
+    cpumask_copy(desc->affinity, mask);
+    desc->handler->set_affinity(desc, mask);
 }
 
 int request_irq(unsigned int irq, unsigned int irqflags,

index acded6a40f..53f36780d1 100644
@@ -490,7 +490,9 @@ bool vgic_migrate_irq(struct vcpu *old, struct vcpu *new, unsigned int irq)
 
     if ( list_empty(&p->inflight) )
     {
+        spin_lock(&p->desc->lock);
         irq_set_affinity(p->desc, cpumask_of(new->processor));
+        spin_unlock(&p->desc->lock);
         spin_unlock_irqrestore(&old->arch.vgic.lock, flags);
         return true;
     }

@@ -498,7 +500,9 @@ bool vgic_migrate_irq(struct vcpu *old, struct vcpu *new, unsigned int irq)
     if ( !list_empty(&p->lr_queue) )
     {
         vgic_remove_irq_from_queues(old, p);
+        spin_lock(&p->desc->lock);
         irq_set_affinity(p->desc, cpumask_of(new->processor));
+        spin_unlock(&p->desc->lock);
         spin_unlock_irqrestore(&old->arch.vgic.lock, flags);
         vgic_inject_irq(new->domain, new, irq, true);
         return true;

@@ -518,6 +522,7 @@ void arch_move_irqs(struct vcpu *v)
     struct domain *d = v->domain;
     struct pending_irq *p;
     struct vcpu *v_target;
+    unsigned long flags;
     int i;
 
     /*

@@ -539,7 +544,13 @@ void arch_move_irqs(struct vcpu *v)
         p = irq_to_pending(v_target, virq);
 
         if ( v_target == v && !test_bit(GIC_IRQ_GUEST_MIGRATING, &p->status) )
+        {
+            if ( !p->desc )
+                continue;
+            spin_lock_irqsave(&p->desc->lock, flags);
             irq_set_affinity(p->desc, cpu_mask);
+            spin_unlock_irqrestore(&p->desc->lock, flags);
+        }
     }
 }
 

@@ -619,8 +630,8 @@ void vgic_enable_irqs(struct vcpu *v, uint32_t r, unsigned int n)
         spin_unlock_irqrestore(&v_target->arch.vgic.lock, flags);
         if ( p->desc != NULL )
         {
-            irq_set_affinity(p->desc, cpumask_of(v_target->processor));
             spin_lock_irqsave(&p->desc->lock, flags);
+            irq_set_affinity(p->desc, cpumask_of(v_target->processor));
             /*
              * The irq cannot be a PPI, we only support delivery of SPIs
              * to guests.

@@ -975,4 +986,3 @@ void vgic_check_inflight_irqs_pending(struct vcpu *v, unsigned int rank, uint32_
  * indent-tabs-mode: nil
  * End:
  */
-

index b7c2d7ce99..fc04741ca1 100644
@@ -159,24 +159,23 @@ static void vgic_mmio_write_target(struct vcpu *vcpu,
     for ( i = 0; i < len; i++ )
     {
         struct vgic_irq *irq = vgic_get_irq(vcpu->domain, NULL, intid + i);
+        struct irq_desc *desc = irq_to_desc(irq->hwintid);
 
-        spin_lock_irqsave(&irq->irq_lock, flags);
+        spin_lock_irqsave(&desc->lock, flags);
+        spin_lock(&irq->irq_lock);
 
         irq->targets = (val >> (i * 8)) & cpu_mask;
         if ( irq->targets )
         {
             irq->target_vcpu = vcpu->domain->vcpu[ffs(irq->targets) - 1];
             if ( irq->hw )
-            {
-                struct irq_desc *desc = irq_to_desc(irq->hwintid);
-
                 irq_set_affinity(desc, cpumask_of(irq->target_vcpu->processor));
-            }
         }
         else
             irq->target_vcpu = NULL;
 
-        spin_unlock_irqrestore(&irq->irq_lock, flags);
+        spin_unlock(&irq->irq_lock);
+        spin_unlock_irqrestore(&desc->lock, flags);
         vgic_put_irq(vcpu->domain, irq);
     }
 }

index 84212bbefe..663fa16995 100644
@@ -815,22 +815,27 @@ void arch_move_irqs(struct vcpu *v)
     {
         struct vgic_irq *irq = vgic_get_irq(d, NULL, i + VGIC_NR_PRIVATE_IRQS);
         unsigned long flags;
+        irq_desc_t *desc;
 
         if ( !irq )
             continue;
 
-        spin_lock_irqsave(&irq->irq_lock, flags);
-
-        /* Only hardware mapped vIRQs that are targeting this vCPU. */
-        if ( irq->hw && irq->target_vcpu == v)
+        if ( !irq->hw )
         {
-            irq_desc_t *desc = irq_to_desc(irq->hwintid);
+            vgic_put_irq(d, irq);
+            continue;
+        }
 
+        desc = irq_to_desc(irq->hwintid);
+        spin_lock_irqsave(&desc->lock, flags);
+        spin_lock(&irq->irq_lock);
+
+        /* Only hardware mapped vIRQs that are targeting this vCPU. */
+        if ( irq->target_vcpu == v )
             irq_set_affinity(desc, cpumask_of(v->processor));
-        }
 
-        spin_unlock_irqrestore(&irq->irq_lock, flags);
-        vgic_put_irq(d, irq);
+        spin_unlock(&irq->irq_lock);
+        spin_unlock_irqrestore(&desc->lock, flags);
     }
 }
 

index 594aaedd22..d5145a4723 100644
@@ -166,6 +166,58 @@ static int init_local_irq_data(unsigned int cpu)
     return 0;
 }
 
+static void balance_irq(int irq, unsigned int from, bool up)
+{
+    static int cpu_next;
+    struct irq_desc *desc = irq_to_desc(irq);
+    unsigned long flags;
+
+    ASSERT(!cpumask_empty(&cpu_online_map));
+
+    spin_lock_irqsave(&desc->lock, flags);
+    if ( likely(!desc->action) )
+        goto out;
+
+    if ( likely(test_bit(_IRQ_GUEST, &desc->status) ||
+                test_bit(_IRQ_MOVE_PENDING, &desc->status)) )
+        goto out;
+
+    /*
+     * Setting affinity to a mask of multiple CPUs causes the GIC drivers to
+     * select one CPU from that mask. If the dying CPU was included in the IRQ's
+     * affinity mask, we cannot determine exactly which CPU the interrupt is
+     * currently routed to, as GIC drivers lack a concrete get_affinity API. So
+     * to be safe we must reroute it to a new, definitely online, CPU. In the
+     * case of CPU going down, we move only the interrupt that could reside on
+     * it. Otherwise, we rearrange all interrupts in a round-robin fashion.
+     */
+    if ( !up && !cpumask_test_cpu(from, desc->affinity) )
+        goto out;
+
+    cpu_next = cpumask_cycle(cpu_next, &cpu_online_map);
+    irq_set_affinity(desc, cpumask_of(cpu_next));
+
+out:
+    spin_unlock_irqrestore(&desc->lock, flags);
+}
+
+static void rebalance_irqs(unsigned int from, bool up)
+{
+    int irq;
+
+    if ( cpumask_empty(&cpu_online_map) )
+        return;
+
+    for ( irq = NR_LOCAL_IRQS; irq < NR_IRQS; irq++ )
+        balance_irq(irq, from, up);
+
+#ifdef CONFIG_GICV3_ESPI
+    for ( irq = ESPI_BASE_INTID; irq < ESPI_BASE_INTID + gic_number_espis();
+          irq++ )
+        balance_irq(irq, from, up);
+#endif
+}
+
 static int cpu_callback(struct notifier_block *nfb, unsigned long action,
                         void *hcpu)
 {

@@ -180,6 +232,15 @@ static int cpu_callback(struct notifier_block *nfb, unsigned long action,
             printk(XENLOG_ERR "Unable to allocate local IRQ for CPU%u\n",
                    cpu);
         break;
+    case CPU_ONLINE:
+        if ( IS_ENABLED(CONFIG_CPU_ONLINE_OFFLINE) &&
+             system_state >= SYS_STATE_active )
+            rebalance_irqs(cpu, true);
+        break;
+    case CPU_DEAD:
+        if ( IS_ENABLED(CONFIG_CPU_ONLINE_OFFLINE) )
+            rebalance_irqs(cpu, false);
+        break;
     }
 
     return notifier_from_errno(rc);

index 0f3c791fc2..2d43d20143 100644
@@ -735,6 +735,12 @@ ret_t do_platform_op(
     {
         int cpu = op->u.cpu_ol.cpuid;
 
+        if ( !IS_ENABLED(CONFIG_CPU_ONLINE_OFFLINE) )
+        {
+            ret = -EOPNOTSUPP;
+            break;
+        }
+
         if ( cpu >= nr_cpu_ids || !cpu_present(cpu) ||
              clocksource_is_tsc() )
         {

@@ -757,6 +763,12 @@ ret_t do_platform_op(
     {
         int cpu = op->u.cpu_ol.cpuid;
 
+        if ( !IS_ENABLED(CONFIG_CPU_ONLINE_OFFLINE) )
+        {
+            ret = -EOPNOTSUPP;
+            break;
+        }
+
         if ( cpu == 0 )
         {
             ret = -EOPNOTSUPP;

index ae4e5a66d7..e3c4c0d5e0 100644
@@ -416,6 +416,7 @@ void cf_check call_function_interrupt(void)
     smp_call_function_interrupt();
 }
 
+#ifdef CONFIG_CPU_ONLINE_OFFLINE
 long cf_check cpu_up_helper(void *data)
 {
     unsigned int cpu = (unsigned long)data;

@@ -443,8 +444,10 @@ long cf_check cpu_down_helper(void *data)
 {
     int cpu = (unsigned long)data;
     int ret = cpu_down(cpu);
+
     /* Have one more go on EBUSY. */
     if ( ret == -EBUSY )
         ret = cpu_down(cpu);
     return ret;
 }
+#endif

index 6bd4e191a7..10d485e633 100644
@@ -53,6 +53,12 @@ static long cf_check smt_up_down_helper(void *data)
     unsigned int cpu, sibling_mask = boot_cpu_data.x86_num_siblings - 1;
     int ret = 0;
 
+    if ( !IS_ENABLED(CONFIG_CPU_ONLINE_OFFLINE) )
+    {
+        ASSERT_UNREACHABLE();
+        return -EOPNOTSUPP;
+    }
+
     opt_smt = up;
 
     for_each_present_cpu ( cpu )

@@ -120,6 +126,12 @@ long arch_do_sysctl(
         long (*fn)(void *data);
         void *hcpu;
 
+        if ( !IS_ENABLED(CONFIG_CPU_ONLINE_OFFLINE) )
+        {
+            ret = -EOPNOTSUPP;
+            break;
+        }
+
         switch ( op )
         {
         case XEN_SYSCTL_CPU_HOTPLUG_ONLINE:

index b372472188..f84f217359 100644
@@ -44,6 +44,17 @@ void smp_send_call_function_mask(const cpumask_t *mask)
     }
 }
 
+#ifdef CONFIG_CPU_ONLINE_OFFLINE
+/*
+ * We currently don't support SMT on ARM so we don't need any special logic for
+ * CPU disabling
+ */
+bool arch_cpu_keep_online(unsigned int cpu)
+{
+    return true;
+}
+#endif
+
 /*
  * Local variables:
  * mode: C

index 572b39b01c..6c33567cb5 100644
@@ -44,9 +44,6 @@ int cpu_add(uint32_t apic_id, uint32_t acpi_id, uint32_t pxm);
 
 void __stop_this_cpu(void);
 
-long cf_check cpu_up_helper(void *data);
-long cf_check cpu_down_helper(void *data);
-
 long cf_check core_parking_helper(void *data);
 bool core_parking_remove(unsigned int cpu);
 uint32_t get_cur_idle_nums(void);

index e3c4c0d5e0..b56d8dfd36 100644
@@ -417,37 +417,9 @@ void cf_check call_function_interrupt(void)
 }
 
 #ifdef CONFIG_CPU_ONLINE_OFFLINE
-long cf_check cpu_up_helper(void *data)
+bool arch_cpu_keep_online(unsigned int cpu)
 {
-    unsigned int cpu = (unsigned long)data;
-    int ret = cpu_up(cpu);
-
-    /* Have one more go on EBUSY. */
-    if ( ret == -EBUSY )
-        ret = cpu_up(cpu);
-
-    if ( !ret && !opt_smt &&
-         cpu_data[cpu].compute_unit_id == INVALID_CUID &&
-         cpumask_weight(per_cpu(cpu_sibling_mask, cpu)) > 1 )
-    {
-        ret = cpu_down_helper(data);
-        if ( ret )
-            printk("Could not re-offline CPU%u (%d)\n", cpu, ret);
-        else
-            ret = -EPERM;
-    }
-
-    return ret;
-}
-
-long cf_check cpu_down_helper(void *data)
-{
-    int cpu = (unsigned long)data;
-    int ret = cpu_down(cpu);
-
-    /* Have one more go on EBUSY. */
-    if ( ret == -EBUSY )
-        ret = cpu_down(cpu);
-    return ret;
+    return opt_smt || cpu_data[cpu].compute_unit_id != INVALID_CUID ||
+           cpumask_weight(per_cpu(cpu_sibling_mask, cpu)) <= 1;
 }
 #endif

index 10d485e633..a4254beabb 100644
@@ -121,13 +121,13 @@ long arch_do_sysctl(
 
     case XEN_SYSCTL_cpu_hotplug:
     {
-        unsigned int cpu = sysctl->u.cpu_hotplug.cpu;
         unsigned int op  = sysctl->u.cpu_hotplug.op;
         long (*fn)(void *data);
         void *hcpu;
 
         if ( !IS_ENABLED(CONFIG_CPU_ONLINE_OFFLINE) )
         {
+            ASSERT_UNREACHABLE();
             ret = -EOPNOTSUPP;
             break;
         }

@@ -135,13 +135,10 @@ long arch_do_sysctl(
         switch ( op )
         {
         case XEN_SYSCTL_CPU_HOTPLUG_ONLINE:
-            fn = cpu_up_helper;
-            hcpu = _p(cpu);
-            break;
-
         case XEN_SYSCTL_CPU_HOTPLUG_OFFLINE:
-            fn = cpu_down_helper;
-            hcpu = _p(cpu);
+            /* Handled by common code */
+            ASSERT_UNREACHABLE();
+            ret = -EOPNOTSUPP;
             break;
 
         case XEN_SYSCTL_CPU_HOTPLUG_SMT_ENABLE:

index a011f541f1..af532d4873 100644
@@ -16,6 +16,7 @@
  * GNU General Public License for more details.
  */
 
+#include <xen/cpu.h>
 #include <asm/hardirq.h>
 #include <asm/processor.h>
 #include <xen/spinlock.h>

@@ -104,6 +105,40 @@ void smp_call_function_interrupt(void)
     irq_exit();
 }
 
+#ifdef CONFIG_CPU_ONLINE_OFFLINE
+long cf_check cpu_up_helper(void *data)
+{
+    unsigned int cpu = (unsigned long)data;
+    int ret = cpu_up(cpu);
+
+    /* Have one more go on EBUSY. */
+    if ( ret == -EBUSY )
+        ret = cpu_up(cpu);
+
+    if ( !ret && !arch_cpu_keep_online(cpu) )
+    {
+        ret = cpu_down_helper(data);
+        if ( ret )
+            printk("Could not re-offline CPU%u (%d)\n", cpu, ret);
+        else
+            ret = -EPERM;
+    }
+
+    return ret;
+}
+
+long cf_check cpu_down_helper(void *data)
+{
+    unsigned int cpu = (unsigned long)data;
+    int ret = cpu_down(cpu);
+
+    /* Have one more go on EBUSY. */
+    if ( ret == -EBUSY )
+        ret = cpu_down(cpu);
+    return ret;
+}
+#endif /* CONFIG_CPU_ONLINE_OFFLINE */
+
 /*
  * Local variables:
  * mode: C

index 8fb5ff0af3..dd21b6ff7e 100644
@@ -475,6 +475,38 @@ long do_sysctl(XEN_GUEST_HANDLE_PARAM(xen_sysctl_t) u_sysctl)
             copyback = 1;
         break;
 
+    case XEN_SYSCTL_cpu_hotplug:
+    {
+        long (*fn)(void *data);
+
+        ret = -EOPNOTSUPP;
+        if ( !IS_ENABLED(CONFIG_CPU_ONLINE_OFFLINE) )
+            break;
+
+        switch ( op->u.cpu_hotplug.op )
+        {
+        case XEN_SYSCTL_CPU_HOTPLUG_ONLINE:
+            fn = cpu_up_helper;
+            break;
+
+        case XEN_SYSCTL_CPU_HOTPLUG_OFFLINE:
+            fn = cpu_down_helper;
+            break;
+
+        default:
+            fn = NULL;
+            break;
+        }
+
+        if ( fn )
+        {
+            ret = continue_hypercall_on_cpu(0, fn, _p(op->u.cpu_hotplug.cpu));
+            break;
+        }
+    }
+
+        /* Use the arch handler for cases not handled here */
+        fallthrough;
     default:
         ret = arch_do_sysctl(op, u_sysctl);
         copyback = 0;

index 2ca9ff1bfc..6c011d73ef 100644
@@ -76,4 +76,10 @@ extern void *stack_base[NR_CPUS];
 void initialize_cpu_data(unsigned int cpu);
 int setup_cpu_root_pgt(unsigned int cpu);
 
+#ifdef CONFIG_CPU_ONLINE_OFFLINE
+bool arch_cpu_keep_online(unsigned int cpu);
+#endif
+long cf_check cpu_up_helper(void *data);
+long cf_check cpu_down_helper(void *data);
+
 #endif /* __XEN_SMP_H__ */

index d65ba0aeae..a1eed309b4 100644
@@ -957,7 +957,7 @@ static int cf_check flask_sysctl(const struct xen_sysctl *op)
     case XEN_SYSCTL_getdomaininfolist:
         return flask_getdomaininfo(dom_xen);
 
-#ifdef CONFIG_X86
+#ifdef CONFIG_CPU_ONLINE_OFFLINE
     case XEN_SYSCTL_cpu_hotplug:
         switch ( op->u.cpu_hotplug.op )
         {

278 lines not on this page · raw message

373 lines not on this page · raw message

new file mode 100644
index 0000000000..34f492c34b
@@ -0,0 +1,15 @@
+/* SPDX-License-Identifier: GPL-2.0-only */
+#ifndef __XEN_HPTOOL_H__
+#define __XEN_HPTOOL_H__
+
+#if defined(__i386__) || defined(__x86_64__)
+int hp_mem_online_func(int argc, char *argv[], xc_interface *xch);
+int hp_mem_query_func(int argc, char *argv[], xc_interface *xch);
+int hp_mem_offline_func(int argc, char *argv[], xc_interface *xch);
+int main_smt_enable(int argc, char *argv[], xc_interface *xch);
+int main_smt_disable(int argc, char *argv[], xc_interface *xch);
+#endif
+
+void show_help(void);
+
+#endif /* __XEN_HPTOOL_H__ */