Add support for a fake, para-virtualised machine

From: Marc Zyngier <[email protected]>

This patch adds support for SMP to mach-virt.

Signed-off-by Marc Zyngier <[email protected]>

Signed-off-by Will Deacon <[email protected]>

---
 arch/arm/mach-virt/Makefile  |   1 +
 arch/arm/mach-virt/headsmp.S |  38 ++++++++
 arch/arm/mach-virt/platsmp.c | 205 +++++++++++++++++++++++++++++++++++++++++++
 arch/arm/mach-virt/virt.c    |   6 ++
 4 files changed, 250 insertions(+)
 create mode 100644 arch/arm/mach-virt/headsmp.S
 create mode 100644 arch/arm/mach-virt/platsmp.c

arch/arm/mach-virt/Makefile +1 -0

index 7ddbfa6..9ce8a28 100644
@@ -3,3 +3,4 @@
 #
 
 obj-y					:= virt.o
+obj-$(CONFIG_SMP)			+= platsmp.o headsmp.o

arch/arm/mach-virt/headsmp.S new file +17 -0

new file mode 100644
index 0000000..e27afb0
@@ -0,0 +1,38 @@
+/*
+ *  Copyright (c) 2012 ARM Limited
+ *  All Rights Reserved
+ *
+ * This program is free software; you can redistribute it and/or modify
+ * it under the terms of the GNU General Public License version 2 as
+ * published by the Free Software Foundation.
+ */
+#include <linux/linkage.h>
+#include <linux/init.h>
+
+	__INIT
+
+/*
+ * This provides a "holding pen" into which all secondary cores are held
+ * until we're ready for them to initialise.
+ */

arch/arm/mach-virt/headsmp.S new file +11 -0

+
+	/*
+	 * we've been released from the holding pen: secondary_stack
+	 * should now contain the SVC stack for this core
+	 */
+	b	secondary_startup
+
+	.align
+1:	.long	.
+	.long	pen_release
+ENDPROC(virt_secondary_startup)

arch/arm/mach-virt/platsmp.c new file +66 -0

new file mode 100644
index 0000000..fe02f51
@@ -0,0 +1,205 @@
+/*
+ * Dummy Virtual Machine - does what it says on the tin.
+ *
+ * SMP operations, shamelessly stolen from:
+ * arch/arm64/kernel/smp.c
+ *
+ * Copyright (C) 2012 ARM Ltd
+ * Author: Catalin Marinas <[email protected]>
+ * Author: Will Deacon <[email protected]>
+ * Author: Marc Zyngier <[email protected]>
+ *
+ * This program is free software; you can redistribute it and/or modify
+ * it under the terms of the GNU General Public License version 2 as
+ * published by the Free Software Foundation.
+ *
+ * This program is distributed in the hope that it will be useful,
+ * but WITHOUT ANY WARRANTY; without even the implied warranty of
+ * MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE.  See the
+ * GNU General Public License for more details.
+ *
+ * You should have received a copy of the GNU General Public License
+ * along with this program.  If not, see <http://www.gnu.org/licenses/>.
+ */
+
+#include <linux/smp.h>
+#include <linux/errno.h>
+#include <linux/delay.h>
+#include <linux/device.h>
+#include <linux/jiffies.h>
+#include <linux/of.h>
+
+#include <asm/cacheflush.h>
+#include <asm/smp_plat.h>
+#include <asm/hardware/gic.h>
+
+extern void virt_secondary_startup(void);
+
+static DEFINE_RAW_SPINLOCK(boot_lock);
+static phys_addr_t cpu_release_addr[NR_CPUS];
+
+/*
+ * Write secondary_holding_pen_release in a way that is guaranteed to be
+ * visible to all observers, irrespective of whether they're taking part
+ * in coherency or not.  This is necessary for the hotplug code to work
+ * reliably.
+ */
+static void __cpuinit write_pen_release(int val)
+{
+	void *start = (void *)&pen_release;
+	unsigned long size = sizeof(pen_release);
+
+	pen_release = val;
+	smp_wmb();
+	__cpuc_flush_dcache_area(start, size);
+	outer_clean_range(__pa(&pen_release), __pa(&pen_release + 1));
+}
+
+/*
+ * Enumerate the possible CPU set from the device tree.
+ */
+static void __init virt_smp_init_cpus(void)
+{
+	const char *enable_method;
+	struct device_node *dn = NULL;
+	int cpu = 0;
+	u32 release_addr;

arch/arm/mach-virt/platsmp.c new file +129 -0

+			pr_err("CPU %d: missing or invalid enable-method property: %s\n",
+			       cpu, enable_method);
+			goto next;
+		}
+
+		/*
+		 * Determine the address from which the CPU is polling.
+		 */
+		if (of_property_read_u32(dn, "cpu-release-addr", &release_addr)) {
+			pr_err("CPU %d: missing or invalid cpu-release-addr property\n",
+			       cpu);
+			goto next;
+		}
+
+		cpu_release_addr[cpu] = release_addr;
+		set_cpu_possible(cpu, true);
+next:
+		cpu++;
+	}
+
+	/* sanity check */
+	if (cpu > NR_CPUS)
+		pr_warning("no. of cores (%d) greater than configured maximum of %d - clipping\n",
+			   cpu, NR_CPUS);
+
+	set_smp_cross_call(gic_raise_softirq);
+}
+
+static void __init virt_smp_prepare_cpus(unsigned int max_cpus)
+{
+	int cpu;
+	void **release_addr;
+	unsigned int ncores = num_possible_cpus();
+
+	/*
+	 * are we trying to boot more cores than exist?
+	 */
+	if (max_cpus > ncores)
+		max_cpus = ncores;
+
+	/*
+	 * Initialise the present map (which describes the set of CPUs
+	 * actually populated at the present time) and release the
+	 * secondaries from the bootloader.
+	 */
+	for_each_possible_cpu(cpu) {
+		if (max_cpus == 0)
+			break;
+
+		if (!cpu_release_addr[cpu])
+			continue;
+
+		release_addr = __va(cpu_release_addr[cpu]);
+		release_addr[0] = (void *)__pa(virt_secondary_startup);
+		smp_wmb();
+		__cpuc_flush_dcache_area(release_addr, sizeof(release_addr[0]));
+		outer_clean_range(__pa(release_addr), __pa(release_addr+1));
+
+		set_cpu_present(cpu, true);
+		max_cpus--;
+	}
+}
+
+static int __cpuinit virt_boot_secondary(unsigned int cpu,
+					 struct task_struct *idle)
+{
+	unsigned long timeout;
+
+	/*
+	 * Set synchronisation state between this boot processor
+	 * and the secondary one
+	 */
+	raw_spin_lock(&boot_lock);
+
+	/*
+	 * Update the pen release flag.
+	 */
+	write_pen_release(cpu);
+
+	/*
+	 * Send the secondary CPU a soft interrupt, causing the
+	 * secondaries to read pen_release.
+	 */
+	gic_raise_softirq(cpumask_of(cpu), 0);
+
+	timeout = jiffies + (1 * HZ);
+	while (time_before(jiffies, timeout)) {
+		if (pen_release == -1UL)
+			break;
+		udelay(10);
+	}
+
+	/*
+	 * Now the secondary core is starting up let it run its
+	 * calibrations, then wait for it to finish
+	 */
+	raw_spin_unlock(&boot_lock);
+
+	return pen_release != -1 ? -ENOSYS : 0;
+}
+
+static void __cpuinit virt_secondary_init(unsigned int cpu)
+{
+	/*
+	 * if any interrupts are already enabled for the primary
+	 * core (e.g. timer irq), then they will not have been enabled
+	 * for us: do so
+	 */
+	gic_secondary_init(0);
+
+	/*
+	 * let the primary processor know we're out of the
+	 * pen, then head off into the C entry point
+	 */
+	write_pen_release(-1);
+
+	/*
+	 * Synchronise with the boot thread.
+	 */
+	raw_spin_lock(&boot_lock);
+	raw_spin_unlock(&boot_lock);
+}
+
+struct smp_operations __initdata virt_smp_ops = {
+	.smp_init_cpus		= virt_smp_init_cpus,
+	.smp_prepare_cpus	= virt_smp_prepare_cpus,
+	.smp_secondary_init	= virt_secondary_init,
+	.smp_boot_secondary	= virt_boot_secondary,
+};

arch/arm/mach-virt/virt.c +6 -0

index 174b9da..d764835 100644
@@ -20,6 +20,7 @@
 
 #include <linux/of_irq.h>
 #include <linux/of_platform.h>
+#include <linux/smp.h>
 
 #include <asm/arch_timer.h>
 #include <asm/hardware/gic.h>

@@ -56,10 +57,15 @@ static struct sys_timer virt_timer = {
 	.init = virt_timer_init,
 };
 
+#ifdef CONFIG_SMP
+extern struct smp_operations virt_smp_ops;
+#endif
+
 DT_MACHINE_START(VIRT, "Dummy Virtual Machine")
 	.init_irq	= gic_init_irq,
 	.handle_irq     = gic_handle_irq,
 	.timer		= &virt_timer,
 	.init_machine	= virt_init,
+	.smp		= smp_ops(virt_smp_ops),
 	.dt_compat	= virt_dt_match,
 MACHINE_END

Quoted 7 lines · Will Deacon

From: Marc Zyngier <[email protected]>

This patch adds support for SMP to mach-virt.

Signed-off-by Marc Zyngier <[email protected]>

Signed-off-by Will Deacon <[email protected]>

Quoted 10 lines

arch/arm/mach-virt/headsmp.S new file +10 -0

+/*
+ * This provides a "holding pen" into which all secondary cores are held
+ * until we're ready for them to initialise.
+ */
+ENTRY(virt_secondary_startup)
+	mrc	p15, 0, r0, c0, c0, 5
+	and	r0, r0, #15
+	adr	r4, 1f
+	ldmia	r4, {r5, r6}
+	sub	r4, r4, r5

arch/arm/mach-virt/headsmp.S line 27

Quoted 4 lines

arch/arm/mach-virt/headsmp.S new file +4 -0

+	add	r6, r6, r4
+pen:	ldr	r7, [r6]
+	cmp	r7, r0
+	bne	pen

Why is the pen is needed? It should only be needed for hotplug on systems that can't reset their cores. I'd hope you could design good virtual h/w.

Quoted 13 lines

arch/arm/mach-virt/platsmp.c new file +13 -0

+/*
+ * Enumerate the possible CPU set from the device tree.
+ */
+static void __init virt_smp_init_cpus(void)
+{
+	const char *enable_method;
+	struct device_node *dn = NULL;
+	int cpu = 0;
+	u32 release_addr;
+
+	while ((dn = of_find_node_by_type(dn, "cpu"))) {
+		if (cpu >= NR_CPUS)
+			goto next;

arch/arm/mach-virt/platsmp.c line 76

Quoted 6 lines

arch/arm/mach-virt/platsmp.c new file +6 -0

+
+		/*
+		 * We currently support only the "spin-table" enable-method.
+		 */
+		enable_method = of_get_property(dn, "enable-method", NULL);
+		if (!enable_method || strcmp(enable_method, "spin-table")) {

Are these documented?

Quoted 3 lines · Rob Herring

Why is the pen is needed? It should only be needed for hotplug on systems that can't reset their cores. I'd hope you could design good virtual h/w.

It's not so much about designing good virtual h/w as it is avoiding tying the platform to it. What we don't want is to mandate that in order to boot this machine, you *must* implement an emulation of some virtual power-controller or SMP booting device. If we go down that route, there's less advantage from having the virtual platform in the first place.

Quoted 10 lines · Will Deacon

Quoted 3 lines · Rob Herring

Why is the pen is needed? It should only be needed for hotplug on systems that can't reset their cores. I'd hope you could design good virtual h/w.

It's not so much about designing good virtual h/w as it is avoiding tying the platform to it. What we don't want is to mandate that in order to boot this machine, you *must* implement an emulation of some virtual power-controller or SMP booting device. If we go down that route, there's less advantage from having the virtual platform in the first place.

There is actually a bigger problem here. Let's say that you have a quad SMP platform. You've arranged for your kernel to boot and only bring one of those cores online.

You then kexec() or reboot. As far as the kernel is concerned, those other two CPUs are not online and are not running any kernel code; however in reality they could be sitting in this 'pen'.

The memory that these 'offline' CPUs is executing then gets overwritten, and that's game over for those CPUs.

So, the 'pen' approach in the kernel is fragile, I'd much rather not have it. It was fine in the beginning for the initial ARM Ltd SMP platforms but in this modern age it has no place in real platforms where there is proper control of the secondary CPUs.

Quoted 2 lines · Russell King - ARM Linux

The memory that these 'offline' CPUs is executing then gets overwritten, and that's game over for those CPUs.

That's not strictly true. The device-tree passed to the kernel should have a /memreserve/ entry for the SMP pen to avoid exactly this scenario. In real hardware, this still sucks because you have spinning CPUs burning up power but that's not such a problem with a virtual platform.

Quoted 8 lines · Will Deacon

Quoted 2 lines · Russell King - ARM Linux

The memory that these 'offline' CPUs is executing then gets overwritten, and that's game over for those CPUs.

That's not strictly true. The device-tree passed to the kernel should have a /memreserve/ entry for the SMP pen to avoid exactly this scenario. In real hardware, this still sucks because you have spinning CPUs burning up power but that's not such a problem with a virtual platform.

Umm. So let's see. If I'm running v3.6 stock kernel and want to kexec into a v3.7 stock kernel. The SMP pen is part of the v3.6 kernel, which will be located at 32K into the RAM. The v3.7 kernel will also want to occupy the same place. At some point you have to overwrite the v3.6 kernel with the v3.7 kernel image.

That happens _before_ the DT has been parsed, so any memreserve stuff will be ignored. And it's at that point that your "offline" secondary CPUs will have their instructions overwritten.

That's fine if the pen ends up being at the same place but that's not something we guarantee.

Quoted 10 lines · Russell King - ARM Linux

Quoted 8 lines · Will Deacon

Quoted 2 lines · Russell King - ARM Linux

The memory that these 'offline' CPUs is executing then gets overwritten, and that's game over for those CPUs.

That's not strictly true. The device-tree passed to the kernel should have a /memreserve/ entry for the SMP pen to avoid exactly this scenario. In real hardware, this still sucks because you have spinning CPUs burning up power but that's not such a problem with a virtual platform.

Quoted 5 lines

Umm. So let's see. If I'm running v3.6 stock kernel and want to kexec into a v3.7 stock kernel. The SMP pen is part of the v3.6 kernel, which will be located at 32K into the RAM. The v3.7 kernel will also want to occupy the same place. At some point you have to overwrite the v3.6 kernel with the v3.7 kernel image.

If the 3.6 kernel didn't bring those CPUs online, they will sit in the bootloader pen (out of the way of the kernel image) rather than the kernel pen so I don't think there will be a problem.

The problem you're describing actually happens when the 3.6 kernel onlines all of the CPUs, because now it has no way to hotplug them off safely. This is also an issue with non-virtualised hardware but we could solve it for the virtual platform by having a para-virtualised device for doing CPU hotplug.

Quoted 6 lines

That happens _before_ the DT has been parsed, so any memreserve stuff will be ignored. And it's at that point that your "offline" secondary CPUs will have their instructions overwritten.

That's fine if the pen ends up being at the same place but that's not something we guarantee.

Having CPUs in limbo between the bootloader the being online in the kernel is something we should just avoid. Isn't that pen __init anyway?

Quoted 4 lines · Will Deacon

The problem you're describing actually happens when the 3.6 kernel onlines all of the CPUs, because now it has no way to hotplug them off safely. This is also an issue with non-virtualised hardware but we could solve it for the virtual platform by having a para-virtualised device for doing CPU hotplug.

That situation exists on ARM Ltd platforms where there's no way to properly return them back to the boot loader. We should not be forcing this ARM Ltd platform deficiency onto other platforms as part of a "design", even virtual platforms.

Most other real-world platforms out there have a way to power off the unused secondary CPUs - Tegra and OMAP both do.

As far as virtual platforms go, how secondary CPUs are dealt with should already have been solved; I really can't imagine that KVM and XEN on other architectures end up with CPUs spinning in a loop inside the guest kernel waiting for the guest OS to ask them to boot. Neither can I imagine that KVM and XEN end up with CPUs spinning in the guest OS when CPUs are asked to be hot-unplugged.

Quoted 4 lines

Quoted 4 lines

That happens _before_ the DT has been parsed, so any memreserve stuff will be ignored. And it's at that point that your "offline" secondary CPUs will have their instructions overwritten.

Quoted 5 lines

Quoted 2 lines

That's fine if the pen ends up being at the same place but that's not something we guarantee.

Having CPUs in limbo between the bootloader the being online in the kernel is something we should just avoid. Isn't that pen __init anyway?

If you have hotplug enabled, all the secondary bringup code should be in the __cpuinit and __cpuinitdata sections.

Quoted 3 lines

Most other real-world platforms out there have a way to power off the unused secondary CPUs - Tegra and OMAP both do.

If a virtual machine powers off a virtual CPU, I doubt we want to power of its corresponding CPU -- that logic can remain in the host. All we need to do is kill the virtual CPU thread, which we can do easily enough. Booting is the more difficult problem because we introduce a reliance on a virtual device being ready incredibly early, essentially hardcoding part of the virtual machine.

Quoted 10 lines · Russell King - ARM Linux

Quoted 8 lines · Will Deacon

Quoted 2 lines · Russell King - ARM Linux

The memory that these 'offline' CPUs is executing then gets overwritten, and that's game over for those CPUs.

That's not strictly true. The device-tree passed to the kernel should have a /memreserve/ entry for the SMP pen to avoid exactly this scenario. In real hardware, this still sucks because you have spinning CPUs burning up power but that's not such a problem with a virtual platform.

Quoted 5 lines

Umm. So let's see. If I'm running v3.6 stock kernel and want to kexec into a v3.7 stock kernel. The SMP pen is part of the v3.6 kernel, which will be located at 32K into the RAM. The v3.7 kernel will also want to occupy the same place. At some point you have to overwrite the v3.6 kernel with the v3.7 kernel image.

If the 3.6 kernel didn't bring those CPUs online, they will sit in the bootloader pen (out of the way of the kernel image) rather than the kernel pen so I don't think there will be a problem.

The problem you're describing actually happens when the 3.6 kernel onlines all of the CPUs, because now it has no way to hotplug them off safely. This is also an issue with non-virtualised hardware but we could solve it for the virtual platform by having a para-virtualised device for doing CPU hotplug.

Quoted 4 lines

That happens _before_ the DT has been parsed, so any memreserve stuff will be ignored. And it's at that point that your "offline" secondary CPUs will have their instructions overwritten.

Quoted 2 lines

That's fine if the pen ends up being at the same place but that's not something we guarantee.

Having CPUs in limbo between the bootloader the being online in the kernel is something we should just avoid. Isn't that pen __init anyway?

Quoted 23 lines · Will Deacon

Quoted 5 lines · Russell King - ARM Linux

Umm. So let's see. If I'm running v3.6 stock kernel and want to kexec into a v3.7 stock kernel. The SMP pen is part of the v3.6 kernel, which will be located at 32K into the RAM. The v3.7 kernel will also want to occupy the same place. At some point you have to overwrite the v3.6 kernel with the v3.7 kernel image.

If the 3.6 kernel didn't bring those CPUs online, they will sit in the bootloader pen (out of the way of the kernel image) rather than the kernel pen so I don't think there will be a problem.

The problem you're describing actually happens when the 3.6 kernel onlines all of the CPUs, because now it has no way to hotplug them off safely. This is also an issue with non-virtualised hardware but we could solve it for the virtual platform by having a para-virtualised device for doing CPU hotplug.

Quoted 6 lines

That happens _before_ the DT has been parsed, so any memreserve stuff will be ignored. And it's at that point that your "offline" secondary CPUs will have their instructions overwritten.

That's fine if the pen ends up being at the same place but that's not something we guarantee.

Quoted 2 lines

Having CPUs in limbo between the bootloader the being online in the kernel is something we should just avoid. Isn't that pen __init anyway?

Aren't we mixing 2 pens here? You must have some simple bootloader containing vector table and a pen that the dtb points to, right? The pen you have in the kernel is only needed when hotplug only does a wfi. As you don't yet support hotplug, then you can drop all the kernel pen code.

If there is no way to reset the core, then couldn't the hotplug code tear down the cpu setup and just jump back to 0x0 which then returns to the bootloader's pen?

Quoted 27 lines · Rob Herring

Quoted 25 lines · Will Deacon

Quoted 5 lines · Russell King - ARM Linux

Umm. So let's see. If I'm running v3.6 stock kernel and want to kexec into a v3.7 stock kernel. The SMP pen is part of the v3.6 kernel, which will be located at 32K into the RAM. The v3.7 kernel will also want to occupy the same place. At some point you have to overwrite the v3.6 kernel with the v3.7 kernel image.

If the 3.6 kernel didn't bring those CPUs online, they will sit in the bootloader pen (out of the way of the kernel image) rather than the kernel pen so I don't think there will be a problem.

The problem you're describing actually happens when the 3.6 kernel onlines all of the CPUs, because now it has no way to hotplug them off safely. This is also an issue with non-virtualised hardware but we could solve it for the virtual platform by having a para-virtualised device for doing CPU hotplug.

Quoted 6 lines

That happens _before_ the DT has been parsed, so any memreserve stuff will be ignored. And it's at that point that your "offline" secondary CPUs will have their instructions overwritten.

That's fine if the pen ends up being at the same place but that's not something we guarantee.

Having CPUs in limbo between the bootloader the being online in the kernel is something we should just avoid. Isn't that pen __init anyway?

Quoted 4 lines

Aren't we mixing 2 pens here? You must have some simple bootloader containing vector table and a pen that the dtb points to, right? The pen you have in the kernel is only needed when hotplug only does a wfi. As you don't yet support hotplug, then you can drop all the kernel pen code.

Yes, both qemu and kvmtool have bootloader pens outside of the kernel but since wfi is not trapped by kvm, the secondaries can be released early due to a spurious wakeup so we need the second pen.

Quoted 27 lines · Rob Herring

Quoted 25 lines · Will Deacon

Quoted 5 lines · Russell King - ARM Linux

Umm. So let's see. If I'm running v3.6 stock kernel and want to kexec into a v3.7 stock kernel. The SMP pen is part of the v3.6 kernel, which will be located at 32K into the RAM. The v3.7 kernel will also want to occupy the same place. At some point you have to overwrite the v3.6 kernel with the v3.7 kernel image.

If the 3.6 kernel didn't bring those CPUs online, they will sit in the bootloader pen (out of the way of the kernel image) rather than the kernel pen so I don't think there will be a problem.

The problem you're describing actually happens when the 3.6 kernel onlines all of the CPUs, because now it has no way to hotplug them off safely. This is also an issue with non-virtualised hardware but we could solve it for the virtual platform by having a para-virtualised device for doing CPU hotplug.

Quoted 6 lines

That happens _before_ the DT has been parsed, so any memreserve stuff will be ignored. And it's at that point that your "offline" secondary CPUs will have their instructions overwritten.

That's fine if the pen ends up being at the same place but that's not something we guarantee.

Having CPUs in limbo between the bootloader the being online in the kernel is something we should just avoid. Isn't that pen __init anyway?

Quoted 4 lines

Aren't we mixing 2 pens here? You must have some simple bootloader containing vector table and a pen that the dtb points to, right? The pen you have in the kernel is only needed when hotplug only does a wfi. As you don't yet support hotplug, then you can drop all the kernel pen code.

Yes, both qemu and kvmtool have bootloader pens outside of the kernel but since wfi is not trapped by kvm, the secondaries can be released early due to a spurious wakeup so we need the second pen.

Quoted 6 lines · Will Deacon

Quoted 4 lines · Rob Herring

Aren't we mixing 2 pens here? You must have some simple bootloader containing vector table and a pen that the dtb points to, right? The pen you have in the kernel is only needed when hotplug only does a wfi. As you don't yet support hotplug, then you can drop all the kernel pen code.

Quoted 3 lines

Yes, both qemu and kvmtool have bootloader pens outside of the kernel but since wfi is not trapped by kvm, the secondaries can be released early due to a spurious wakeup so we need the second pen.

Actually, KVM traps WFI and puts the vcpu thread on a wait queue (we are actually giving more guaranties than the architecture offers here).

We should be able to remove the loop and rely on WFI.

Quoted 11 lines · Marc Zyngier

Quoted 9 lines · Will Deacon

Quoted 4 lines · Rob Herring

Aren't we mixing 2 pens here? You must have some simple bootloader containing vector table and a pen that the dtb points to, right? The pen you have in the kernel is only needed when hotplug only does a wfi. As you don't yet support hotplug, then you can drop all the kernel pen code.

Yes, both qemu and kvmtool have bootloader pens outside of the kernel but since wfi is not trapped by kvm, the secondaries can be released early due to a spurious wakeup so we need the second pen.

Quoted 2 lines

Actually, KVM traps WFI and puts the vcpu thread on a wait queue (we are actually giving more guaranties than the architecture offers here).

Ok, if we can rely on this behaviour in the future this sounds promising. Can somebody comment from the Xen side of things please?

Hi!

Sorry for taking so long to reply but I admit I missed the thread entirely until now. I would appreciate if you could CC xen-devel at lists.xen.org and/or me for virtualization related discussions.

On the Xen side there are not going to be any problems related to SMP and holding pens, because secondary CPUs only come into existence after the first CPU calls the HYPERVISOR_vcpu_op hypercall (still missing on ARM).

Quoted 1 line · Stefano Stabellini

Hi!

Hi Stefano,

Quoted 4 lines

Sorry for taking so long to reply but I admit I missed the thread entirely until now. I would appreciate if you could CC xen-devel at lists.xen.org and/or me for virtualization related discussions.

I did CC xen-arm at lists.xen.org,

but you've dropped that list in your
reply...

Quoted 4 lines

On the Xen side there are not going to be any problems related to SMP and holding pens, because secondary CPUs only come into existence after the first CPU calls the HYPERVISOR_vcpu_op hypercall (still missing on ARM).

Ok. Would you be willing/able to wrap this in a PSCI interface when you do add SMP support for ARM?

Quoted 10 lines · Will Deacon

Quoted 1 line · Stefano Stabellini

Hi!

Hi Stefano,

Quoted 4 lines

Sorry for taking so long to reply but I admit I missed the thread entirely until now. I would appreciate if you could CC xen-devel at lists.xen.org and/or me for virtualization related discussions.

Quoted 1 line

I did CC xen-arm at lists.xen.org,

Thanks!
I don't know why I didn't receive it then :-\

In any case xen-devel is the right place for upstream related discussion.

Quoted 11 lines · Will Deacon

Quoted 1 line · Stefano Stabellini

Hi!

Hi Stefano,

Quoted 4 lines

Sorry for taking so long to reply but I admit I missed the thread entirely until now. I would appreciate if you could CC xen-devel at lists.xen.org and/or me for virtualization related discussions.

I did CC xen-arm at lists.xen.org,

Thanks!
I don't know why I didn't receive it then :-\

In any case xen-devel is the right place for upstream related discussion.

Quoted 2 lines

but you've dropped that list in your
reply...

Yeah.. that's because I didn't receive the original email and I had to download the mailing list archives, sorry about that.

Quoted 6 lines

Quoted 5 lines

On the Xen side there are not going to be any problems related to SMP and holding pens, because secondary CPUs only come into existence after the first CPU calls the HYPERVISOR_vcpu_op hypercall (still missing on ARM).

Quoted 2 lines

Ok. Would you be willing/able to wrap this in a PSCI interface when you do add SMP support for ARM?

I grepped for PSCI in the Linux tree but I couldn't find anything. Also I googled for ARM PSCI and nothing of value came up. Can I find a doc that explains what PSCI is anywhere?

Quoted 11 lines

Quoted 7 lines

Quoted 4 lines

On the Xen side there are not going to be any problems related to SMP and holding pens, because secondary CPUs only come into existence after the first CPU calls the HYPERVISOR_vcpu_op hypercall (still missing on ARM).

Ok. Would you be willing/able to wrap this in a PSCI interface when you do add SMP support for ARM?

I grepped for PSCI in the Linux tree but I couldn't find anything. Also I googled for ARM PSCI and nothing of value came up. Can I find a doc that explains what PSCI is anywhere?

There is an initial version of the document here (requires registration):

https://silver.arm.com/download/download.tm?pv=1303201

Note that there are some updates due to clarify things a bit in areas such as function ID numbers and the PCS. Marc and I also plan to hack something up in kvm/mach-virt for booting secondary CPUs.

Quoted 16 lines · Will Deacon

Quoted 11 lines

Quoted 7 lines

Quoted 4 lines

On the Xen side there are not going to be any problems related to SMP and holding pens, because secondary CPUs only come into existence after the first CPU calls the HYPERVISOR_vcpu_op hypercall (still missing on ARM).

Ok. Would you be willing/able to wrap this in a PSCI interface when you do add SMP support for ARM?

I grepped for PSCI in the Linux tree but I couldn't find anything. Also I googled for ARM PSCI and nothing of value came up. Can I find a doc that explains what PSCI is anywhere?

There is an initial version of the document here (requires registration):

https://silver.arm.com/download/download.tm?pv=1303201

Quoted 3 lines

Note that there are some updates due to clarify things a bit in areas such as function ID numbers and the PCS. Marc and I also plan to hack something up in kvm/mach-virt for booting secondary CPUs.

Ideally the functionalities would be discoverable via DT, so that PSCI, PSCI-HVC and a paravirtualized Xen interface based on hypercalls and xenbus could all coexist without issues, reusing the same internal Linux APIs.

Quoted 3 lines · Rob Herring

If there is no way to reset the core, then couldn't the hotplug code tear down the cpu setup and just jump back to 0x0 which then returns to the bootloader's pen?

I think hotplug really should be implemented with a virtio device. We just trap back to the emulation and kill the vcpu thread.

Hello,

When running Linux on a para-virtualised platform (that is, one where the guest is aware that it is dealing with virtual devices sitting on things like virtio or xenbus) we require very little in the way of platform code and piggy-backing on top of an existing platform can require a lot of device emulation for very little gain.

These two patches introduce mach-virt: a very simple, DT-based machine which can be used with kvmtool in conjunction with virtio-based devices. It's not hard to imagine the same machine being targetted by Xen, which currently emulates a minimal variant of the vexpress platform.

Note that this patch series depends on the timer rework from Mark Rutland, posted on Friday:

http://lists.infradead.org/pipermail/linux-arm-kernel/2012-November/135651.html

All feedback welcome. We suspect that most controversy will be around the name of the thing :)

Will

Marc Zyngier (2):
ARM: Dummy Virtual Machine platform support
ARM: SMP support for mach-virt

 arch/arm/Kconfig             |   2 +
 arch/arm/Makefile            |   1 +
 arch/arm/mach-virt/Kconfig   |   9 ++
 arch/arm/mach-virt/Makefile  |   6 ++
 arch/arm/mach-virt/headsmp.S |  38 ++++++++
 arch/arm/mach-virt/platsmp.c | 205 +++++++++++++++++++++++++++++++++++++++++++
 arch/arm/mach-virt/virt.c    |  71 +++++++++++++++
 7 files changed, 332 insertions(+)
 create mode 100644 arch/arm/mach-virt/Kconfig
 create mode 100644 arch/arm/mach-virt/Makefile
 create mode 100644 arch/arm/mach-virt/headsmp.S
 create mode 100644 arch/arm/mach-virt/platsmp.c
 create mode 100644 arch/arm/mach-virt/virt.c

Quoted 14 lines · Will Deacon

When running Linux on a para-virtualised platform (that is, one where the guest is aware that it is dealing with virtual devices sitting on things like virtio or xenbus) we require very little in the way of platform code and piggy-backing on top of an existing platform can require a lot of device emulation for very little gain.

These two patches introduce mach-virt: a very simple, DT-based machine which can be used with kvmtool in conjunction with virtio-based devices. It's not hard to imagine the same machine being targetted by Xen, which currently emulates a minimal variant of the vexpress platform.

Note that this patch series depends on the timer rework from Mark Rutland, posted on Friday:

Quoted 4 lines

http://lists.infradead.org/pipermail/linux-arm-kernel/2012-November/135651.html

All feedback welcome. We suspect that most controversy will be around the name of the thing :)

We've discussed this before at conferences. I don't know that we concluded this wasn't needed, but it certainly leaned that direction.

So what has changed? You're not going to save code space because we're building multiple platforms together. You'll save some boot time, but a stripped down dtb with only the minimal peripherals would probably save nearly as much time.

However, I do have concerns with using VExpress as
the guest. For example, you can't support a non-PAE guest with 4GB of RAM on VExpress (maybe if the vexpress code gets all memory map info from DT).

Is this really complete? Will we need reset, poweroff, hotplug, and suspend/resume support for example? Unlike most initial platform submissions which are minimal, I think seeing full support would be useful here. Then we can better gauge how much we are really saving.

Hi Rob,

[fixing Arnd's address, as I apparently can't spell]

Quoted 15 lines · Rob Herring

Quoted 14 lines · Will Deacon

When running Linux on a para-virtualised platform (that is, one where the guest is aware that it is dealing with virtual devices sitting on things like virtio or xenbus) we require very little in the way of platform code and piggy-backing on top of an existing platform can require a lot of device emulation for very little gain.

These two patches introduce mach-virt: a very simple, DT-based machine which can be used with kvmtool in conjunction with virtio-based devices. It's not hard to imagine the same machine being targetted by Xen, which currently emulates a minimal variant of the vexpress platform.

Note that this patch series depends on the timer rework from Mark Rutland, posted on Friday:

Quoted 7 lines

Quoted 4 lines

http://lists.infradead.org/pipermail/linux-arm-kernel/2012-November/135651.html

All feedback welcome. We suspect that most controversy will be around the name of the thing :)

We've discussed this before at conferences. I don't know that we concluded this wasn't needed, but it certainly leaned that direction.

I too leaned that direction before I started looking at kvm in detail and, since then, I've changed my mind when it comes to para-virtualisation.

The reason for this is that there is absolutely no reason to emulate some components of a real platform and then bolt virtual devices onto it once you've got enough to get it going. It leads to a right royal mess in userspace, where you have to write a load of non-reusable emulation code and it leads to churn in the kernel because you're constantly at odds with people trying to develop the platform code based on the actual hardware they have.

With a virtualisation-capable ARMv7 system, all you *need* to boot SMP Debian is:

- A v7 CPU with virt extensions
- vGIC
- architected timers

*everything* else can be described using virtio devices in the device-tree, essentially allowing you to generate platforms based on the above and boot the same kernel on them.

Quoted 4 lines

So what has changed? You're not going to save code space because we're building multiple platforms together. You'll save some boot time, but a stripped down dtb with only the minimal peripherals would probably save nearly as much time.

It's really got nothing to do with code space or boot speed. What it *is* about is avoiding the tight coupling with a real platform and suffering as a result. Yes, you can strip down the DT for a real platform but you'll likely still have to emulate things like the SP804 in order to boot. That's not to mention any platform-specific system register interfaces which are required early on.

We can't even re-use the socfpga code (which is incredibly minimal) without emulating the dw_apb_timer.

That to me is highlighting where we need to do more work on DT driving the initialization. The platforms are still aware of what kind of timers and interrupt controllers are present. They should not be. There's work in progress for both of those.

Lorenzo's DT MPIDR patches should trim down smp code some. The DT spin table code could probably be common. I think I could use it on highbank as well. If we decide the pen code stays, then it should be common rather than creating yet another copy.

Quoted 9 lines · Rob Herring

That to me is highlighting where we need to do more work on DT driving the initialization. The platforms are still aware of what kind of timers and interrupt controllers are present. They should not be. There's work in progress for both of those.

Lorenzo's DT MPIDR patches should trim down smp code some. The DT spin table code could probably be common. I think I could use it on highbank as well. If we decide the pen code stays, then it should be common rather than creating yet another copy.

I don't want to rain on the "everything should be common" parade here. However, for the best part of last year I've been working on kernel support for big.LITTLE systems, and the handling of CPU hotplug (including SMP secondary boot) is far from being a trivial task. Managing the simple bringing up or down of a CPU in such an environment required hundreds of new lines of code. That is far from a simple holding pen or spinning table to say the least.

[ For the curious, I'll post this code here soon for review. ]

So my point of view is: if you do not need a holding pen because you can hold individual CPUs in reset, then don't. Many platforms with support in the kernel can do that, yet they copied the holding pen code just because it is there. And that is total crap.

on the topic of a para-virtualised machine, I think that it should simply implement the PSCI calls to bring up CPUs _without_ any holding pen nor spinning tables. You issue the appropriate PSCI call with the physical address for secondary_startup() as argument and you're done. The host intercepts that call and free a new CPU instance in response. That's all.

Hi Nicolas,

Quoted 6 lines · Nicolas Pitre

on the topic of a para-virtualised machine, I think that it should simply implement the PSCI calls to bring up CPUs _without_ any holding pen nor spinning tables. You issue the appropriate PSCI call with the physical address for secondary_startup() as argument and you're done. The host intercepts that call and free a new CPU instance in response. That's all.

I'd be happy to go with this suggestion if it wasn't for one thing:
platforms that do not implement a secure mode. For these platforms, smc will be an undefined instruction at the exception level where it is executed and therefore cannot be trapped by the hypervisor.

If that situation requires a pen, I see no benefit from having two boot schemes where one of them would work in every case.

Quoted 10 lines · Will Deacon

Hi Nicolas,

Quoted 6 lines · Nicolas Pitre

on the topic of a para-virtualised machine, I think that it should simply implement the PSCI calls to bring up CPUs _without_ any holding pen nor spinning tables. You issue the appropriate PSCI call with the physical address for secondary_startup() as argument and you're done. The host intercepts that call and free a new CPU instance in response. That's all.

Quoted 4 lines

I'd be happy to go with this suggestion if it wasn't for one thing:
platforms that do not implement a secure mode. For these platforms, smc will be an undefined instruction at the exception level where it is executed and therefore cannot be trapped by the hypervisor.

Really? I thought the hypervisor could virtualize SMC calls. Or is that considered a security hazard?

I don't remember all the PSCI spec details, but I think there was some provision for this case i.e. the SMC call could be a HYP call instead. And if that's not in the spec, then it probably should be added and implemented as if it was.

Quoted 2 lines

If that situation requires a pen, I see no benefit from having two boot schemes where one of them would work in every case.

We always have the choice between several schemes in device drivers for example, depending on the hardware generation. Yet we always implement the better scheme for the newest hardware for performance reasons, even if an older one could work in all cases.

A holding pen is a rather stupid scheme. Please let's try to do without it if possible.

Quoted 9 lines · Nicolas Pitre

Quoted 8 lines · Will Deacon

Quoted 6 lines · Nicolas Pitre

on the topic of a para-virtualised machine, I think that it should simply implement the PSCI calls to bring up CPUs _without_ any holding pen nor spinning tables. You issue the appropriate PSCI call with the physical address for secondary_startup() as argument and you're done. The host intercepts that call and free a new CPU instance in response. That's all.

Quoted 7 lines

Quoted 4 lines

I'd be happy to go with this suggestion if it wasn't for one thing:
platforms that do not implement a secure mode. For these platforms, smc will be an undefined instruction at the exception level where it is executed and therefore cannot be trapped by the hypervisor.

Really? I thought the hypervisor could virtualize SMC calls. Or is that considered a security hazard?

If the security extensions aren't implemented, the hypervisor can't trap the smc instruction.

Quoted 4 lines

I don't remember all the PSCI spec details, but I think there was some provision for this case i.e. the SMC call could be a HYP call instead. And if that's not in the spec, then it probably should be added and implemented as if it was.

Well, this depends on the guest taking an undefined instruction exception on the smc, then deciding to issue an hvc instead and *then* having the hypervisor somehow translate that into a PSCI invocation. It could work, but it sounds easy to mess up and relies on the PSCI firmware co-existing with things like kvm.

Quoted 26 lines · Will Deacon

Quoted 9 lines · Nicolas Pitre

Quoted 8 lines · Will Deacon

Quoted 6 lines · Nicolas Pitre

on the topic of a para-virtualised machine, I think that it should simply implement the PSCI calls to bring up CPUs _without_ any holding pen nor spinning tables. You issue the appropriate PSCI call with the physical address for secondary_startup() as argument and you're done. The host intercepts that call and free a new CPU instance in response. That's all.

Quoted 7 lines

Quoted 4 lines

I'd be happy to go with this suggestion if it wasn't for one thing:
platforms that do not implement a secure mode. For these platforms, smc will be an undefined instruction at the exception level where it is executed and therefore cannot be trapped by the hypervisor.

Really? I thought the hypervisor could virtualize SMC calls. Or is that considered a security hazard?

If the security extensions aren't implemented, the hypervisor can't trap the smc instruction.

Quoted 4 lines

I don't remember all the PSCI spec details, but I think there was some provision for this case i.e. the SMC call could be a HYP call instead. And if that's not in the spec, then it probably should be added and implemented as if it was.

Quoted 5 lines

Well, this depends on the guest taking an undefined instruction exception on the smc, then deciding to issue an hvc instead and *then* having the hypervisor somehow translate that into a PSCI invocation. It could work, but it sounds easy to mess up and relies on the PSCI firmware co-existing with things like kvm.

We can have enable-method DT entries independent of the SoC and one of them can be psci-hvc.

Just for clarification, AArch32 with virtualisation mandates the security extensions, so the SMC can be trapped.

On AArch64 it is a bit
tricky since the presence of EL3 is not mandate, in which case SMC would undef (don't as why ;). That's where we can have different enable methods specified via the DT.

Quoted 7 lines · Catalin Marinas

Quoted 5 lines · Will Deacon

Well, this depends on the guest taking an undefined instruction exception on the smc, then deciding to issue an hvc instead and *then* having the hypervisor somehow translate that into a PSCI invocation. It could work, but it sounds easy to mess up and relies on the PSCI firmware co-existing with things like kvm.

Quoted 5 lines

We can have enable-method DT entries independent of the SoC and one of them can be psci-hvc.

Just for clarification, AArch32 with virtualisation mandates the security extensions, so the SMC can be trapped.

Good. Therefore this one is settled.

Quoted 5 lines · Will Deacon

Well, this depends on the guest taking an undefined instruction exception on the smc, then deciding to issue an hvc instead and *then* having the hypervisor somehow translate that into a PSCI invocation. It could work, but it sounds easy to mess up and relies on the PSCI firmware co-existing with things like kvm.

We can have enable-method DT entries independent of the SoC and one of them can be psci-hvc.

Just for clarification, AArch32 with virtualisation mandates the security extensions, so the SMC can be trapped. On AArch64 it is a bit tricky since the presence of EL3 is not mandate, in which case SMC would undef (don't as why ;). That's where we can have different enable methods specified via the DT.

Quoted 5 lines · Catalin Marinas

Just for clarification, AArch32 with virtualisation mandates the security extensions, so the SMC can be trapped. On AArch64 it is a bit tricky since the presence of EL3 is not mandate, in which case SMC would undef (don't as why ;). That's where we can have different enable methods specified via the DT.

Not entirely true: only ARMv7 mandates the security extensions in this manner. You can still have ARMv8 CPUs running AArch32 code without the security extensions.

Quoted 15 lines · Rob Herring

Quoted 14 lines · Will Deacon

When running Linux on a para-virtualised platform (that is, one where the guest is aware that it is dealing with virtual devices sitting on things like virtio or xenbus) we require very little in the way of platform code and piggy-backing on top of an existing platform can require a lot of device emulation for very little gain.

These two patches introduce mach-virt: a very simple, DT-based machine which can be used with kvmtool in conjunction with virtio-based devices. It's not hard to imagine the same machine being targetted by Xen, which currently emulates a minimal variant of the vexpress platform.

Note that this patch series depends on the timer rework from Mark Rutland, posted on Friday:

Quoted 7 lines

Quoted 4 lines

http://lists.infradead.org/pipermail/linux-arm-kernel/2012-November/135651.html

All feedback welcome. We suspect that most controversy will be around the name of the thing :)

We've discussed this before at conferences. I don't know that we concluded this wasn't needed, but it certainly leaned that direction.

I too leaned that direction before I started looking at kvm in detail and, since then, I've changed my mind when it comes to para-virtualisation.

The reason for this is that there is absolutely no reason to emulate some components of a real platform and then bolt virtual devices onto it once you've got enough to get it going. It leads to a right royal mess in userspace, where you have to write a load of non-reusable emulation code and it leads to churn in the kernel because you're constantly at odds with people trying to develop the platform code based on the actual hardware they have.

With a virtualisation-capable ARMv7 system, all you *need* to boot SMP Debian is:

- A v7 CPU with virt extensions
- vGIC
- architected timers

*everything* else can be described using virtio devices in the device-tree, essentially allowing you to generate platforms based on the above and boot the same kernel on them.

Quoted 4 lines

So what has changed? You're not going to save code space because we're building multiple platforms together. You'll save some boot time, but a stripped down dtb with only the minimal peripherals would probably save nearly as much time.

It's really got nothing to do with code space or boot speed. What it *is* about is avoiding the tight coupling with a real platform and suffering as a result. Yes, you can strip down the DT for a real platform but you'll likely still have to emulate things like the SP804 in order to boot. That's not to mention any platform-specific system register interfaces which are required early on.

We can't even re-use the socfpga code (which is incredibly minimal) without emulating the dw_apb_timer.

Quoted 4 lines

However, I do have concerns with using VExpress as
the guest. For example, you can't support a non-PAE guest with 4GB of RAM on VExpress (maybe if the vexpress code gets all memory map info from DT).

Yes, vexpress is even less suitable for this.

Quoted 4 lines

Is this really complete? Will we need reset, poweroff, hotplug, and suspend/resume support for example? Unlike most initial platform submissions which are minimal, I think seeing full support would be useful here. Then we can better gauge how much we are really saving.

The code is complete in the sense that you can boot an SMP guest running Debian with console, network, block etc. etc. but you're right to point out the absence of power-management support.

However, power-management in KVM guests is a *much* larger problem and not one that has been solved adequately as of yet. There are suggestions that it should be handled entirely in firmware, with the guest making smc calls to request power-management operations but this is yet to materialise and, as such, we can't yet use it here.

We could look at building a virtio-based power controller but that's going to come up too late for SMP booting (although will give us hotplug, reset etc).