Skip to contentSkip to conversation

AARCH64 support on machvirt machine model using KVM

PATCH In review Mian M. Hamayun 3 participants 20 messages Sep 27, 2013 – Nov 26, 2013

lore
Share & tools
lore
#0 lore raw

v3 cover AARCH64 support on machvirt machine model using KVM

From: "Mian M. Hamayun" <[email protected]>

This is the v3 of patch series that implements KVM support in QEMU for the ARMv8 Cortex A57 CPU. It depends on the recently mainlined AArch64 preparation patch series and machvirt patches version v7, and uses the already available KVM in-kernel GIC support.
This implementation supports both 64-bit and 32-bit guests on AARCH64.

As a reference, KVM Tool and the AArch64 bootwrapper were used, as well as public documentation from ARM. The following work has been tested with SMP capabilities for both 64 and 32-bit guests, under ARMv8 Fast and Foundation Models (Open Embedded userspace with an emulated MMC).

The v1 of this patch series related to AArch64 CPU model for Versatile Express was sponsored by Huawei, and developed in collaboration between Huawei Technologies Duesseldorf GmbH - European Research Center Munich (ERC) and Virtual Open Systems.

A working tree of this implementation is available on the "kvm-aarch64-v3" branch of the following github repository.

https://github.com/virtualopensystems/qemu/tree/kvm-aarch64-v3

Summary of Changes:

Changes v2 -> v3
* Based on AArch64 Preparation Patchset (mainlined) and machvirt patch version v7
* 32 and 64-bit KVM hooks have been separated into kvm_32.c and kvm_64.c, whereas common code resides in kvm.c
* SMP support is now implemented using PSCI method instead of the boot injection mechanism, as implemented in the previous versions
* 32-bit SMP guest support is now available

Changes v1 -> v2
* Based on AArch64 Preparation Patchset V5 and machvirt patches.
* Implemented for Machvirt Machine Model.
* Architecture-specific CPU initialization code improved. Removed hardcoding from register set/get loops and introduced CPU target type array to find appropriate ARMv8 CPU type supported by KVM.
* Disable the PSCI method in case of AArch64 and use the spin-table method instead for booting secondary CPUs.
* 32-bit guest support still missing

v1
* Based on AArch64 Preparation Patchset V4
* Implemented for Versatile Express Machine Model
* Support for SMP using bootcode injection
* No 32-bit guest support

John Rigby (2):
ARM: arm64 kvm headers from kernel arm64-kvm tree
AARCH64: add a57core

Mian M. Hamayun (9):
AARCH64: Add A57 CPU to default AArch64 configuration and enable KVM
AARCH64: Separate 32-bit specific code from common KVM hooks
AARCH64: Add AARCH64 CPU initialization, get and put registers support
target-arm: Parameterize the bootloader selection and setup mechanism
AARCH64: Add boot support for aarch64 processor
AARCH64: Enable SMP support for aarch64 processors using PSCI method
AARCH64: Enable configure support for 32-bit guests on AARCH64
AARCH64: Add flags and boot parameters for 32-bit guests on AARCH64
AARCH64: Add 32-bit mode selection parameter

 configure                           |   3 +-
 default-configs/aarch64-softmmu.mak |   1 +
 hw/arm/boot.c                       | 147 ++++++++++++--
 hw/arm/virt.c                       |   8 +
 hw/cpu/Makefile.objs                |   1 +
 hw/cpu/a57mpcore.c                  | 122 ++++++++++++
 linux-headers/asm-arm64/kvm.h       | 168 ++++++++++++++++
 linux-headers/asm-arm64/kvm_para.h  |   1 +
 linux-headers/linux/kvm.h           |   1 +
 qemu-options.hx                     |   8 +
 target-arm/Makefile.objs            |   5 +
 target-arm/cpu.c                    |  18 +-
 target-arm/kvm.c                    | 363 ----------------------------------
 target-arm/kvm_32.c                 | 382 ++++++++++++++++++++++++++++++++++++
 target-arm/kvm_64.c                 | 146 ++++++++++++++
 vl.c                                |   4 +
 16 files changed, 997 insertions(+), 381 deletions(-)
 create mode 100644 hw/cpu/a57mpcore.c
 create mode 100644 linux-headers/asm-arm64/kvm.h
 create mode 100644 linux-headers/asm-arm64/kvm_para.h
 create mode 100644 target-arm/kvm_32.c
 create mode 100644 target-arm/kvm_64.c
Signature · 1 line
1.8.1.2
#1 lore raw

PATCH v3 1/11 ARM: arm64 kvm headers from kernel arm64-kvm tree

From: John Rigby <[email protected]>

Also add the KVM_REG_ARM64 register type to linux/kvm header file.

---
 linux-headers/asm-arm64/kvm.h      | 168 +++++++++++++++++++++++++++++++++++++
 linux-headers/asm-arm64/kvm_para.h |   1 +
 linux-headers/linux/kvm.h          |   1 +
 3 files changed, 170 insertions(+)
 create mode 100644 linux-headers/asm-arm64/kvm.h
 create mode 100644 linux-headers/asm-arm64/kvm_para.h
linux-headers/asm-arm64/kvm.h new file +168 -0
new file mode 100644
index 0000000..5031f42
@@ -0,0 +1,168 @@
+/*
+ * Copyright (C) 2012,2013 - ARM Ltd
+ * Author: Marc Zyngier <[email protected]>
+ *
+ * Derived from arch/arm/include/uapi/asm/kvm.h:
+ * Copyright (C) 2012 - Virtual Open Systems and Columbia University
+ * Author: Christoffer Dall <[email protected]>
+ *
+ * This program is free software; you can redistribute it and/or modify
+ * it under the terms of the GNU General Public License version 2 as
+ * published by the Free Software Foundation.
+ *
+ * This program is distributed in the hope that it will be useful,
+ * but WITHOUT ANY WARRANTY; without even the implied warranty of
+ * MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE.  See the
+ * GNU General Public License for more details.
+ *
+ * You should have received a copy of the GNU General Public License
+ * along with this program.  If not, see <http://www.gnu.org/licenses/>.
+ */
+
+#ifndef __ARM_KVM_H__
+#define __ARM_KVM_H__
+
+#define KVM_SPSR_EL1	0
+#define KVM_SPSR_SVC	KVM_SPSR_EL1
+#define KVM_SPSR_ABT	1
+#define KVM_SPSR_UND	2
+#define KVM_SPSR_IRQ	3
+#define KVM_SPSR_FIQ	4
+#define KVM_NR_SPSR	5
+
+#ifndef __ASSEMBLY__
+#include <asm/types.h>
+#include <asm/ptrace.h>
+
+#define __KVM_HAVE_GUEST_DEBUG
+#define __KVM_HAVE_IRQ_LINE
+
+#define KVM_REG_SIZE(id)						\
+	(1U << (((id) & KVM_REG_SIZE_MASK) >> KVM_REG_SIZE_SHIFT))
+
+struct kvm_regs {
+	struct user_pt_regs regs;	/* sp = sp_el0 */
+
+	__u64	sp_el1;
+	__u64	elr_el1;
+
+	__u64	spsr[KVM_NR_SPSR];
+
+	struct user_fpsimd_state fp_regs;
+};
+
+/* Supported Processor Types */
+#define KVM_ARM_TARGET_AEM_V8		0
+#define KVM_ARM_TARGET_FOUNDATION_V8	1
+#define KVM_ARM_TARGET_CORTEX_A57	2
+
+#define KVM_ARM_NUM_TARGETS		3
+
+/* KVM_ARM_SET_DEVICE_ADDR ioctl id encoding */
+#define KVM_ARM_DEVICE_TYPE_SHIFT	0
+#define KVM_ARM_DEVICE_TYPE_MASK	(0xffff << KVM_ARM_DEVICE_TYPE_SHIFT)
+#define KVM_ARM_DEVICE_ID_SHIFT		16
+#define KVM_ARM_DEVICE_ID_MASK		(0xffff << KVM_ARM_DEVICE_ID_SHIFT)
+
+/* Supported device IDs */
+#define KVM_ARM_DEVICE_VGIC_V2		0
+
+/* Supported VGIC address types  */
+#define KVM_VGIC_V2_ADDR_TYPE_DIST	0
+#define KVM_VGIC_V2_ADDR_TYPE_CPU	1
+
+#define KVM_VGIC_V2_DIST_SIZE		0x1000
+#define KVM_VGIC_V2_CPU_SIZE		0x2000
+
+#define KVM_ARM_VCPU_POWER_OFF		0 /* CPU is started in OFF state */
+#define KVM_ARM_VCPU_EL1_32BIT		1 /* CPU running a 32bit VM */
+
+struct kvm_vcpu_init {
+	__u32 target;
+	__u32 features[7];
+};
+
+struct kvm_sregs {
+};
+
+struct kvm_fpu {
+};
+
+struct kvm_guest_debug_arch {
+};
+
+struct kvm_debug_exit_arch {
+};
+
+struct kvm_sync_regs {
+};
+
+struct kvm_arch_memory_slot {
+};
+
+/* If you need to interpret the index values, here is the key: */
+#define KVM_REG_ARM_COPROC_MASK		0x000000000FFF0000
+#define KVM_REG_ARM_COPROC_SHIFT	16
+
+/* Normal registers are mapped as coprocessor 16. */
+#define KVM_REG_ARM_CORE		(0x0010 << KVM_REG_ARM_COPROC_SHIFT)
+#define KVM_REG_ARM_CORE_REG(name)	(offsetof(struct kvm_regs, name) / sizeof(__u32))
+
+/* Some registers need more space to represent values. */
+#define KVM_REG_ARM_DEMUX		(0x0011 << KVM_REG_ARM_COPROC_SHIFT)
+#define KVM_REG_ARM_DEMUX_ID_MASK	0x000000000000FF00
+#define KVM_REG_ARM_DEMUX_ID_SHIFT	8
+#define KVM_REG_ARM_DEMUX_ID_CCSIDR	(0x00 << KVM_REG_ARM_DEMUX_ID_SHIFT)
+#define KVM_REG_ARM_DEMUX_VAL_MASK	0x00000000000000FF
+#define KVM_REG_ARM_DEMUX_VAL_SHIFT	0
+
+/* AArch64 system registers */
+#define KVM_REG_ARM64_SYSREG		(0x0013 << KVM_REG_ARM_COPROC_SHIFT)
+#define KVM_REG_ARM64_SYSREG_OP0_MASK	0x000000000000c000
+#define KVM_REG_ARM64_SYSREG_OP0_SHIFT	14
+#define KVM_REG_ARM64_SYSREG_OP1_MASK	0x0000000000003800
+#define KVM_REG_ARM64_SYSREG_OP1_SHIFT	11
+#define KVM_REG_ARM64_SYSREG_CRN_MASK	0x0000000000000780
+#define KVM_REG_ARM64_SYSREG_CRN_SHIFT	7
+#define KVM_REG_ARM64_SYSREG_CRM_MASK	0x0000000000000078
+#define KVM_REG_ARM64_SYSREG_CRM_SHIFT	3
+#define KVM_REG_ARM64_SYSREG_OP2_MASK	0x0000000000000007
+#define KVM_REG_ARM64_SYSREG_OP2_SHIFT	0
+
+/* KVM_IRQ_LINE irq field index values */
+#define KVM_ARM_IRQ_TYPE_SHIFT		24
+#define KVM_ARM_IRQ_TYPE_MASK		0xff
+#define KVM_ARM_IRQ_VCPU_SHIFT		16
+#define KVM_ARM_IRQ_VCPU_MASK		0xff
+#define KVM_ARM_IRQ_NUM_SHIFT		0
+#define KVM_ARM_IRQ_NUM_MASK		0xffff
+
+/* irq_type field */
+#define KVM_ARM_IRQ_TYPE_CPU		0
+#define KVM_ARM_IRQ_TYPE_SPI		1
+#define KVM_ARM_IRQ_TYPE_PPI		2
+
+/* out-of-kernel GIC cpu interrupt injection irq_number field */
+#define KVM_ARM_IRQ_CPU_IRQ		0
+#define KVM_ARM_IRQ_CPU_FIQ		1
+
+/* Highest supported SPI, from VGIC_NR_IRQS */
+#define KVM_ARM_IRQ_GIC_MAX		127
+
+/* PSCI interface */
+#define KVM_PSCI_FN_BASE		0x95c1ba5e
+#define KVM_PSCI_FN(n)			(KVM_PSCI_FN_BASE + (n))
+
+#define KVM_PSCI_FN_CPU_SUSPEND		KVM_PSCI_FN(0)
+#define KVM_PSCI_FN_CPU_OFF		KVM_PSCI_FN(1)
+#define KVM_PSCI_FN_CPU_ON		KVM_PSCI_FN(2)
+#define KVM_PSCI_FN_MIGRATE		KVM_PSCI_FN(3)
+
+#define KVM_PSCI_RET_SUCCESS		0
+#define KVM_PSCI_RET_NI			((unsigned long)-1)
+#define KVM_PSCI_RET_INVAL		((unsigned long)-2)
+#define KVM_PSCI_RET_DENIED		((unsigned long)-3)
+
+#endif
+
+#endif /* __ARM_KVM_H__ */
linux-headers/asm-arm64/kvm_para.h new file +1 -0
new file mode 100644
index 0000000..14fab8f
@@ -0,0 +1 @@
+#include <asm-generic/kvm_para.h>
linux-headers/linux/kvm.h +1 -0
index c614070..4200779 100644
@@ -783,6 +783,7 @@ struct kvm_dirty_tlb {
 #define KVM_REG_IA64		0x3000000000000000ULL
 #define KVM_REG_ARM		0x4000000000000000ULL
 #define KVM_REG_S390		0x5000000000000000ULL
+#define KVM_REG_ARM64		0x6000000000000000ULL
 
 #define KVM_REG_SIZE_SHIFT	52
 #define KVM_REG_SIZE_MASK	0x00f0000000000000ULL
Signature · 1 line
1.8.1.2
#2 lore raw

PATCH v3 2/11 AARCH64: add a57core

In reply to Mian M. Hamayun

From: John Rigby <[email protected]>

Just an copy of a15 with a57 substituting a15 for now.

---
 hw/cpu/Makefile.objs |   1 +
 hw/cpu/a57mpcore.c   | 122 +++++++++++++++++++++++++++++++++++++++++++++++++++
 target-arm/cpu.c     |   9 ++++
 3 files changed, 132 insertions(+)
 create mode 100644 hw/cpu/a57mpcore.c
hw/cpu/Makefile.objs +1 -0
index df287c1..22e9567 100644
@@ -1,5 +1,6 @@
 obj-$(CONFIG_ARM11MPCORE) += arm11mpcore.o
 obj-$(CONFIG_A9MPCORE) += a9mpcore.o
 obj-$(CONFIG_A15MPCORE) += a15mpcore.o
+obj-$(CONFIG_A57MPCORE) += a57mpcore.o
 obj-$(CONFIG_ICC_BUS) += icc_bus.o
 
hw/cpu/a57mpcore.c new file +122 -0

122 lines not on this page · raw message

target-arm/cpu.c +9 -0
index d40f2a7..5d811b9 100644
@@ -684,6 +684,14 @@ static void cortex_a15_initfn(Object *obj)
     define_arm_cp_regs(cpu, cortexa15_cp_reginfo);
 }
 
+static void cortex_a57_initfn(Object *obj)
+{
+    ARMCPU *cpu = ARM_CPU(obj);
+    set_feature(&cpu->env, ARM_FEATURE_V8);
+    set_feature(&cpu->env, ARM_FEATURE_AARCH64);
+    /* TODO: See if we need to set some more features for ARMv8 ? */
+}
+
 static void ti925t_initfn(Object *obj)
 {
     ARMCPU *cpu = ARM_CPU(obj);
@@ -872,6 +880,7 @@ static const ARMCPUInfo arm_cpus[] = {
     { .name = "cortex-a8",   .initfn = cortex_a8_initfn },
     { .name = "cortex-a9",   .initfn = cortex_a9_initfn },
     { .name = "cortex-a15",  .initfn = cortex_a15_initfn },
+    { .name = "cortex-a57",  .initfn = cortex_a57_initfn },
     { .name = "ti925t",      .initfn = ti925t_initfn },
     { .name = "sa1100",      .initfn = sa1100_initfn },
     { .name = "sa1110",      .initfn = sa1110_initfn },
Signature · 1 line
1.8.1.2
#3 lore raw

Re: [Qemu-devel] [PATCH v3 02/11] AARCH64: add a57core

Hi,

Am 27.09.2013 12:10, schrieb Mian M. Hamayun:

Quoted 4 lines

From: John Rigby <[email protected]>

Just an copy of a15 with a57 substituting a15 for now.

Quoted 8 lines
---
 hw/cpu/Makefile.objs |   1 +
 hw/cpu/a57mpcore.c   | 122 +++++++++++++++++++++++++++++++++++++++++++++++++++
 target-arm/cpu.c     |   9 ++++
 3 files changed, 132 insertions(+)
 create mode 100644 hw/cpu/a57mpcore.c

I had previously gently nack'ed this patch - conversions are still queued on qom-next currently though. Having PMM's kernel load patches now, I'll try if I can still get this posted today.

In particular use of QOM realize and embedding of child device with avoidance of child instantiation during realization.

Regards,
Andreas

Quoted 168 lines
hw/cpu/Makefile.objs +1 -0
index df287c1..22e9567 100644
@@ -1,5 +1,6 @@
 obj-$(CONFIG_ARM11MPCORE) += arm11mpcore.o
 obj-$(CONFIG_A9MPCORE) += a9mpcore.o
 obj-$(CONFIG_A15MPCORE) += a15mpcore.o
+obj-$(CONFIG_A57MPCORE) += a57mpcore.o
 obj-$(CONFIG_ICC_BUS) += icc_bus.o
 
hw/cpu/a57mpcore.c new file +122 -0
new file mode 100644
index 0000000..4be277f
@@ -0,0 +1,122 @@
+/*
+ * Cortex-A57MPCore internal peripheral emulation.
+ *
+ * Copyright (c) 2012 Linaro Limited.
+ * Written by Peter Maydell.
+ *
+ * This program is free software; you can redistribute it and/or modify
+ * it under the terms of the GNU General Public License as published by
+ * the Free Software Foundation; either version 2 of the License, or
+ * (at your option) any later version.
+ *
+ * This program is distributed in the hope that it will be useful,
+ * but WITHOUT ANY WARRANTY; without even the implied warranty of
+ * MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE.  See the
+ * GNU General Public License for more details.
+ *
+ * You should have received a copy of the GNU General Public License along
+ * with this program; if not, see <http://www.gnu.org/licenses/>.
+ */
+
+#include "hw/sysbus.h"
+#include "sysemu/kvm.h"
+
+/* A57MP private memory region.  */
+
+#define TYPE_A57MPCORE_PRIV "a57mpcore_priv"
+#define A57MPCORE_PRIV(obj) \
+    OBJECT_CHECK(A57MPPrivState, (obj), TYPE_A57MPCORE_PRIV)
+
+typedef struct A57MPPrivState {
+    /*< private >*/
+    SysBusDevice parent_obj;
+    /*< public >*/
+
+    uint32_t num_cpu;
+    uint32_t num_irq;
+    MemoryRegion container;
+    DeviceState *gic;
+} A57MPPrivState;
+
+static void a57mp_priv_set_irq(void *opaque, int irq, int level)
+{
+    A57MPPrivState *s = (A57MPPrivState *)opaque;
+    qemu_set_irq(qdev_get_gpio_in(s->gic, irq), level);
+}
+
+static int a57mp_priv_init(SysBusDevice *dev)
+{
+    A57MPPrivState *s = A57MPCORE_PRIV(dev);
+    SysBusDevice *busdev;
+    const char *gictype = "arm_gic";
+
+    if (kvm_irqchip_in_kernel()) {
+        gictype = "kvm-arm-gic";
+    }
+
+    s->gic = qdev_create(NULL, gictype);
+    qdev_prop_set_uint32(s->gic, "num-cpu", s->num_cpu);
+    qdev_prop_set_uint32(s->gic, "num-irq", s->num_irq);
+    qdev_prop_set_uint32(s->gic, "revision", 2);
+    qdev_init_nofail(s->gic);
+    busdev = SYS_BUS_DEVICE(s->gic);
+
+    /* Pass through outbound IRQ lines from the GIC */
+    sysbus_pass_irq(dev, busdev);
+
+    /* Pass through inbound GPIO lines to the GIC */
+    qdev_init_gpio_in(DEVICE(dev), a57mp_priv_set_irq, s->num_irq - 32);
+
+    /* Memory map (addresses are offsets from PERIPHBASE):
+     *  0x0000-0x0fff -- reserved
+     *  0x1000-0x1fff -- GIC Distributor
+     *  0x2000-0x2fff -- GIC CPU interface
+     *  0x4000-0x4fff -- GIC virtual interface control (not modelled)
+     *  0x5000-0x5fff -- GIC virtual interface control (not modelled)
+     *  0x6000-0x7fff -- GIC virtual CPU interface (not modelled)
+     */
+    memory_region_init(&s->container, OBJECT(s),
+                       "a57mp-priv-container", 0x8000);
+    memory_region_add_subregion(&s->container, 0x1000,
+                                sysbus_mmio_get_region(busdev, 0));
+    memory_region_add_subregion(&s->container, 0x2000,
+                                sysbus_mmio_get_region(busdev, 1));
+
+    sysbus_init_mmio(dev, &s->container);
+    return 0;
+}
+
+static Property a57mp_priv_properties[] = {
+    DEFINE_PROP_UINT32("num-cpu", A57MPPrivState, num_cpu, 1),
+    /* The Cortex-A57MP may have anything from 0 to 224 external interrupt
+     * IRQ lines (with another 32 internal). We default to 128+32, which
+     * is the number provided by the Cortex-A57MP test chip in the
+     * Versatile Express A57 development board.
+     * Other boards may differ and should set this property appropriately.
+     */
+    DEFINE_PROP_UINT32("num-irq", A57MPPrivState, num_irq, 160),
+    DEFINE_PROP_END_OF_LIST(),
+};
+
+static void a57mp_priv_class_init(ObjectClass *klass, void *data)
+{
+    DeviceClass *dc = DEVICE_CLASS(klass);
+    SysBusDeviceClass *k = SYS_BUS_DEVICE_CLASS(klass);
+    k->init = a57mp_priv_init;
+    dc->props = a57mp_priv_properties;
+    /* We currently have no savable state */
+}
+
+static const TypeInfo a57mp_priv_info = {
+    .name  = TYPE_A57MPCORE_PRIV,
+    .parent = TYPE_SYS_BUS_DEVICE,
+    .instance_size  = sizeof(A57MPPrivState),
+    .class_init = a57mp_priv_class_init,
+};
+
+static void a57mp_register_types(void)
+{
+    type_register_static(&a57mp_priv_info);
+}
+
+type_init(a57mp_register_types)
target-arm/cpu.c +9 -0
index d40f2a7..5d811b9 100644
@@ -684,6 +684,14 @@ static void cortex_a15_initfn(Object *obj)
     define_arm_cp_regs(cpu, cortexa15_cp_reginfo);
 }
 
+static void cortex_a57_initfn(Object *obj)
+{
+    ARMCPU *cpu = ARM_CPU(obj);
+    set_feature(&cpu->env, ARM_FEATURE_V8);
+    set_feature(&cpu->env, ARM_FEATURE_AARCH64);
+    /* TODO: See if we need to set some more features for ARMv8 ? */
+}
+
 static void ti925t_initfn(Object *obj)
 {
     ARMCPU *cpu = ARM_CPU(obj);
@@ -872,6 +880,7 @@ static const ARMCPUInfo arm_cpus[] = {
     { .name = "cortex-a8",   .initfn = cortex_a8_initfn },
     { .name = "cortex-a9",   .initfn = cortex_a9_initfn },
     { .name = "cortex-a15",  .initfn = cortex_a15_initfn },
+    { .name = "cortex-a57",  .initfn = cortex_a57_initfn },
     { .name = "ti925t",      .initfn = ti925t_initfn },
     { .name = "sa1100",      .initfn = sa1100_initfn },
     { .name = "sa1110",      .initfn = sa1110_initfn },
Signature · 2 lines
SUSE LINUX Products GmbH, Maxfeldstr. 5, 90409 Nürnberg, Germany
GF: Jeff Hawn, Jennifer Guild, Felix Imendörffer; HRB 16746 AG Nürnberg
#4 lore raw

Re: [Qemu-devel] [PATCH v3 02/11] AARCH64: add a57core

Quoted 10 lines · Andreas Färber

Hi,

Am 27.09.2013 12:10, schrieb Mian M. Hamayun:

Quoted 3 lines

From: John Rigby <[email protected]>

Just an copy of a15 with a57 substituting a15 for now.

I had previously gently nack'ed this patch - conversions are still queued on qom-next currently though. Having PMM's kernel load patches now, I'll try if I can still get this posted today.

It's also already had a strong nak from me because it's not really an A57 model. We should be using 'cpu host' until there's a public TRM for a real A57 model.

-- PMM

#5 lore raw

Re: [Qemu-devel] [PATCH v3 02/11] AARCH64: add a57core

Quoted 7 lines · Peter Maydell
Quoted 6 lines · Andreas Färber

Hi,

Am 27.09.2013 12:10, schrieb Mian M. Hamayun:

Quoted 3 lines

From: John Rigby <[email protected]>

Just an copy of a15 with a57 substituting a15 for now.

Quoted 8 lines
Quoted 3 lines

I had previously gently nack'ed this patch - conversions are still queued on qom-next currently though. Having PMM's kernel load patches now, I'll try if I can still get this posted today.

It's also already had a strong nak from me because it's not really an A57 model. We should be using 'cpu host' until there's a public TRM for a real A57 model.

-- PMM

I agree with your concerns and feedback but as I remember from our last KVM/ARM telco, we agreed that I should not wait for the -cpu host support to become available and post the next patch series anyways.

Also there are multiple patches under review for providing the -cpu host or equivalent support in QEMU and/or KVM such as the one posted by Peter Maydell and another CPU=Host patch from Anup Patel. We will update our patch series, once we have consensus on which implementation to actually use for providing the -cpu host support.

--
Hamayun

#6 lore raw

Re: [Qemu-devel] [PATCH v3 02/11] AARCH64: add a57core

Am 30.09.2013 17:53, schrieb Mian M. Hamayun:

Quoted 9 lines · Peter Maydell
Quoted 8 lines · Andreas Färber
Quoted 7 lines

Am 27.09.2013 12:10, schrieb Mian M. Hamayun:

Quoted 3 lines

From: John Rigby <[email protected]>

Just an copy of a15 with a57 substituting a15 for now.

I had previously gently nack'ed this patch - conversions are still queued on qom-next currently though. Having PMM's kernel load patches now, I'll try if I can still get this posted today.

Quoted 7 lines
Quoted 3 lines

It's also already had a strong nak from me because it's not really an A57 model. We should be using 'cpu host' until there's a public TRM for a real A57 model.

I agree with your concerns and feedback but as I remember from our last KVM/ARM telco, we agreed that I should not wait for the -cpu host support to become available and post the next patch series anyways.

In such a case please use "RFC" rather than "PATCH" to clarify that it is known not yet ready for applying. :-)

Andreas

Quoted 9 lines

Also there are multiple patches under review for providing the -cpu host or equivalent support in QEMU and/or KVM such as the one posted by Peter Maydell and another CPU=Host patch from Anup Patel. We will update our patch
series, once we have consensus on which implementation to actually use for providing the -cpu host support.

Signature · 1 line
Hamayun
Signature · 2 lines
SUSE LINUX Products GmbH, Maxfeldstr. 5, 90409 Nürnberg, Germany
GF: Jeff Hawn, Jennifer Guild, Felix Imendörffer; HRB 16746 AG Nürnberg
#7 lore raw

Re: [Qemu-devel] [PATCH v3 02/11] AARCH64: add a57core

In reply to Mian M. Hamayun
Quoted 3 lines · Mian M. Hamayun

Also there are multiple patches under review for providing the -cpu host or equivalent support in QEMU and/or KVM such as the one posted by Peter Maydell and another CPU=Host patch from Anup Patel.

Yeah, I agree it's a bit up in the air at the moment API wise but I think we've more or less converged on a decision.

https://git.linaro.org/gitweb?p=people/pmaydell/qemu-arm.git;a=shortlog;h=refs/heads/aarch64-kvm
has the QEMU side stuff as I currently have it, and it doesn't depend on the kernel patches being committed.

-- PMM

#8 lore raw

PATCH v3 3/11 AARCH64: Add A57 CPU to default AArch64 configuration and enable KVM

In reply to Mian M. Hamayun

From: "Mian M. Hamayun" <[email protected]>

Introduce the A57 cpu to the default AArch64 configuration and enable KVM for 64-bit guests only.

---
 configure                           | 2 +-
 default-configs/aarch64-softmmu.mak | 1 +
 2 files changed, 2 insertions(+), 1 deletion(-)
configure +1 -1
index 2b83936..7298b69 100755
@@ -4481,7 +4481,7 @@ case "$target_name" in
   *)
 esac
 case "$target_name" in
-  arm|i386|x86_64|ppcemb|ppc|ppc64|s390x)
+  arm|aarch64|i386|x86_64|ppcemb|ppc|ppc64|s390x)
     # Make sure the target and host cpus are compatible
     if test "$kvm" = "yes" -a "$target_softmmu" = "yes" -a \
       \( "$target_name" = "$cpu" -o \
default-configs/aarch64-softmmu.mak +1 -0
index 175362f..0eb3d96 100644
@@ -37,6 +37,7 @@ CONFIG_USB_MUSB=y
 CONFIG_ARM11MPCORE=y
 CONFIG_A9MPCORE=y
 CONFIG_A15MPCORE=y
+CONFIG_A57MPCORE=y
 
 CONFIG_ARM_GIC=y
 CONFIG_ARM_GIC_KVM=$(CONFIG_KVM)
Signature · 1 line
1.8.1.2
#9 lore raw

PATCH v3 4/11 AARCH64: Separate 32-bit specific code from common KVM hooks

In reply to Mian M. Hamayun

From: "Mian M. Hamayun" <[email protected]>

This commit separates the 32-bit (ARMv7) specific KVM hooks from the common code. It also adds the stub functions for 64-bit (ARMv8).

The makefile objects are also tweaked accordingly to compile code either of ARMv7 or ARMv8 depending on the AARCH64 variable.

---
 target-arm/Makefile.objs |   5 +
 target-arm/kvm.c         | 363 --------------------------------------------
 target-arm/kvm_32.c      | 382 +++++++++++++++++++++++++++++++++++++++++++++++
 target-arm/kvm_64.c      |  39 +++++
 4 files changed, 426 insertions(+), 363 deletions(-)
 create mode 100644 target-arm/kvm_32.c
 create mode 100644 target-arm/kvm_64.c
target-arm/Makefile.objs +5 -0
index 6453f5c..4b37d29 100644
@@ -1,6 +1,11 @@
 obj-y += arm-semi.o
 obj-$(CONFIG_SOFTMMU) += machine.o
 obj-$(CONFIG_KVM) += kvm.o
+ifeq ($(TARGET_AARCH64),y)
+obj-$(CONFIG_KVM) += kvm_64.o
+else
+obj-$(CONFIG_KVM) += kvm_32.o
+endif
 obj-$(CONFIG_NO_KVM) += kvm-stub.o
 obj-y += translate.o op_helper.o helper.o cpu.o
 obj-y += neon_helper.o iwmmxt_helper.o
target-arm/kvm.c +0 -363

381 lines not on this page · raw message

target-arm/kvm_32.c new file +382 -0

382 lines not on this page · raw message

target-arm/kvm_64.c new file +39 -0
new file mode 100644
index 0000000..a6af968
@@ -0,0 +1,39 @@
+/*
+ * ARM implementation of KVM hooks for AARCH64
+ *
+ * Copyright Christoffer Dall 2009-2010
+ *           Mian-M. Hamayun 2013, Virtual Open Systems
+ *
+ * This work is licensed under the terms of the GNU GPL, version 2 or later.
+ * See the COPYING file in the top-level directory.
+ *
+ */
+
+#include <stdio.h>
+#include <sys/types.h>
+
+#include <linux/kvm.h>
+
+#include "qemu-common.h"
+#include "sysemu/kvm.h"
+#include "kvm_arm.h"
+
+int kvm_arch_init_vcpu(CPUState *cs)
+{
+    return 0;
+}
+
+int kvm_arch_put_registers(CPUState *cs, int level)
+{
+    return 0;
+}
+
+int kvm_arch_get_registers(CPUState *cs)
+{
+    return 0;
+}
+
+void kvm_arch_reset_vcpu(CPUState *cs)
+{
+}
+
Signature · 1 line
1.8.1.2
#10 lore raw

PATCH v3 5/11 AARCH64: Add AARCH64 CPU initialization, get and put registers support

In reply to Mian M. Hamayun

From: "Mian M. Hamayun" <[email protected]>

The cpu init function tries to initialize with all possible cpu types, as KVM does not provide a means to detect the real cpu type and simply refuses to initialize on cpu type mis-match. By using the loop based init function, we avoid the need to modify code if the underlying platform is different, such as Fast Models instead of Foundation Models.

Get and Put Registers deal with the basic state of AARCH64 CPUs.

---
 target-arm/kvm_64.c | 106 ++++++++++++++++++++++++++++++++++++++++++++++++++--
 1 file changed, 103 insertions(+), 3 deletions(-)
target-arm/kvm_64.c +103 -3
index a6af968..9685727 100644
@@ -18,19 +18,119 @@
 #include "sysemu/kvm.h"
 #include "kvm_arm.h"
 
+#define AARCH64_CORE_REG(x) (KVM_REG_ARM64 | KVM_REG_SIZE_U64 | \
+                            KVM_REG_ARM_CORE | KVM_REG_ARM_CORE_REG(x))
+
+static uint32_t kvm_arm_targets[KVM_ARM_NUM_TARGETS] = {
+    KVM_ARM_TARGET_AEM_V8,
+    KVM_ARM_TARGET_FOUNDATION_V8,
+    KVM_ARM_TARGET_CORTEX_A57
+};
+
 int kvm_arch_init_vcpu(CPUState *cs)
 {
-    return 0;
+    struct kvm_vcpu_init init;
+    int ret, i;
+
+    memset(init.features, 0, sizeof(init.features));
+    /* Find an appropriate target CPU type.
+     * KVM does not provide means to detect the host CPU type on aarch64,
+     * and simply refuses to initialize, if the CPU type mis-matches;
+     * so we try each possible CPU type on aarch64 before giving up! */
+    for (i = 0; i < KVM_ARM_NUM_TARGETS; ++i) {
+        init.target = kvm_arm_targets[i];
+        ret = kvm_vcpu_ioctl(cs, KVM_ARM_VCPU_INIT, &init);
+        if (!ret)
+            break;
+    }
+
+    return ret;
 }
 
 int kvm_arch_put_registers(CPUState *cs, int level)
 {
-    return 0;
+    struct kvm_one_reg reg;
+    int i;
+    int ret;
+
+    ARMCPU *cpu = ARM_CPU(cs);
+    CPUARMState *env = &cpu->env;
+
+    for (i = 0; i < ARRAY_SIZE(env->xregs); i++) {
+        reg.id = AARCH64_CORE_REG(regs.regs[i]);
+        reg.addr = (uintptr_t) &env->xregs[i];
+        ret = kvm_vcpu_ioctl(cs, KVM_SET_ONE_REG, &reg);
+        if (ret) {
+            return ret;
+        }
+    }
+
+    reg.id = AARCH64_CORE_REG(regs.sp);
+    reg.addr = (uintptr_t) &env->xregs[31];
+    ret = kvm_vcpu_ioctl(cs, KVM_SET_ONE_REG, &reg);
+    if (ret) {
+        return ret;
+    }
+
+    reg.id = AARCH64_CORE_REG(regs.pstate);
+    reg.addr = (uintptr_t) &env->pstate;
+    ret = kvm_vcpu_ioctl(cs, KVM_SET_ONE_REG, &reg);
+    if (ret) {
+        return ret;
+    }
+
+    reg.id = AARCH64_CORE_REG(regs.pc);
+    reg.addr = (uintptr_t) &env->pc;
+    ret = kvm_vcpu_ioctl(cs, KVM_SET_ONE_REG, &reg);
+    if (ret) {
+        return ret;
+    }
+
+    /* TODO: Set Rest of Registers */
+    return ret;
 }
 
 int kvm_arch_get_registers(CPUState *cs)
 {
-    return 0;
+    struct kvm_one_reg reg;
+    int i;
+    int ret;
+
+    ARMCPU *cpu = ARM_CPU(cs);
+    CPUARMState *env = &cpu->env;
+
+    for (i = 0; i < ARRAY_SIZE(env->xregs); i++) {
+        reg.id = AARCH64_CORE_REG(regs.regs[i]);
+        reg.addr = (uintptr_t) &env->xregs[i];
+        ret = kvm_vcpu_ioctl(cs, KVM_GET_ONE_REG, &reg);
+        if (ret) {
+            return ret;
+        }
+    }
+
+    reg.id = AARCH64_CORE_REG(regs.sp);
+    reg.addr = (uintptr_t) &env->xregs[31];
+    ret = kvm_vcpu_ioctl(cs, KVM_GET_ONE_REG, &reg);
+    if (ret) {
+        return ret;
+    }
+
+    reg.id = AARCH64_CORE_REG(regs.pstate);
+    reg.addr = (uintptr_t) &env->pstate;
+    ret = kvm_vcpu_ioctl(cs, KVM_GET_ONE_REG, &reg);
+    if (ret) {
+        return ret;
+    }
+
+    reg.id = AARCH64_CORE_REG(regs.pc);
+    reg.addr = (uintptr_t) &env->pc;
+    ret = kvm_vcpu_ioctl(cs, KVM_GET_ONE_REG, &reg);
+    if (ret) {
+        return ret;
+    }
+
+    /* TODO: Set Rest of Registers */
+    return ret;
 }
 
 void kvm_arch_reset_vcpu(CPUState *cs)
Signature · 1 line
1.8.1.2
#11 lore raw

Re: [Qemu-devel] [PATCH v3 05/11] AARCH64: Add AARCH64 CPU initialization, get and put registers support

Quoted 9 lines · Mian M. Hamayun

From: "Mian M. Hamayun" <[email protected]>

The cpu init function tries to initialize with all possible cpu types, as KVM does not provide a means to detect the real cpu type and simply refuses to initialize on cpu type mis-match. By using the loop based init function, we avoid the need to modify code if the underlying platform is different, such as Fast Models instead of Foundation Models.

Get and Put Registers deal with the basic state of AARCH64 CPUs.

Hi; this is another review mail mostly to flag up a few things I'm fixing up as I go along with these patches.

target-arm/kvm_64.c line 59

Quoted 11 lines

int kvm_arch_put_registers(CPUState *cs, int level) {

target-arm/kvm_64.c +8 -1
-    return 0;
+    struct kvm_one_reg reg;
+    int i;
+    int ret;
+
+    ARMCPU *cpu = ARM_CPU(cs);
+    CPUARMState *env = &cpu->env;
+
+    for (i = 0; i < ARRAY_SIZE(env->xregs); i++) {

This upper bound using ARRAY_SIZE means we'll try to do a read/write of xregs[31], which isn't a core register (it's SP for QEMU, as you can see below).

Quoted 7 lines
target-arm/kvm_64.c +7 -0
+        reg.id = AARCH64_CORE_REG(regs.regs[i]);
+        reg.addr = (uintptr_t) &env->xregs[i];
+        ret = kvm_vcpu_ioctl(cs, KVM_SET_ONE_REG, &reg);
+        if (ret) {
+            return ret;
+        }
+    }

target-arm/kvm_64.c line 119

Quoted 10 lines
target-arm/kvm_64.c +9 -0
+
+    reg.id = AARCH64_CORE_REG(regs.sp);
+    reg.addr = (uintptr_t) &env->xregs[31];
+    ret = kvm_vcpu_ioctl(cs, KVM_SET_ONE_REG, &reg);
+    if (ret) {
+        return ret;
+    }
 
+    reg.id = AARCH64_CORE_REG(regs.pstate);
+    reg.addr = (uintptr_t) &env->pstate;

This will tell the kernel to read/write 64 bits of data into env->pstate, which is a uint32_t. We need to use a local uint64_t to pass the data through.

(I don't know why the kernel thinks PSTATE is a 64 bit register...)

thanks
-- PMM

#12 lore raw

PATCH v3 6/11 target-arm: Parameterize the bootloader selection and setup mechanism

In reply to Mian M. Hamayun

From: "Mian M. Hamayun" <[email protected]>

This commit replaces the constant indices used in bootloaders, such as for specifying the Board ID and kernel arguments with variable parameters. This change is used as mechanism to minimize code changes for different bootloaders, for example different bootloaders will be used for different architectures (ARMv7 vs. ARMv8).

Similary pointers are introduced to select appropriate bootloaders for boot and secondary cpus.

---
 hw/arm/boot.c | 81 ++++++++++++++++++++++++++++++++++++++++++++++++-----------
 1 file changed, 66 insertions(+), 15 deletions(-)
hw/arm/boot.c +66 -15
index 967397b..4c1170e 100644
@@ -18,10 +18,9 @@
 #include "qemu/config-file.h"
 
 #define KERNEL_ARGS_ADDR 0x100
-#define KERNEL_LOAD_ADDR 0x00010000
 
 /* The worlds second smallest bootloader.  Set r0-r2, then jump to kernel.  */
-static uint32_t bootloader[] = {
+static uint32_t bootloader_arm32[] = {
   0xe3a00000, /* mov     r0, #0 */
   0xe59f1004, /* ldr     r1, [pc, #4] */
   0xe59f2004, /* ldr     r2, [pc, #4] */
@@ -48,7 +47,7 @@ static uint32_t bootloader[] = {
 #define DSB_INSN 0xf57ff04f
 #define CP15_DSB_INSN 0xee070f9a /* mcr cp15, 0, r0, c7, c10, 4 */
 
-static uint32_t smpboot[] = {
+static uint32_t smpboot_arm32[] = {
   0xe59f2028, /* ldr r2, gic_cpu_if */
   0xe59f0028, /* ldr r0, startaddr */
   0xe3a01001, /* mov r1, #1 */
@@ -65,13 +64,60 @@ static uint32_t smpboot[] = {
   0           /* bootreg: Boot register address is held here */
 };
 
+/*
+ * The bootloaders to be used are referenced by the following pointers
+ * An appropriate bootloader is selected depending on the architecture
+ * i.e. ARMv7, ARMv8 (AARCH64 and AARCH32)
+ */
+static uint32_t *bootloader = NULL;
+static uint32_t  bootloader_array_size = 0;
+
+static uint32_t *smpboot = NULL;
+static uint32_t  smpboot_array_size = 0;
+
+/*
+ * An index gives the location in the bootloader array, where we put the board
+ * ID, kernel arguments and kernel entry addresses. These are different for
+ * ARMv7 and ARMv8 bootloaders defined above.
+ */
+static uint32_t kernel_boardid_index = 0;
+static uint32_t kernel_args_index    = 0;
+static uint32_t kernel_entry_index   = 0;
+
+/*
+ * Similarly, the kernel loading address also depends on the architecture,
+ * i.e. its different for ARMv7, ARMv8 (AARCH64 and AARCH32)
+ */
+static uint32_t kernel_load_addr = 0x0;
+
+static void setup_boot_env_32(void)
+{
+    bootloader = bootloader_arm32;
+    bootloader_array_size = ARRAY_SIZE(bootloader_arm32);
+    smpboot = smpboot_arm32;
+    smpboot_array_size = ARRAY_SIZE(smpboot_arm32);
+
+    kernel_boardid_index = bootloader_array_size - 3;
+    kernel_args_index    = bootloader_array_size - 2;
+    kernel_entry_index   = bootloader_array_size - 1;
+    return;
+}
+
+static void setup_boot_env(ARMCPU *cpu)
+{
+    /* ARMv7 */
+    kernel_load_addr = 0x00010000;
+    setup_boot_env_32();
+    return;
+}
+
 static void default_write_secondary(ARMCPU *cpu,
                                     const struct arm_boot_info *info)
 {
     int n;
-    smpboot[ARRAY_SIZE(smpboot) - 1] = info->smp_bootreg_addr;
-    smpboot[ARRAY_SIZE(smpboot) - 2] = info->gic_cpu_if_addr;
-    for (n = 0; n < ARRAY_SIZE(smpboot); n++) {
+    smpboot[smpboot_array_size - 1] = info->smp_bootreg_addr;
+    smpboot[smpboot_array_size - 2] = info->gic_cpu_if_addr;
+    for (n = 0; n < smpboot_array_size; n++) {
         /* Replace DSB with the pre-v7 DSB if necessary. */
         if (!arm_feature(&cpu->env, ARM_FEATURE_V7) &&
             smpboot[n] == DSB_INSN) {
@@ -79,7 +125,8 @@ static void default_write_secondary(ARMCPU *cpu,
         }
         smpboot[n] = tswap32(smpboot[n]);
     }
-    rom_add_blob_fixed("smpboot", smpboot, sizeof(smpboot),
+    rom_add_blob_fixed("smpboot", smpboot,
+                       smpboot_array_size * sizeof(uint32_t),
                        info->smp_loader_start);
 }
 
@@ -360,6 +407,9 @@ void arm_load_kernel(ARMCPU *cpu, struct arm_boot_info *info)
     hwaddr entry;
     int big_endian;
 
+    /* Select the bootloader to use and setup array indices, kernel entry etc */
+    setup_boot_env(cpu);
+
     /* Load the kernel.  */
     if (!info->kernel_filename) {
         fprintf(stderr, "Kernel image must be specified\n");
@@ -406,9 +456,9 @@ void arm_load_kernel(ARMCPU *cpu, struct arm_boot_info *info)
                                   &is_linux);
     }
     if (kernel_size < 0) {
-        entry = info->loader_start + KERNEL_LOAD_ADDR;
+        entry = info->loader_start + kernel_load_addr;
         kernel_size = load_image_targphys(info->kernel_filename, entry,
-                                          info->ram_size - KERNEL_LOAD_ADDR);
+                                          info->ram_size - kernel_load_addr);
         is_linux = 1;
     }
     if (kernel_size < 0) {
@@ -439,7 +489,7 @@ void arm_load_kernel(ARMCPU *cpu, struct arm_boot_info *info)
         }
         info->initrd_size = initrd_size;
 
-        bootloader[4] = info->board_id;
+        bootloader[kernel_boardid_index] = info->board_id;
 
         /* for device tree boot, we pass the DTB directly in r2. Otherwise
          * we point to the kernel args.
@@ -454,9 +504,9 @@ void arm_load_kernel(ARMCPU *cpu, struct arm_boot_info *info)
             if (load_dtb(dtb_start, info)) {
                 exit(1);
             }
-            bootloader[5] = dtb_start;
+            bootloader[kernel_args_index] = dtb_start;
         } else {
-            bootloader[5] = info->loader_start + KERNEL_ARGS_ADDR;
+            bootloader[kernel_args_index] = info->loader_start + KERNEL_ARGS_ADDR;
             if (info->ram_size >= (1ULL << 32)) {
                 fprintf(stderr, "qemu: RAM size must be less than 4GB to boot"
                         " Linux kernel using ATAGS (try passing a device tree"
@@ -464,11 +514,12 @@ void arm_load_kernel(ARMCPU *cpu, struct arm_boot_info *info)
                 exit(1);
             }
         }
-        bootloader[6] = entry;
-        for (n = 0; n < sizeof(bootloader) / 4; n++) {
+        bootloader[kernel_entry_index] = entry;
+        for (n = 0; n < bootloader_array_size; n++) {
             bootloader[n] = tswap32(bootloader[n]);
         }
-        rom_add_blob_fixed("bootloader", bootloader, sizeof(bootloader),
+        rom_add_blob_fixed("bootloader", bootloader,
+                           bootloader_array_size * sizeof(uint32_t),
                            info->loader_start);
         if (info->nb_cpus > 1) {
             info->write_secondary_boot(cpu, info);
Signature · 1 line
1.8.1.2
#13 lore raw

Re: [Qemu-devel] [PATCH v3 06/11] target-arm: Parameterize the bootloader selection and setup mechanism

Quoted 10 lines · Mian M. Hamayun

From: "Mian M. Hamayun" <[email protected]>

This commit replaces the constant indices used in bootloaders, such as for specifying the Board ID and kernel arguments with variable parameters. This change is used as mechanism to minimize code changes for different bootloaders, for example different bootloaders will be used for different architectures (ARMv7 vs. ARMv8).

Similary pointers are introduced to select appropriate bootloaders for boot and secondary cpus.

So I think parameterizing this a bit is good, but I don't think this code is quite the right approach...

Quoted 62 lines
---
 hw/arm/boot.c | 81 ++++++++++++++++++++++++++++++++++++++++++++++++-----------
 1 file changed, 66 insertions(+), 15 deletions(-)
hw/arm/boot.c +30 -3
index 967397b..4c1170e 100644
@@ -18,10 +18,9 @@
 #include "qemu/config-file.h"
 
 #define KERNEL_ARGS_ADDR 0x100
-#define KERNEL_LOAD_ADDR 0x00010000
 
 /* The worlds second smallest bootloader.  Set r0-r2, then jump to kernel.  */
-static uint32_t bootloader[] = {
+static uint32_t bootloader_arm32[] = {
   0xe3a00000, /* mov     r0, #0 */
   0xe59f1004, /* ldr     r1, [pc, #4] */
   0xe59f2004, /* ldr     r2, [pc, #4] */
@@ -48,7 +47,7 @@ static uint32_t bootloader[] = {
 #define DSB_INSN 0xf57ff04f
 #define CP15_DSB_INSN 0xee070f9a /* mcr cp15, 0, r0, c7, c10, 4 */
 
-static uint32_t smpboot[] = {
+static uint32_t smpboot_arm32[] = {
   0xe59f2028, /* ldr r2, gic_cpu_if */
   0xe59f0028, /* ldr r0, startaddr */
   0xe3a01001, /* mov r1, #1 */
@@ -65,13 +64,60 @@ static uint32_t smpboot[] = {
   0           /* bootreg: Boot register address is held here */
 };
 
+/*
+ * The bootloaders to be used are referenced by the following pointers
+ * An appropriate bootloader is selected depending on the architecture
+ * i.e. ARMv7, ARMv8 (AARCH64 and AARCH32)
+ */
+static uint32_t *bootloader = NULL;
+static uint32_t  bootloader_array_size = 0;
+
+static uint32_t *smpboot = NULL;
+static uint32_t  smpboot_array_size = 0;
+
+/*
+ * An index gives the location in the bootloader array, where we put the board
+ * ID, kernel arguments and kernel entry addresses. These are different for
+ * ARMv7 and ARMv8 bootloaders defined above.
+ */
+static uint32_t kernel_boardid_index = 0;
+static uint32_t kernel_args_index    = 0;
+static uint32_t kernel_entry_index   = 0;
+
+/*
+ * Similarly, the kernel loading address also depends on the architecture,
+ * i.e. its different for ARMv7, ARMv8 (AARCH64 and AARCH32)
+ */
+static uint32_t kernel_load_addr = 0x0;
+
+static void setup_boot_env_32(void)
+{

hw/arm/boot.c static uint32_t smpboot[] = { · line 104

Quoted 10 lines
hw/arm/boot.c +10 -0
+    bootloader = bootloader_arm32;
+    bootloader_array_size = ARRAY_SIZE(bootloader_arm32);
+    smpboot = smpboot_arm32;
+    smpboot_array_size = ARRAY_SIZE(smpboot_arm32);
+
+    kernel_boardid_index = bootloader_array_size - 3;
+    kernel_args_index    = bootloader_array_size - 2;
+    kernel_entry_index   = bootloader_array_size - 1;
+    return;
+}

...you wind up with a per-loader function which sets a bunch of globals, so the information relating to that loader is more scattered than necessary. Also, if we find that we need something other than boardid/args/entrypoint to be patched in, we need to update every loader. And we're fixing up the secondary boot loader separately from the primary cpu loader, just because it happens to need slightly different things doing to it.

My suggestion would be that we have a struct defining a loader:

typedef struct ARMInsnList {
    uint32_t insn;
    uint32_t fixup;
};

and then have each loader be an array ARMInsnList[];

fixup is one of a set of constants defining what if any action to take on that word:
#define FIXUP_NONE 0 /* do nothing */
#define FIXUP_TERMINATOR 1 /* end of list */
#define FIXUP_BOARDID 2 /* replace with board ID */
#define FIXUP_ARGPTR 3 /* replace with arg pointer */
#define FIXUP_ENTRYPOINT 4 /* replace with entry point */
#define FIXUP_DSB 5 /* if pre-v7, replace with cp15 DSB */
#define FIXUP_GIC_CPUIF 6 /* replace with GIC CPU i/f address */

and so on. The 32 bit loader then looks something like:

static const ARMInsnList bootloader_arm32[] = {
{ 0xe3a00000 } , /* mov r0, #0 */

  { 0xe59f1004 }, /* ldr     r1, [pc, #4] */
  { 0xe59f2004 }, /* ldr     r2, [pc, #4] */
  { 0xe59ff004 }, /* ldr     pc, [pc, #4] */

{ 0, FIXUP_BOARDID },
{ 0, FIXUP_ARGPTR },
{ 0, FIXUP_ENTRYPOINT },
{ 0, FIXUP_TERMINATOR }
};

(note that since we made FIXUP_NONE 0 we can omit it as the struct initialization will default to that.)

Then we can have a common bit of code that does the "create the fixed up instructions based on the list of insns and fixups"; it can figure out the length from the presence of the terminator, and it will be able to handle both the primary and secondary loaders, with scope for more flexibility if we need it via further fixup types. (It'll need to allocate an array to fill in since the insns aren't consecutive in an ARMInsnList but that's not a big deal.)

I'll have a go at writing up a patch for this this evening...

-- PMM

#14 lore raw

PATCH v3 7/11 AARCH64: Add boot support for aarch64 processor

In reply to Mian M. Hamayun

From: "Mian M. Hamayun" <[email protected]>

This commit adds support for booting a single AArch64 CPU by setting appropriate registers. The bootloader includes placehoders for Board-ID that are used to implement uniform indexing across different bootloaders. We also introduce Cortex-A57 to virt platform.

---
 hw/arm/boot.c | 57 +++++++++++++++++++++++++++++++++++++++++++++++++++++++++
 hw/arm/virt.c |  8 ++++++++
 2 files changed, 65 insertions(+)
hw/arm/boot.c +57 -0
index 4c1170e..0471eb8 100644
@@ -19,6 +19,23 @@
 
 #define KERNEL_ARGS_ADDR 0x100
 
+#ifdef TARGET_AARCH64
+static uint32_t bootloader_arm64[] = {
+    0x580000c0, 	/* ldr	x0, 18 ; Load the lower 32-bits of DTB */
+    0xaa1f03e1, 	/* mov	x1, xzr */
+    0xaa1f03e2, 	/* mov	x2, xzr */
+    0xaa1f03e3, 	/* mov	x3, xzr */
+    0x58000084, 	/* ldr	x4, 20 ; Load the lower 32-bits of kernel entry */
+    0xd61f0080, 	/* br	x4     ; Jump to the kernel entry point */
+    0x00000000, 	/* .word @DTB Lower 32-bits */
+    0x00000000, 	/* .word @DTB Higher 32-bits */
+    0x00000000, 	/* .word @Kernel Entry Lower 32-bits */
+    0x00000000,  	/* .word @Kernel Entry Higher 32-bits */
+    0x00000000, 	/* .word @Board ID Lower 32-bits -- Placeholder */
+    0x00000000  	/* .word @Board ID Higher 32-bits -- Placeholder */
+};
+#endif
+
 /* The worlds second smallest bootloader.  Set r0-r2, then jump to kernel.  */
 static uint32_t bootloader_arm32[] = {
   0xe3a00000, /* mov     r0, #0 */
@@ -103,11 +120,37 @@ static void setup_boot_env_32(void)
     return;
 }
 
+#ifdef TARGET_AARCH64
+static void setup_boot_env_64(void)
+{
+    bootloader = bootloader_arm64;
+    bootloader_array_size = ARRAY_SIZE(bootloader_arm64);
+
+    kernel_args_index    = bootloader_array_size - 6;
+    kernel_entry_index   = bootloader_array_size - 4;
+    kernel_boardid_index = bootloader_array_size - 2;
+    return;
+}
+#endif
+
 static void setup_boot_env(ARMCPU *cpu)
 {
+#ifdef TARGET_AARCH64
+    CPUARMState *env = &cpu->env;
+    if(env->aarch64) {
+        /* AARCH64 Mode */
+        kernel_load_addr = 0x00080000;
+        setup_boot_env_64();
+    }
+    else {
+        /* AARCH32 Mode */
+        /* TODO: Specify Kernel Load Address for AARCH32 */
+    }
+#else
     /* ARMv7 */
     kernel_load_addr = 0x00010000;
     setup_boot_env_32();
+#endif
     return;
 }
 
@@ -380,8 +423,22 @@ static void do_cpu_reset(void *opaque)
             env->regs[15] = info->entry & 0xfffffffe;
             env->thumb = info->entry & 1;
         } else {
+#ifdef TARGET_AARCH64
+            if(env->aarch64) {
+                env->pstate = PSR_D_BIT | PSR_A_BIT | PSR_I_BIT | PSR_F_BIT | PSR_MODE_EL1h;
+            } else {
+                hw_error("AArch32 mode is currently not supported\n");
+            }
+            env->xregs[0] =  0;
+            env->xregs[1] = -1;
+#endif
             if (CPU(cpu) == first_cpu) {
+#ifdef TARGET_AARCH64
+                env->xregs[2] = bootloader[kernel_args_index];
+                env->pc = info->loader_start;
+#else
                 env->regs[15] = info->loader_start;
+#endif
                 if (!info->dtb_filename) {
                     if (old_param) {
                         set_kernel_args_old(info);
hw/arm/virt.c +8 -0
index 448a0e5..8043fd4 100644
@@ -118,6 +118,14 @@ static VirtBoardInfo machines[] = {
         .memmap = a15memmap,
         .irqmap = a15irqmap,
     },
+    {
+        .cpu_model = "cortex-a57",
+        .cpu_compatible = "arm,arm-v8",
+        .qdevname = "a57mpcore_priv",
+        .gic_compatible = "arm,cortex-a15-gic",
+        .memmap = a15memmap,
+        .irqmap = a15irqmap,
+    },
 };
 
 static VirtBoardInfo *find_machine_info(const char *cpu)
Signature · 1 line
1.8.1.2
#15 lore raw

Re: [Qemu-devel] [PATCH v3 07/11] AARCH64: Add boot support for aarch64 processor

Quoted 6 lines · Mian M. Hamayun

From: "Mian M. Hamayun" <[email protected]>

This commit adds support for booting a single AArch64 CPU by setting appropriate registers. The bootloader includes placehoders for Board-ID that are used to implement uniform indexing across different bootloaders. We also introduce Cortex-A57 to virt platform.

Hi; apologies for not getting round to a closer look at this patch earlier. The changes are mostly fairly minor so my current plan is to put a version of this patch in the set I'm hoping to send out later this week...

Quoted 44 lines
---
 hw/arm/boot.c | 57 +++++++++++++++++++++++++++++++++++++++++++++++++++++++++
 hw/arm/virt.c |  8 ++++++++
 2 files changed, 65 insertions(+)
hw/arm/boot.c +23 -0
index 4c1170e..0471eb8 100644
@@ -19,6 +19,23 @@
 
 #define KERNEL_ARGS_ADDR 0x100
 
+#ifdef TARGET_AARCH64
+static uint32_t bootloader_arm64[] = {
+    0x580000c0,        /* ldr  x0, 18 ; Load the lower 32-bits of DTB */
+    0xaa1f03e1,        /* mov  x1, xzr */
+    0xaa1f03e2,        /* mov  x2, xzr */
+    0xaa1f03e3,        /* mov  x3, xzr */
+    0x58000084,        /* ldr  x4, 20 ; Load the lower 32-bits of kernel entry */
+    0xd61f0080,        /* br   x4     ; Jump to the kernel entry point */
+    0x00000000,        /* .word @DTB Lower 32-bits */
+    0x00000000,        /* .word @DTB Higher 32-bits */
+    0x00000000,        /* .word @Kernel Entry Lower 32-bits */
+    0x00000000,        /* .word @Kernel Entry Higher 32-bits */
+    0x00000000,        /* .word @Board ID Lower 32-bits -- Placeholder */
+    0x00000000         /* .word @Board ID Higher 32-bits -- Placeholder */
+};
+#endif
+
 /* The worlds second smallest bootloader.  Set r0-r2, then jump to kernel.  */
 static uint32_t bootloader_arm32[] = {
   0xe3a00000, /* mov     r0, #0 */
@@ -103,11 +120,37 @@ static void setup_boot_env_32(void)
     return;
 }
 
+#ifdef TARGET_AARCH64
+static void setup_boot_env_64(void)
+{
+    bootloader = bootloader_arm64;
+    bootloader_array_size = ARRAY_SIZE(bootloader_arm64);
+

hw/arm/boot.c static void setup_boot_env_32(void) · line 138

Quoted 10 lines
hw/arm/boot.c +8 -0
+    kernel_args_index    = bootloader_array_size - 6;
+    kernel_entry_index   = bootloader_array_size - 4;
+    kernel_boardid_index = bootloader_array_size - 2;
+    return;
+}
+#endif
+
 static void setup_boot_env(ARMCPU *cpu)
 {
+#ifdef TARGET_AARCH64

You don't need this ifdef -- env->aarch64 exists on 32 bit builds and is always zero, so just doing a runtime check should be fine.

hw/arm/boot.c static void setup_boot_env_32(void) · line 142

Quoted 4 lines
hw/arm/boot.c +4 -0
+    CPUARMState *env = &cpu->env;
+    if(env->aarch64) {
+        /* AARCH64 Mode */
+        kernel_load_addr = 0x00080000;

This offset should ideally be pulled out of the decompressed image header.

Quoted 15 lines
hw/arm/boot.c +8 -0
+        setup_boot_env_64();
+    }
+    else {
+        /* AARCH32 Mode */
+        /* TODO: Specify Kernel Load Address for AARCH32 */
+    }
+#else
     /* ARMv7 */
     kernel_load_addr = 0x00010000;
     setup_boot_env_32();
+#endif
     return;
 }
 

hw/arm/boot.c static void do_cpu_reset(void *opaque) · line 430

Quoted 8 lines
hw/arm/boot.c +5 -0
@@ -380,8 +423,22 @@ static void do_cpu_reset(void *opaque)
             env->regs[15] = info->entry & 0xfffffffe;
             env->thumb = info->entry & 1;
         } else {
+#ifdef TARGET_AARCH64
+            if(env->aarch64) {
+                env->pstate = PSR_D_BIT | PSR_A_BIT | PSR_I_BIT | PSR_F_BIT | PSR_MODE_EL1h;
+            } else {
+                hw_error("AArch32 mode is currently not supported\n");

TARGET_AARCH64 doesn't necessarily mean this is a 64
bit CPU; this ifdef breaks 32 bit CPUs in an aarch64-softmmu binary.

hw/arm/boot.c static void do_cpu_reset(void *opaque) · line 437

Quoted 7 lines
hw/arm/boot.c +6 -0
+            }
+            env->xregs[0] =  0;
+            env->xregs[1] = -1;
+#endif
             if (CPU(cpu) == first_cpu) {
+#ifdef TARGET_AARCH64
+                env->xregs[2] = bootloader[kernel_args_index];

Why are we setting X[2] here? The bootloader code is going to immediately clear it... Similarly X[0] and X[1] above.

hw/arm/boot.c static void do_cpu_reset(void *opaque) · line 440

Quoted 3 lines
hw/arm/boot.c +2 -0
+                env->pc = info->loader_start;
+#else
                 env->regs[15] = info->loader_start;

This ifdef should be an "if (aarch64)" runtime check too.

Quoted 11 lines
hw/arm/boot.c +1 -0
+#endif
                 if (!info->dtb_filename) {
                     if (old_param) {
                         set_kernel_args_old(info);
hw/arm/virt.c +0 -0
index 448a0e5..8043fd4 100644
@@ -118,6 +118,14 @@ static VirtBoardInfo machines[] = {
         .memmap = a15memmap,
         .irqmap = a15irqmap,

hw/arm/virt.c static VirtBoardInfo machines[] = { · line 129

Quoted 10 lines
hw/arm/virt.c +8 -0
     },
+    {
+        .cpu_model = "cortex-a57",
+        .cpu_compatible = "arm,arm-v8",
+        .qdevname = "a57mpcore_priv",
+        .gic_compatible = "arm,cortex-a15-gic",
+        .memmap = a15memmap,
+        .irqmap = a15irqmap,
+    },
 };

This hunk shouldn't really be in this patch I think.

thanks
-- PMM

#16 lore raw

PATCH v3 8/11 AARCH64: Enable SMP support for aarch64 processors using PSCI method

In reply to Mian M. Hamayun

From: "Mian M. Hamayun" <[email protected]>

We enable SMP support for aarch64 processors using the PSCI method, by setting the appropriate CPU feature flags at initilializtion time.

Secondary boot code for non-aarch64 processors is disabled in case of compilation for aarch64.

---
 hw/arm/boot.c       | 4 ++++
 target-arm/kvm_64.c | 7 +++++++
 2 files changed, 11 insertions(+)
hw/arm/boot.c +4 -0
index 0471eb8..ddafd3b 100644
@@ -157,6 +157,7 @@ static void setup_boot_env(ARMCPU *cpu)
 static void default_write_secondary(ARMCPU *cpu,
                                     const struct arm_boot_info *info)
 {
+#ifndef TARGET_AARCH64
     int n;
     smpboot[smpboot_array_size - 1] = info->smp_bootreg_addr;
     smpboot[smpboot_array_size - 2] = info->gic_cpu_if_addr;
@@ -171,15 +172,18 @@ static void default_write_secondary(ARMCPU *cpu,
     rom_add_blob_fixed("smpboot", smpboot,
                        smpboot_array_size * sizeof(uint32_t),
                        info->smp_loader_start);
+#endif
 }
 
 static void default_reset_secondary(ARMCPU *cpu,
                                     const struct arm_boot_info *info)
 {
+#ifndef TARGET_AARCH64
     CPUARMState *env = &cpu->env;
 
     stl_phys_notdirty(info->smp_bootreg_addr, 0);
     env->regs[15] = info->smp_loader_start;
+#endif
 }
 
 #define WRITE_WORD(p, value) do { \
target-arm/kvm_64.c +7 -0
index 9685727..146b7c4 100644
@@ -27,12 +27,19 @@ static uint32_t kvm_arm_targets[KVM_ARM_NUM_TARGETS] = {
     KVM_ARM_TARGET_CORTEX_A57
 };
 
+#define ARM_VCPU_FEATURE_FLAGS(cpuid, is_aarch32)  		\
+((!!(cpuid) << KVM_ARM_VCPU_POWER_OFF) | (is_aarch32 << KVM_ARM_VCPU_EL1_32BIT))
+
 int kvm_arch_init_vcpu(CPUState *cs)
 {
     struct kvm_vcpu_init init;
     int ret, i;
 
+    ARMCPU *cpu = ARM_CPU(cs);
+    CPUARMState *env = &cpu->env;
+
     memset(init.features, 0, sizeof(init.features));
+    init.features[0] = ARM_VCPU_FEATURE_FLAGS(cs->cpu_index, !env->aarch64);
     /* Find an appropriate target CPU type.
      * KVM does not provide means to detect the host CPU type on aarch64,
      * and simply refuses to initialize, if the CPU type mis-matches;
Signature · 1 line
1.8.1.2
#17 lore raw

PATCH v3 9/11 AARCH64: Enable configure support for 32-bit guests on AARCH64

In reply to Mian M. Hamayun

From: "Mian M. Hamayun" <[email protected]>

---
 configure | 1 +
 1 file changed, 1 insertion(+)
configure +1 -0
index 7298b69..dbaf56f 100755
@@ -4485,6 +4485,7 @@ case "$target_name" in
     # Make sure the target and host cpus are compatible
     if test "$kvm" = "yes" -a "$target_softmmu" = "yes" -a \
       \( "$target_name" = "$cpu" -o \
+      \( "$target_name" = "arm" -a "$cpu" = "aarch64" \) -o \
       \( "$target_name" = "ppcemb" -a "$cpu" = "ppc" \) -o \
       \( "$target_name" = "ppc64"  -a "$cpu" = "ppc" \) -o \
       \( "$target_name" = "ppc"    -a "$cpu" = "ppc64" \) -o \
Signature · 1 line
1.8.1.2
#18 lore raw

PATCH v3 10/11 AARCH64: Add flags and boot parameters for 32-bit guests on AARCH64

In reply to Mian M. Hamayun

From: "Mian M. Hamayun" <[email protected]>

This commit adds the necessary flags and kernel load address to enable booting of 32-bit guests on AArch64 processors.

The actual enable/disable mechanism is not included in this commit, which should tweak the value of env->aarch64 variable for this purpose.

---
 hw/arm/boot.c | 9 +++++++--
 1 file changed, 7 insertions(+), 2 deletions(-)
hw/arm/boot.c +7 -2
index ddafd3b..2cfa9bf 100644
@@ -20,6 +20,10 @@
 #define KERNEL_ARGS_ADDR 0x100
 
 #ifdef TARGET_AARCH64
+#define COMPAT_PSR_F_BIT    0x00000040
+#define COMPAT_PSR_I_BIT    0x00000080
+#define COMPAT_PSR_MODE_SVC 0x00000013
+
 static uint32_t bootloader_arm64[] = {
     0x580000c0, 	/* ldr	x0, 18 ; Load the lower 32-bits of DTB */
     0xaa1f03e1, 	/* mov	x1, xzr */
@@ -144,7 +148,8 @@ static void setup_boot_env(ARMCPU *cpu)
     }
     else {
         /* AARCH32 Mode */
-        /* TODO: Specify Kernel Load Address for AARCH32 */
+        kernel_load_addr = 0x00008000;
+        setup_boot_env_32();
     }
 #else
     /* ARMv7 */
@@ -431,7 +436,7 @@ static void do_cpu_reset(void *opaque)
             if(env->aarch64) {
                 env->pstate = PSR_D_BIT | PSR_A_BIT | PSR_I_BIT | PSR_F_BIT | PSR_MODE_EL1h;
             } else {
-                hw_error("AArch32 mode is currently not supported\n");
+                env->pstate = COMPAT_PSR_I_BIT | COMPAT_PSR_F_BIT | COMPAT_PSR_MODE_SVC;
             }
             env->xregs[0] =  0;
             env->xregs[1] = -1;
Signature · 1 line
1.8.1.2
#19 lore raw

PATCH v3 11/11 AARCH64: Add 32-bit mode selection parameter

In reply to Mian M. Hamayun

From: "Mian M. Hamayun" <[email protected]>

This commit introduces a commandline argument to select the AARCH64 or AARCH32 mode for processor initilization.

---
 qemu-options.hx  | 8 ++++++++
 target-arm/cpu.c | 9 +++++++--
 vl.c             | 4 ++++
 3 files changed, 19 insertions(+), 2 deletions(-)
qemu-options.hx +8 -0
index 5dc8b75..a2dab99 100644
@@ -72,6 +72,14 @@ STEXI
 Select CPU model (@code{-cpu help} for list and additional feature selection)
 ETEXI
 
+DEF("aarch32-mode", 0, QEMU_OPTION_aarch32_mode, \
+    "-aarch32-mode   enable aarch32 mode support on aarch64\n", QEMU_ARCH_ARM)
+STEXI
+@item -aarch32-mode
+@findex -aarch32-mode
+Enable aarch32 guest support on aarch64.
+ETEXI
+
 DEF("smp", HAS_ARG, QEMU_OPTION_smp,
     "-smp [cpus=]n[,maxcpus=cpus][,cores=cores][,threads=threads][,sockets=sockets]\n"
     "                set the number of CPUs to 'n' [default=1]\n"
target-arm/cpu.c +7 -2
index 5d811b9..56e8e56 100644
@@ -27,6 +27,8 @@
 #include "sysemu/sysemu.h"
 #include "sysemu/kvm.h"
 
+extern int aarch32_mode;
+
 static void arm_cpu_set_pc(CPUState *cs, vaddr value)
 {
     ARMCPU *cpu = ARM_CPU(cs);
@@ -85,8 +87,11 @@ static void arm_cpu_reset(CPUState *s)
     }
 
     if (arm_feature(env, ARM_FEATURE_AARCH64)) {
-        /* 64 bit CPUs always start in 64 bit mode */
-        env->aarch64 = 1;
+        if(aarch32_mode) {
+            env->aarch64 = 0;  /* Boot a 32-bit Guest */
+        } else {
+            env->aarch64 = 1;  /* Boot a 64-bit Guest */
+        }
     }
 
 #if defined(CONFIG_USER_ONLY)
vl.c +4 -0
index 4e709d5..e0f2cf3 100644
@@ -207,6 +207,7 @@ CharDriverState *virtcon_hds[MAX_VIRTIO_CONSOLES];
 CharDriverState *sclp_hds[MAX_SCLP_CONSOLES];
 int win2k_install_hack = 0;
 int singlestep = 0;
+int aarch32_mode = 0;
 int smp_cpus = 1;
 int max_cpus = 0;
 int smp_cores = 1;
@@ -3113,6 +3114,9 @@ int main(int argc, char **argv, char **envp)
             case QEMU_OPTION_dtb:
                 qemu_opts_set(qemu_find_opts("machine"), 0, "dtb", optarg);
                 break;
+            case QEMU_OPTION_aarch32_mode:
+                aarch32_mode = 1;
+                break;
             case QEMU_OPTION_cdrom:
                 drive_add(IF_DEFAULT, 2, optarg, CDROM_OPTS);
                 break;
Signature · 1 line
1.8.1.2