arm64: Early CPU feature override, and an application to VHE

As we are about to change the way a VHE system boots, let's provide the core helper, in the form of a stub hypercall that enables VHE and replicates the full EL1 context at EL2, thanks to EL1 and VHE-EL2 being extremely similar.

On exception return, the kernel carries on at EL2. Fancy!

Nothing calls this new hypercall yet, so no functional change.

Signed-off-by Marc Zyngier <[email protected]>

---
 arch/arm64/include/asm/virt.h |  7 +++-
 arch/arm64/kernel/hyp-stub.S  | 70 ++++++++++++++++++++++++++++++++++-
 2 files changed, 74 insertions(+), 3 deletions(-)

arch/arm64/include/asm/virt.h +6 -1

index ee6a48df89d9..7379f35ae2c6 100644
@@ -35,8 +35,13 @@
  */
 #define HVC_RESET_VECTORS 2
 
+/*
+ * HVC_VHE_RESTART - Upgrade the CPU from EL1 to EL2, if possible
+ */
+#define HVC_VHE_RESTART        3
+
 /* Max number of HYP stub hypercalls */
-#define HVC_STUB_HCALL_NR 3
+#define HVC_STUB_HCALL_NR 4
 
 /* Error returned when an invalid stub number is passed into x0 */
 #define HVC_STUB_ERR   0xbadca11

arch/arm64/kernel/hyp-stub.S +1 -1

index 160f5881a0b7..6ffdc1f7778b 100644
@@ -8,9 +8,9 @@
 
 #include <linux/init.h>
 #include <linux/linkage.h>
-#include <linux/irqchip/arm-gic-v3.h>
 
 #include <asm/assembler.h>
+#include <asm/el2_setup.h>
 #include <asm/kvm_arm.h>
 #include <asm/kvm_asm.h>
 #include <asm/ptrace.h>

@@ -47,10 +47,16 @@ SYM_CODE_END(__hyp_stub_vectors)
 
 SYM_CODE_START_LOCAL(el1_sync)
        cmp     x0, #HVC_SET_VECTORS

arch/arm64/kernel/hyp-stub.S +61 -1

+
 2:     cmp     x0, #HVC_SOFT_RESTART
        b.ne    3f
        mov     x0, x2

@@ -70,6 +76,66 @@ SYM_CODE_START_LOCAL(el1_sync)
        eret
 SYM_CODE_END(el1_sync)
 
+// nVHE? No way! Give me the real thing!
+SYM_CODE_START_LOCAL(mutate_to_vhe)
+       // Sanity check: MMU *must* be off
+       mrs     x0, sctlr_el2
+       tbnz    x0, #0, 1f
+
+       // Needs to be VHE capable, obviously
+       mrs     x0, id_aa64mmfr1_el1
+       ubfx    x0, x0, #ID_AA64MMFR1_VHE_SHIFT, #4
+       cbz     x0, 1f
+
+       // Engage the VHE magic!
+       mov_q   x0, HCR_HOST_VHE_FLAGS
+       msr     hcr_el2, x0
+       isb
+
+       // Doesn't do much on VHE, but still, worth a shot
+       init_el2_state vhe
+
+       // Use the EL1 allocated stack, per-cpu offset
+       mrs     x0, sp_el1
+       mov     sp, x0
+       mrs     x0, tpidr_el1
+       msr     tpidr_el2, x0
+
+       // FP configuration, vectors
+       mrs_s   x0, SYS_CPACR_EL12
+       msr     cpacr_el1, x0
+       mrs_s   x0, SYS_VBAR_EL12
+       msr     vbar_el1, x0
+
+       // Transfert the MM state from EL1 to EL2
+       mrs_s   x0, SYS_TCR_EL12
+       msr     tcr_el1, x0
+       mrs_s   x0, SYS_TTBR0_EL12
+       msr     ttbr0_el1, x0
+       mrs_s   x0, SYS_TTBR1_EL12
+       msr     ttbr1_el1, x0
+       mrs_s   x0, SYS_MAIR_EL12
+       msr     mair_el1, x0
+       isb
+
+       // Invalidate TLBs before enabling the MMU
+       tlbi    vmalle1
+       dsb     nsh
+
+       // Enable the EL2 S1 MMU, as set up from EL1
+       mrs_s   x0, SYS_SCTLR_EL12
+       set_sctlr_el1   x0
+
+       // Hack the exception return to stay at EL2
+       mrs     x0, spsr_el1
+       and     x0, x0, #~PSR_MODE_MASK
+       mov     x1, #PSR_MODE_EL2h
+       orr     x0, x0, x1
+       msr     spsr_el1, x0
+
+1:     eret
+SYM_CODE_END(mutate_to_vhe)
+
 .macro invalid_vector  label
 SYM_CODE_START_LOCAL(\label)
        b \label
--

2.29.2

_______________________________________________
kvmarm mailing list
[email protected]
https://lists.cs.columbia.edu/mailman/listinfo/kvmarm

Quoted 59 lines · Marc Zyngier

As we are about to change the way a VHE system boots, let's provide the core helper, in the form of a stub hypercall that enables VHE and replicates the full EL1 context at EL2, thanks to EL1 and VHE-EL2 being extremely similar.

On exception return, the kernel carries on at EL2. Fancy!

Nothing calls this new hypercall yet, so no functional change.

Signed-off-by Marc Zyngier <[email protected]>

---
 arch/arm64/include/asm/virt.h |  7 +++-
 arch/arm64/kernel/hyp-stub.S  | 70 ++++++++++++++++++++++++++++++++++-
 2 files changed, 74 insertions(+), 3 deletions(-)

arch/arm64/include/asm/virt.h +6 -1

index ee6a48df89d9..7379f35ae2c6 100644
@@ -35,8 +35,13 @@
  */
 #define HVC_RESET_VECTORS 2
 
+/*
+ * HVC_VHE_RESTART - Upgrade the CPU from EL1 to EL2, if possible
+ */
+#define HVC_VHE_RESTART        3
+
 /* Max number of HYP stub hypercalls */
-#define HVC_STUB_HCALL_NR 3
+#define HVC_STUB_HCALL_NR 4
 
 /* Error returned when an invalid stub number is passed into x0 */
 #define HVC_STUB_ERR   0xbadca11

arch/arm64/kernel/hyp-stub.S +2 -2

index 160f5881a0b7..6ffdc1f7778b 100644
@@ -8,9 +8,9 @@
 
 #include <linux/init.h>
 #include <linux/linkage.h>
-#include <linux/irqchip/arm-gic-v3.h>
 
 #include <asm/assembler.h>
+#include <asm/el2_setup.h>
 #include <asm/kvm_arm.h>
 #include <asm/kvm_asm.h>
 #include <asm/ptrace.h>

@@ -47,10 +47,16 @@ SYM_CODE_END(__hyp_stub_vectors)
 
 SYM_CODE_START_LOCAL(el1_sync)
        cmp     x0, #HVC_SET_VECTORS
-       b.ne    2f
+       b.ne    1f
        msr     vbar_el2, x1
        b       9f
 

arch/arm64/kernel/hyp-stub.S SYM_CODE_END(__hyp_stub_vectors) · line 58

Quoted 5 lines

arch/arm64/kernel/hyp-stub.S +5 -0

+1:     cmp     x0, #HVC_VHE_RESTART
+       b.ne    2f
+       mov     x0, #HVC_SOFT_RESTART
+       adr     x1, mutate_to_vhe
+       // fall through...

Does "br x1" work here instead of falling through into HVC_SOFT_RESTART? Just curious about the reason for falling-through.

In order to be able to override CPU features at boot time, let's add a command line parser that matches options of the form "cpureg.feature=value", and store the corresponding value into the override val/mask pair.

No features are currently defined, so no expected change in functionnality.

Signed-off-by Marc Zyngier <[email protected]>

---
 arch/arm64/kernel/Makefile         |   2 +-
 arch/arm64/kernel/head.S           |   1 +
 arch/arm64/kernel/idreg-override.c | 119 +++++++++++++++++++++++++++++
 3 files changed, 121 insertions(+), 1 deletion(-)
 create mode 100644 arch/arm64/kernel/idreg-override.c

arch/arm64/kernel/Makefile +1 -1

index 86364ab6f13f..2262f0392857 100644
@@ -17,7 +17,7 @@ obj-y                 := debug-monitors.o entry.o irq.o fpsimd.o              \
                           return_address.o cpuinfo.o cpu_errata.o              \
                           cpufeature.o alternative.o cacheinfo.o               \
                           smp.o smp_spin_table.o topology.o smccc-call.o       \
-                          syscall.o proton-pack.o
+                          syscall.o proton-pack.o idreg-override.o
 
 targets                        += efi-entry.o
 

arch/arm64/kernel/head.S +1 -0

index d74e5f84042e..b3c4dd04f74b 100644
@@ -435,6 +435,7 @@ SYM_FUNC_START_LOCAL(__primary_switched)
 
        mov     x0, x21                         // pass FDT address in x0
        bl      early_fdt_map                   // Try mapping the FDT early
+       bl      init_shadow_regs
        bl      switch_to_vhe
 #if defined(CONFIG_KASAN_GENERIC) || defined(CONFIG_KASAN_SW_TAGS)
        bl      kasan_early_init

arch/arm64/kernel/idreg-override.c new file +108 -0

new file mode 100644
index 000000000000..392f93b67103
@@ -0,0 +1,119 @@
+// SPDX-License-Identifier: GPL-2.0
+/*
+ * Early cpufeature override framework
+ *
+ * Copyright (C) 2020 Google LLC
+ * Author: Marc Zyngier <[email protected]>
+ */
+
+#include <linux/kernel.h>
+#include <linux/libfdt.h>
+
+#include <asm/cacheflush.h>
+#include <asm/setup.h>
+
+struct reg_desc {
+       const char * const      name;
+       u64 * const             val;
+       u64 * const             mask;
+       struct {
+               const char * const      name;
+               u8                       shift;
+       }                       fields[];
+};
+
+static const struct reg_desc * const regs[] __initdata = {
+};
+
+static int __init find_field(const char *cmdline, const struct reg_desc *reg,
+                            int f, u64 *v)
+{
+       char buf[256], *str;
+       size_t len;
+
+       snprintf(buf, ARRAY_SIZE(buf), "%s.%s=", reg->name, reg->fields[f].name);
+
+       str = strstr(cmdline, buf);
+       if (!(str == cmdline || (str > cmdline && *(str - 1) == ' ')))
+               return -1;
+
+       str += strlen(buf);
+       len = strcspn(str, " ");
+       len = min(len, ARRAY_SIZE(buf) - 1);
+       strncpy(buf, str, len);
+       buf[len] = 0;
+
+       return kstrtou64(buf, 0, v);
+}
+
+static void __init match_options(const char *cmdline)
+{
+       int i;
+
+       for (i = 0; i < ARRAY_SIZE(regs); i++) {
+               int f;
+
+               if (!regs[i]->val || !regs[i]->mask)
+                       continue;
+
+               for (f = 0; regs[i]->fields[f].name; f++) {
+                       u64 v;
+
+                       if (find_field(cmdline, regs[i], f, &v))
+                               continue;
+
+                       *regs[i]->val  |= (v & 0xf) << regs[i]->fields[f].shift;
+                       *regs[i]->mask |= 0xfUL << regs[i]->fields[f].shift;
+               }
+       }
+}
+
+static __init void parse_cmdline(void)
+{
+       if (!IS_ENABLED(CONFIG_CMDLINE_FORCE)) {
+               const u8 *prop;
+               void *fdt;
+               int node;
+
+               fdt = get_early_fdt_ptr();
+               if (!fdt)
+                       goto out;
+
+               node = fdt_path_offset(fdt, "/chosen");
+               if (node < 0)
+                       goto out;
+
+               prop = fdt_getprop(fdt, node, "bootargs", NULL);
+               if (!prop)
+                       goto out;
+
+               match_options(prop);
+
+               if (!IS_ENABLED(CONFIG_CMDLINE_EXTEND))
+                       return;
+       }
+
+out:
+       match_options(CONFIG_CMDLINE);
+}
+
+void __init init_shadow_regs(void)
+{
+       int i;
+
+       for (i = 0; i < ARRAY_SIZE(regs); i++) {
+               if (regs[i]->val)
+                       *regs[i]->val  = 0;
+               if (regs[i]->mask)
+                       *regs[i]->mask = 0;

arch/arm64/kernel/idreg-override.c new file +1 -1

+}
--

2.29.2

_______________________________________________
kvmarm mailing list
[email protected]
https://lists.cs.columbia.edu/mailman/listinfo/kvmarm

On Mon, Jan 4, 2021 at 8:20 AM Marc Zyngier <[email protected]> wrote:

Quoted 43 lines

arch/arm64/kernel/idreg-override.c new file +43 -0

+static __init void parse_cmdline(void)
+{
+       if (!IS_ENABLED(CONFIG_CMDLINE_FORCE)) {
+               const u8 *prop;
+               void *fdt;
+               int node;
+
+               fdt = get_early_fdt_ptr();
+               if (!fdt)
+                       goto out;
+
+               node = fdt_path_offset(fdt, "/chosen");
+               if (node < 0)
+                       goto out;
+
+               prop = fdt_getprop(fdt, node, "bootargs", NULL);
+               if (!prop)
+                       goto out;
+
+               match_options(prop);
+
+               if (!IS_ENABLED(CONFIG_CMDLINE_EXTEND))
+                       return;
+       }
+
+out:
+       match_options(CONFIG_CMDLINE);
+}
+
+void __init init_shadow_regs(void)
+{
+       int i;
+
+       for (i = 0; i < ARRAY_SIZE(regs); i++) {
+               if (regs[i]->val)
+                       *regs[i]->val  = 0;
+               if (regs[i]->mask)
+                       *regs[i]->mask = 0;
+       }
+
+       parse_cmdline();
+
+       for (i = 0; i < ARRAY_SIZE(regs); i++) {

Quoted 7 lines

(quoted diff) +2 -0

+               if (regs[i]->val)
+                       __flush_dcache_area(regs[i]->val, 

sizeof(*regs[i]->val));

(quoted diff) +2 -0

+               if (regs[i]->mask)
+                       __flush_dcache_area(regs[i]->mask, 

sizeof(*regs[i]->mask));
+ }

Could you shed some light on the usage of __flush_dcache_area here? Thanks.

As we want to be able to disable VHE at runtime, let's match "id_aa64mmfr1.vh=" from the command line as an override. This doesn't have much effect yet as out boot code doesn't look

at the cpufeature, but only at the HW registers.

Signed-off-by Marc Zyngier <[email protected]>

---
 arch/arm64/include/asm/cpufeature.h |  3 +++
 arch/arm64/kernel/cpufeature.c      |  6 +++++-
 arch/arm64/kernel/idreg-override.c  | 12 ++++++++++++
 3 files changed, 20 insertions(+), 1 deletion(-)

arch/arm64/include/asm/cpufeature.h +3 -0

index 465d2cb63bfc..ef9487dfadca 100644
@@ -813,6 +813,9 @@ static inline unsigned int get_vmid_bits(u64 mmfr1)
        return 8;
 }
 
+extern u64 id_aa64mmfr1_val;
+extern u64 id_aa64mmfr1_mask;
+
 u32 get_kvm_ipa_limit(void);
 void dump_cpu_features(void);
 

arch/arm64/kernel/cpufeature.c +5 -1

index 2dbfc0096b0d..623a74090c9e 100644
@@ -555,6 +555,9 @@ static const struct arm64_ftr_bits ftr_raz[] = {
 
 #define ARM64_FTR_REG(id, table) ARM64_FTR_REG_OVERRIDE(id, table, NULL, NULL)
 
+u64 id_aa64mmfr1_val;
+u64 id_aa64mmfr1_mask;
+
 static const struct __ftr_reg_entry {
        u32                     sys_id;
        struct arm64_ftr_reg    *reg;

@@ -602,7 +605,8 @@ static const struct __ftr_reg_entry {
 
        /* Op1 = 0, CRn = 0, CRm = 7 */
        ARM64_FTR_REG(SYS_ID_AA64MMFR0_EL1, ftr_id_aa64mmfr0),
-       ARM64_FTR_REG(SYS_ID_AA64MMFR1_EL1, ftr_id_aa64mmfr1),
+       ARM64_FTR_REG_OVERRIDE(SYS_ID_AA64MMFR1_EL1, ftr_id_aa64mmfr1,
+                              &id_aa64mmfr1_val, &id_aa64mmfr1_mask),
        ARM64_FTR_REG(SYS_ID_AA64MMFR2_EL1, ftr_id_aa64mmfr2),
 
        /* Op1 = 0, CRn = 1, CRm = 2 */

arch/arm64/kernel/idreg-override.c +12 -1

index 392f93b67103..75d9845f489b 100644
@@ -10,6 +10,7 @@
 #include <linux/libfdt.h>
 
 #include <asm/cacheflush.h>
+#include <asm/cpufeature.h>
 #include <asm/setup.h>
 
 struct reg_desc {

@@ -22,7 +23,18 @@ struct reg_desc {
        }                       fields[];
 };
 
+static const struct reg_desc mmfr1 __initdata = {
+       .name           = "id_aa64mmfr1",
+       .val            = &id_aa64mmfr1_val,
+       .mask           = &id_aa64mmfr1_mask,
+       .fields         = {
+               { "vh", ID_AA64MMFR1_VHE_SHIFT },
+               {}
+       },
+};
+
 static const struct reg_desc * const regs[] __initdata = {
+       &mmfr1,
 };
 
 static int __init find_field(const char *cmdline, const struct reg_desc *reg,
--

2.29.2

_______________________________________________
kvmarm mailing list
[email protected]
https://lists.cs.columbia.edu/mailman/listinfo/kvmarm

For completeness, let's document the HVC_VHE_RESTART stub.

Signed-off-by Marc Zyngier <[email protected]>

---
 Documentation/virt/kvm/arm/hyp-abi.rst | 9 +++++++++
 1 file changed, 9 insertions(+)

Documentation/virt/kvm/arm/hyp-abi.rst +0 -0

index 83cadd8186fa..3961c529769b 100644

Documentation/virt/kvm/arm/hyp-abi.rst +3 -1

+  being off, and VHE not being disabled by any other mean (comment line option,
+  for example).
+
 Any other value of r0/x0 triggers a hypervisor-specific handling,
 which is not documented here.
 
--

2.29.2

_______________________________________________
kvmarm mailing list
[email protected]
https://lists.cs.columbia.edu/mailman/listinfo/kvmarm