arm64: head: Switch endianness before populating the ID map
Ard Biesheuvel
maintainerPATCH 1/1 arm64: head: Switch endianness before populating the ID map
Ensure that the endianness used for populating the ID map matches the endianness that the running kernel will be using, as this is no longer guaranteed now that create_idmap() is invoked before init_kernel_el().
Note that doing so is only safe if the MMU is off, as switching the endianness with the MMU on results in the active ID map to become invalid. So also clear the C and M bits when toggling the EE bit in SCTLR, and mark the MMU as disabled at boot.
Note that the same issue has resulted in preserve_boot_args() recording the contents of registers X0 ... X3 in the wrong byte order, although this is arguably a very minor concern.
--- arch/arm64/include/asm/sysreg.h | 3 ++- arch/arm64/kernel/head.S | 23 ++++++++++++++++++++++- 2 files changed, 24 insertions(+), 2 deletions(-)
arch/arm64/include/asm/sysreg.h +2 -1
@@ -575,6 +575,7 @@#define SCTLR_ELx_DSSBS (BIT(44))#define SCTLR_ELx_ATA (BIT(43)) +#define SCTLR_ELx_EE_SHIFT 25#define SCTLR_ELx_ENIA_SHIFT 31#define SCTLR_ELx_ITFSB (BIT(37))
@@ -583,7 +584,7 @@#define SCTLR_ELx_LSMAOE (BIT(29))#define SCTLR_ELx_nTLSMD (BIT(28))#define SCTLR_ELx_ENDA (BIT(27)) -#define SCTLR_ELx_EE (BIT(25)) +#define SCTLR_ELx_EE (BIT(SCTLR_ELx_EE_SHIFT))#define SCTLR_ELx_EIS (BIT(22))#define SCTLR_ELx_IESB (BIT(21))#define SCTLR_ELx_TSCXT (BIT(20))
arch/arm64/kernel/head.S +22 -1
@@ -129,10 +129,31 @@ SYM_CODE_START_LOCAL(record_mmu_state) mrs x19, sctlr_el1 b.ne 0f mrs x19, sctlr_el2 -0: tst x19, #SCTLR_ELx_C // Z := (C == 0) +0: +CPU_LE( tbnz x19, #SCTLR_ELx_EE_SHIFT, 1f ) +CPU_BE( tbz x19, #SCTLR_ELx_EE_SHIFT, 1f ) + tst x19, #SCTLR_ELx_C // Z := (C == 0) and x19, x19, #SCTLR_ELx_M // isolate M bit csel x19, xzr, x19, eq // clear x19 if Z ret + + /* + * Set the correct endianness early so all memory accesses issued + * before init_kernel_el() occur in the correct byte order. Note that + * this means the MMU must be disabled, or the active ID map will end + * up getting interpreted with the wrong byte order. + */ +1: eor x19, x19, #SCTLR_ELx_EE + bic x19, x19, #SCTLR_ELx_M + b.ne 2f + pre_disable_mmu_workaround + msr sctlr_el2, x19 + b 3f + pre_disable_mmu_workaround +2: msr sctlr_el1, x19 +3: isb + mov x19, xzr + ret SYM_CODE_END(record_mmu_state) /*
Signature · 1 line
2.39.0
Ard Biesheuvel
maintainerQuoted 8 lines · Ard Biesheuvel
Ensure that the endianness used for populating the ID map matches the endianness that the running kernel will be using, as this is no longer guaranteed now that
create_idmap()is invoked beforeinit_kernel_el().Note that doing so is only safe if the MMU is off, as switching the endianness with the MMU on results in the active ID map to become invalid. So also clear the C and M bits when toggling the EE bit in
not the C bit - I ripped that out at the last minute
Quoted 75 lines
SCTLR, and mark the MMU as disabled at boot.
Note that the same issue has resulted in
preserve_boot_args()recording the contents of registers X0 ... X3 in the wrong byte order, although this is arguably a very minor concern.Fixes 32b135a7fafe ("arm64: head: avoid cache invalidation when entering with the MMU on")Reported-by Nathan Chancellor <[email protected]>Signed-off-by Ard Biesheuvel <[email protected]>--- arch/arm64/include/asm/sysreg.h | 3 ++- arch/arm64/kernel/head.S | 23 ++++++++++++++++++++++- 2 files changed, 24 insertions(+), 2 deletions(-)arch/arm64/include/asm/sysreg.h +2
-1@@ -575,6 +575,7 @@#define SCTLR_ELx_DSSBS (BIT(44))#define SCTLR_ELx_ATA (BIT(43)) +#define SCTLR_ELx_EE_SHIFT 25#define SCTLR_ELx_ENIA_SHIFT 31#define SCTLR_ELx_ITFSB (BIT(37))@@ -583,7 +584,7 @@#define SCTLR_ELx_LSMAOE (BIT(29))#define SCTLR_ELx_nTLSMD (BIT(28))#define SCTLR_ELx_ENDA (BIT(27)) -#define SCTLR_ELx_EE (BIT(25)) +#define SCTLR_ELx_EE (BIT(SCTLR_ELx_EE_SHIFT))#define SCTLR_ELx_EIS (BIT(22))#define SCTLR_ELx_IESB (BIT(21))#define SCTLR_ELx_TSCXT (BIT(20))arch/arm64/kernel/head.S +22
-2@@ -129,10 +129,31 @@ SYM_CODE_START_LOCAL(record_mmu_state) mrs x19, sctlr_el1 b.ne 0f mrs x19, sctlr_el2 -0: tst x19, #SCTLR_ELx_C // Z := (C == 0) +0: +CPU_LE( tbnz x19, #SCTLR_ELx_EE_SHIFT, 1f ) +CPU_BE( tbz x19, #SCTLR_ELx_EE_SHIFT, 1f ) + tst x19, #SCTLR_ELx_C // Z := (C == 0) and x19, x19, #SCTLR_ELx_M // isolate M bit csel x19, xzr, x19, eq // clear x19 if Z ret + + /* + * Set the correct endianness early so all memory accesses issued + * before init_kernel_el() occur in the correct byte order. Note that + * this means the MMU must be disabled, or the active ID map will end + * up getting interpreted with the wrong byte order. + */ +1: eor x19, x19, #SCTLR_ELx_EE + bic x19, x19, #SCTLR_ELx_M + b.ne 2f + pre_disable_mmu_workaround + msr sctlr_el2, x19 + b 3f + pre_disable_mmu_workaround +2: msr sctlr_el1, x19 +3: isb + mov x19, xzr + ret SYM_CODE_END(record_mmu_state) /* --2.39.0
Quoted 9 lines · Ard Biesheuvel
Ensure that the endianness used for populating the ID map matches the endianness that the running kernel will be using, as this is no longer guaranteed now that
create_idmap()is invoked beforeinit_kernel_el().Note that doing so is only safe if the MMU is off, as switching the endianness with the MMU on results in the active ID map to become invalid. So also clear the C and M bits when toggling the EE bit in SCTLR, and mark the MMU as disabled at boot.
Quoted 7 lines
Note that the same issue has resulted in
preserve_boot_args()recording the contents of registers X0 ... X3 in the wrong byte order, although this is arguably a very minor concern.Fixes 32b135a7fafe ("arm64: head: avoid cache invalidation when entering with the MMU on")Reported-by Nathan Chancellor <[email protected]>Signed-off-by Ard Biesheuvel <[email protected]>
Thanks a lot for the quick fix!
Quoted 66 lines
--- arch/arm64/include/asm/sysreg.h | 3 ++- arch/arm64/kernel/head.S | 23 ++++++++++++++++++++++- 2 files changed, 24 insertions(+), 2 deletions(-)arch/arm64/include/asm/sysreg.h +2
-1@@ -575,6 +575,7 @@#define SCTLR_ELx_DSSBS (BIT(44))#define SCTLR_ELx_ATA (BIT(43)) +#define SCTLR_ELx_EE_SHIFT 25#define SCTLR_ELx_ENIA_SHIFT 31#define SCTLR_ELx_ITFSB (BIT(37))@@ -583,7 +584,7 @@#define SCTLR_ELx_LSMAOE (BIT(29))#define SCTLR_ELx_nTLSMD (BIT(28))#define SCTLR_ELx_ENDA (BIT(27)) -#define SCTLR_ELx_EE (BIT(25)) +#define SCTLR_ELx_EE (BIT(SCTLR_ELx_EE_SHIFT))#define SCTLR_ELx_EIS (BIT(22))#define SCTLR_ELx_IESB (BIT(21))#define SCTLR_ELx_TSCXT (BIT(20))arch/arm64/kernel/head.S +22
-1@@ -129,10 +129,31 @@ SYM_CODE_START_LOCAL(record_mmu_state) mrs x19, sctlr_el1 b.ne 0f mrs x19, sctlr_el2 -0: tst x19, #SCTLR_ELx_C // Z := (C == 0) +0: +CPU_LE( tbnz x19, #SCTLR_ELx_EE_SHIFT, 1f ) +CPU_BE( tbz x19, #SCTLR_ELx_EE_SHIFT, 1f ) + tst x19, #SCTLR_ELx_C // Z := (C == 0) and x19, x19, #SCTLR_ELx_M // isolate M bit csel x19, xzr, x19, eq // clear x19 if Z ret + + /* + * Set the correct endianness early so all memory accesses issued + * before init_kernel_el() occur in the correct byte order. Note that + * this means the MMU must be disabled, or the active ID map will end + * up getting interpreted with the wrong byte order. + */ +1: eor x19, x19, #SCTLR_ELx_EE + bic x19, x19, #SCTLR_ELx_M + b.ne 2f + pre_disable_mmu_workaround + msr sctlr_el2, x19 + b 3f + pre_disable_mmu_workaround +2: msr sctlr_el1, x19 +3: isb + mov x19, xzr + ret SYM_CODE_END(record_mmu_state) /*Signature · 1 line
2.39.0
Catalin Marinas
maintainerQuoted 10 lines · Ard Biesheuvel
Ensure that the endianness used for populating the ID map matches the endianness that the running kernel will be using, as this is no longer guaranteed now that
create_idmap()is invoked beforeinit_kernel_el().Note that doing so is only safe if the MMU is off, as switching the endianness with the MMU on results in the active ID map to become invalid. So also clear the C and M bits when toggling the EE bit in SCTLR, and mark the MMU as disabled at boot.
[...]
Applied to arm64 (for-next/efi-boot-mmu-on), thanks!
[1/1] arm64: head: Switch endianness before populating the ID map
https://git.kernel.org/arm64/c/2ced0f30a426
Signature · 1 line
Catalin