arm64: Permit EFI boot with MMU and caches on
Ard Biesheuvel
maintainerv6 cover arm64: Permit EFI boot with MMU and caches on
The purpose of this series is to remove any explicit cache maintenance for coherency during early boot. Software managed coherency is error prone and tedious, and running with the MMU off is generally bad for performance, and it becomes unnecessary if we simply retain the cacheable 1:1 mapping of all of system RAM provided by EFI, and use it to populate the ID map page tables. After setting up this preliminary ID map, we disable the MMU, drop to EL1, reprogram the MAIR, TCR and SCTLR registers as before, and proceed as usual, avoiding the need for any manipulations of memory while the MMU and caches are off.
The only properties of the firmware provided 1:1 map we rely on is that it does not require any explicit cache maintenance for coherency, and that it covers the entire memory footprint of the image, including the BSS and padding at the end - all else is under control of the kernel itself, as before.
Patch #6 is optional, and moved to the end. It reduces the likelihood that an image needs to be moved around in memory only to align the swapper vmap'ed stack, but it only affects 64k granule configurations, and with KASLR, the image is usually copied around anyway.
Changes since v5:
- add a special entry point into the boot sequence that is to be used by EFI only, and only permit booting with the MMU enabled when using that boot path;
- omit the final patch that would need to go via the EFI tree in any case - adding the new entrypoint specific for EFI makes it conflict even more badly, and I'll try to revisit this during the merge window or simply defer the final piece for the next release;
Changes since v4:
- add patch to align the callers of finalise_el2()
- also clean HYP text to the PoC when booting at EL2 with the MMU on
- add a warning and a taint when doing non-EFI boot with the MMU and caches enabled
- rebase onto zboot changes in efi/next - this means that patches #6 and #7 will not apply onto arm64/for-next so a shared stable branch will be needed if we want to queue this up for v6.2
Changes since v3:
- drop EFI_LOADER_CODE memory type patch that has been queued in the mean time
- rebased onto [partial] series that moves efi-entry.S into the libstub/ source directory
- fixed a correctness issue in patch #2
Ard Biesheuvel (6):
arm64: head: Move all finalise_el2 calls to after __enable_mmu
arm64: kernel: move identity map out of .text mapping
arm64: head: record the MMU state at primary entry
arm64: head: avoid cache invalidation when entering with the MMU on
arm64: head: Clean the ID map and the HYP text to the PoC if needed
arm64: lds: reduce effective minimum image alignment to 64k
arch/arm64/include/asm/efi.h | 7 -- arch/arm64/kernel/head.S | 99 +++++++++++++++----- arch/arm64/kernel/image-vars.h | 2 +- arch/arm64/kernel/setup.c | 9 +- arch/arm64/kernel/sleep.S | 6 +- arch/arm64/kernel/vmlinux.lds.S | 13 ++- arch/arm64/mm/proc.S | 2 - drivers/firmware/efi/libstub/arm64-stub.c | 2 +- include/linux/efi.h | 6 +- 9 files changed, 104 insertions(+), 42 deletions(-)
Signature · 7 lines
2.35.1 _______________________________________________ linux-arm-kernel mailing list [email protected] http://lists.infradead.org/mailman/listinfo/linux-arm-kernel
Ard Biesheuvel
maintainerPATCH v6 1/6 arm64: head: Move all finalise_el2 calls to after __enable_mmu
In the primary boot path, finalise_el2() is called much later than on the secondary boot or resume-from-suspend paths, and this does not appear to be intentional.
Since we aim to do as little as possible before enabling the MMU and caches, align secondary and resume with primary boot, and defer the call to after the MMU is turned on. This also removes the need to clean finalise_el2() to the PoC once we enable support for booting with the MMU on.
--- arch/arm64/kernel/head.S | 5 ++++- arch/arm64/kernel/sleep.S | 5 ++++- 2 files changed, 8 insertions(+), 2 deletions(-)
arch/arm64/kernel/head.S +4 -1
@@ -587,7 +587,6 @@ SYM_FUNC_START_LOCAL(secondary_startup) * Common entry point for secondary CPUs. */ mov x20, x0 // preserve boot mode - bl finalise_el2 bl __cpu_secondary_check52bitva #if VA_BITS > 48 ldr_l x0, vabits_actual
@@ -603,6 +602,10 @@ SYM_FUNC_END(secondary_startup) SYM_FUNC_START_LOCAL(__secondary_switched) mov x0, x20 bl set_cpu_boot_mode_flag + + mov x0, x20 + bl finalise_el2 + str_l xzr, __early_cpu_boot_status, x3 adr_l x5, vectors msr vbar_el1, x5
arch/arm64/kernel/sleep.S +4 -1
@@ -100,7 +100,7 @@ SYM_FUNC_END(__cpu_suspend_enter) .pushsection ".idmap.text", "awx" SYM_CODE_START(cpu_resume) bl init_kernel_el - bl finalise_el2 + mov x19, x0 // preserve boot mode #if VA_BITS > 48 ldr_l x0, vabits_actual #endif
@@ -116,6 +116,9 @@ SYM_CODE_END(cpu_resume) .popsection SYM_FUNC_START(_cpu_resume) + mov x0, x19 + bl finalise_el2 + mrs x1, mpidr_el1 adr_l x8, mpidr_hash // x8 = struct mpidr_hash virt address
Signature · 7 lines
2.35.1 _______________________________________________ linux-arm-kernel mailing list [email protected] http://lists.infradead.org/mailman/listinfo/linux-arm-kernel
Ard Biesheuvel
maintainerPATCH v6 2/6 arm64: kernel: move identity map out of .text mapping
In reply to Ard BiesheuvelReorganize the ID map slightly so that only code that is executed with the MMU off or via the 1:1 mapping remains. This allows us to move the identity map out of the .text segment, as it will no longer need executable permissions via the kernel mapping.
--- arch/arm64/kernel/head.S | 28 +++++++++++--------- arch/arm64/kernel/vmlinux.lds.S | 2 +- arch/arm64/mm/proc.S | 2 -- 3 files changed, 16 insertions(+), 16 deletions(-)
arch/arm64/kernel/head.S +15 -13
@@ -543,19 +543,6 @@ SYM_INNER_LABEL(init_el2, SYM_L_LOCAL) eret SYM_FUNC_END(init_kernel_el) -/* - * Sets the __boot_cpu_mode flag depending on the CPU boot mode passed - * in w0. See arch/arm64/include/asm/virt.h for more info. - */ -SYM_FUNC_START_LOCAL(set_cpu_boot_mode_flag) - adr_l x1, __boot_cpu_mode - cmp w0, #BOOT_CPU_MODE_EL2 - b.ne 1f - add x1, x1, #4 -1: str w0, [x1] // Save CPU boot mode - ret -SYM_FUNC_END(set_cpu_boot_mode_flag) - /* * This provides a "holding pen" for platforms to hold all secondary * cores are held until we're ready for them to initialise.
@@ -599,6 +586,7 @@ SYM_FUNC_START_LOCAL(secondary_startup) br x8 SYM_FUNC_END(secondary_startup) + .text SYM_FUNC_START_LOCAL(__secondary_switched) mov x0, x20 bl set_cpu_boot_mode_flag
@@ -631,6 +619,19 @@ SYM_FUNC_START_LOCAL(__secondary_too_slow) b __secondary_too_slow SYM_FUNC_END(__secondary_too_slow) +/* + * Sets the __boot_cpu_mode flag depending on the CPU boot mode passed + * in w0. See arch/arm64/include/asm/virt.h for more info. + */ +SYM_FUNC_START_LOCAL(set_cpu_boot_mode_flag) + adr_l x1, __boot_cpu_mode + cmp w0, #BOOT_CPU_MODE_EL2 + b.ne 1f + add x1, x1, #4 +1: str w0, [x1] // Save CPU boot mode + ret +SYM_FUNC_END(set_cpu_boot_mode_flag) + /* * The booting CPU updates the failed status @__early_cpu_boot_status, * with MMU turned off.
@@ -662,6 +663,7 @@ SYM_FUNC_END(__secondary_too_slow) * Checks if the selected granule size is supported by the CPU. * If it isn't, park the CPU */ + .section ".idmap.text","awx" SYM_FUNC_START(__enable_mmu) mrs x3, ID_AA64MMFR0_EL1 ubfx x3, x3, #ID_AA64MMFR0_EL1_TGRAN_SHIFT, 4
arch/arm64/kernel/vmlinux.lds.S +1 -1
@@ -179,7 +179,6 @@ SECTIONS LOCK_TEXT KPROBES_TEXT HYPERVISOR_TEXT - IDMAP_TEXT *(.gnu.warning) . = ALIGN(16); *(.got) /* Global offset table */
@@ -206,6 +205,7 @@ SECTIONS TRAMP_TEXT HIBERNATE_TEXT KEXEC_TEXT + IDMAP_TEXT . = ALIGN(PAGE_SIZE); }
arch/arm64/mm/proc.S +0 -2
@@ -110,7 +110,6 @@ SYM_FUNC_END(cpu_do_suspend) * * x0: Address of context pointer */ - .pushsection ".idmap.text", "awx" SYM_FUNC_START(cpu_do_resume) ldp x2, x3, [x0] ldp x4, x5, [x0, #16]
@@ -166,7 +165,6 @@ alternative_else_nop_endif isb ret SYM_FUNC_END(cpu_do_resume) - .popsection #endif .pushsection ".idmap.text", "awx"
Signature · 7 lines
2.35.1 _______________________________________________ linux-arm-kernel mailing list [email protected] http://lists.infradead.org/mailman/listinfo/linux-arm-kernel
Ard Biesheuvel
maintainerPATCH v6 3/6 arm64: head: record the MMU state at primary entry
In reply to Ard BiesheuvelPrepare for being able to deal with primary entry with the MMU and caches enabled, by recording whether or not we entered with the MMU on in register x19 and in a global variable. (Note that writing to this variable does not require cache invalidation, nor is it required for storing the bootargs in that case, so omit the cache maintenance).
Since boot with the MMU enabled is not permitted by the bare metal boot protocol, ensure that a diagnostic is emitted and a taint bit set if the MMU was found to be enabled on a non-EFI boot. We will make an exception for EFI boot later, which has strict requirements for the mapping of system memory, permitting us to relax the boot protocol and hand over from the EFI stub to the core kernel with MMU and caches left enabled. To reduce the likelihood that bare metal boot will violate this requirement, introduce a separate entry point for EFI boot, which is different from the one that is invoked by branching to offset #0 in the image.
While at it, add 'pre_disable_mmu_workaround' macro invocations to init_kernel_el, as its manipulation of SCTLR_ELx may amount to disabling of the MMU after subsequent patches.
--- arch/arm64/kernel/head.S | 28 ++++++++++++++++++-- arch/arm64/kernel/image-vars.h | 2 +- arch/arm64/kernel/setup.c | 9 +++++-- 3 files changed, 34 insertions(+), 5 deletions(-)
arch/arm64/kernel/head.S +26 -2
@@ -77,6 +77,7 @@ * primary lowlevel boot path: * * Register Scope Purpose + * x19 primary_entry() .. start_kernel() whether we entered with the MMU on * x20 primary_entry() .. __primary_switch() CPU boot mode * x21 primary_entry() .. start_kernel() FDT pointer passed at boot in x0 * x22 create_idmap() .. start_kernel() ID map VA of the DT blob
@@ -85,8 +86,13 @@ * x25 primary_entry() .. start_kernel() supported VA size * x28 create_idmap() callee preserved temp register */ -SYM_CODE_START(primary_entry) - bl preserve_boot_args +SYM_CODE_START(efi_primary_entry) + bl record_mmu_state + b 0f + +SYM_INNER_LABEL(primary_entry, SYM_L_LOCAL) + mov x19, xzr // MMU must be off on bare metal boot +0: bl preserve_boot_args bl init_kernel_el // w0=cpu_boot_mode mov x20, x0 bl create_idmap
@@ -109,6 +115,18 @@ SYM_CODE_START(primary_entry) b __primary_switch SYM_CODE_END(primary_entry) +SYM_CODE_START_LOCAL(record_mmu_state) + mrs x19, CurrentEL + cmp x19, #CurrentEL_EL2 + mrs x19, sctlr_el1 + b.ne 0f + mrs x19, sctlr_el2 +0: tst x19, #SCTLR_ELx_C // Z := (C == 0) + and x19, x19, #SCTLR_ELx_M // isolate M bit + csel x19, xzr, x19, eq // clear x19 if Z + ret +SYM_CODE_END(record_mmu_state) + /* * Preserve the arguments passed by the bootloader in x0 .. x3 */
@@ -119,11 +137,14 @@ SYM_CODE_START_LOCAL(preserve_boot_args) stp x21, x1, [x0] // x0 .. x3 at kernel entry stp x2, x3, [x0, #16] + cbnz x19, 0f // skip cache invalidation if MMU is on dmb sy // needed before dc ivac with // MMU off add x1, x0, #0x20 // 4 x 8 bytes b dcache_inval_poc // tail call +0: str_l x19, mmu_enabled_at_boot, x0 + ret SYM_CODE_END(preserve_boot_args) SYM_FUNC_START_LOCAL(clear_page_tables)
@@ -497,6 +518,7 @@ SYM_FUNC_START(init_kernel_el) SYM_INNER_LABEL(init_el1, SYM_L_LOCAL) mov_q x0, INIT_SCTLR_EL1_MMU_OFF + pre_disable_mmu_workaround msr sctlr_el1, x0 isb mov_q x0, INIT_PSTATE_EL1
@@ -529,11 +551,13 @@ SYM_INNER_LABEL(init_el2, SYM_L_LOCAL) cbz x0, 1f /* Set a sane SCTLR_EL1, the VHE way */ + pre_disable_mmu_workaround msr_s SYS_SCTLR_EL12, x1 mov x2, #BOOT_CPU_FLAG_E2H b 2f 1: + pre_disable_mmu_workaround msr sctlr_el1, x1 mov x2, xzr 2:
arch/arm64/kernel/image-vars.h +1 -1
@@ -11,7 +11,7 @@#endif PROVIDE (__efistub_kernel_size = _edata - _text); -PROVIDE (__efistub_primary_entry_offset = primary_entry - _text); +PROVIDE (__efistub_primary_entry_offset = efi_primary_entry - _text);/* * The EFI stub has its own symbol namespace prefixed by __efistub_, to
arch/arm64/kernel/setup.c +7 -2
@@ -58,6 +58,7 @@ static int num_standard_resources;static struct resource *standard_resources;phys_addr_t __fdt_pointer__initdata ; +u64 mmu_enabled_at_boot__initdata ;/* * Standard memory resources
@@ -332,8 +333,12 @@ void __init __no_sanitize_address setup_arch(char **cmdline_p)xen_early_init ();efi_init (); -if (!efi_enabled (EFI_BOOT ) && ((u64 )_text %MIN_KIMG_ALIGN ) !=0 ) -pr_warn (FW_BUG "Kernel image misaligned at boot, please fix your bootloader!" ); +if (!efi_enabled (EFI_BOOT )) { +if ((u64 )_text %MIN_KIMG_ALIGN ) +pr_warn (FW_BUG "Kernel image misaligned at boot, please fix your bootloader!" ); +WARN_TAINT (mmu_enabled_at_boot,TAINT_FIRMWARE_WORKAROUND , +FW_BUG "Booted with MMU enabled!" ); + }arm64_memblock_init ();
Signature · 7 lines
2.35.1 _______________________________________________ linux-arm-kernel mailing list [email protected] http://lists.infradead.org/mailman/listinfo/linux-arm-kernel
Ard Biesheuvel
maintainerRe: [PATCH v6 3/6] arm64: head: record the MMU state at primary entry
Quoted 7 lines · Ard Biesheuvel
Prepare for being able to deal with primary entry with the MMU and caches enabled, by recording whether or not we entered with the MMU on in register x19 and in a global variable. (Note that writing to this variable does not require cache invalidation, nor is it required for storing the bootargs in that case, so omit the cache maintenance).
Quoted 11 lines
Since boot with the MMU enabled is not permitted by the bare metal boot protocol, ensure that a diagnostic is emitted and a taint bit set if the MMU was found to be enabled on a non-EFI boot. We will make an exception for EFI boot later, which has strict requirements for the mapping of system memory, permitting us to relax the boot protocol and hand over from the EFI stub to the core kernel with MMU and caches left enabled. To reduce the likelihood that bare metal boot will violate this requirement, introduce a separate entry point for EFI boot, which is different from the one that is invoked by branching to offset #0 in the image.
This is actually much more fiddly than I though:
- the existing arm64 stub can link to any symbol so there, it does not make any difference
- the zboot stub needs to find the right entry point into the image, as the one pointed to by the header is no longer correct.
I confused myself into thinking that we can just grab it from the PE/COFF header, but that is the header point *into* the EFI stub, not the one that the stub calls on its way out.
Grabbing stuff from the PE/COFF header is needed anyway, and can be done in a generic manner.
Grabbing a special EFI-only entry point means mangling the ELF to find the relative offset from the start of the binary representation of the image, and passing that information along with the compressed blob to the decompressor.
So the options are:
- go back to the old approach (and add a panic() in the right place)
- add a branch to the EFI entrypoint at a known offset (in the image header?)
- other ideas?
Quoted 136 lines
While at it, add '
pre_disable_mmu_workaround' macro invocations toinit_kernel_el, as its manipulation ofSCTLR_ELxmay amount to disabling of the MMU after subsequent patches.Signed-off-by Ard Biesheuvel <[email protected]>--- arch/arm64/kernel/head.S | 28 ++++++++++++++++++-- arch/arm64/kernel/image-vars.h | 2 +- arch/arm64/kernel/setup.c | 9 +++++-- 3 files changed, 34 insertions(+), 5 deletions(-)arch/arm64/kernel/head.S +26
-2@@ -77,6 +77,7 @@ * primary lowlevel boot path: * * Register Scope Purpose + * x19 primary_entry() .. start_kernel() whether we entered with the MMU on * x20 primary_entry() .. __primary_switch() CPU boot mode * x21 primary_entry() .. start_kernel() FDT pointer passed at boot in x0 * x22 create_idmap() .. start_kernel() ID map VA of the DT blob@@ -85,8 +86,13 @@ * x25 primary_entry() .. start_kernel() supported VA size * x28 create_idmap() callee preserved temp register */ -SYM_CODE_START(primary_entry) - bl preserve_boot_args +SYM_CODE_START(efi_primary_entry) + bl record_mmu_state + b 0f + +SYM_INNER_LABEL(primary_entry, SYM_L_LOCAL) + mov x19, xzr // MMU must be off on bare metal boot +0: bl preserve_boot_args bl init_kernel_el // w0=cpu_boot_mode mov x20, x0 bl create_idmap@@ -109,6 +115,18 @@ SYM_CODE_START(primary_entry) b __primary_switch SYM_CODE_END(primary_entry) +SYM_CODE_START_LOCAL(record_mmu_state) + mrs x19, CurrentEL + cmp x19, #CurrentEL_EL2 + mrs x19, sctlr_el1 + b.ne 0f + mrs x19, sctlr_el2 +0: tst x19, #SCTLR_ELx_C // Z := (C == 0) + and x19, x19, #SCTLR_ELx_M // isolate M bit + csel x19, xzr, x19, eq // clear x19 if Z + ret +SYM_CODE_END(record_mmu_state) + /* * Preserve the arguments passed by the bootloader in x0 .. x3 */@@ -119,11 +137,14 @@ SYM_CODE_START_LOCAL(preserve_boot_args) stp x21, x1, [x0] // x0 .. x3 at kernel entry stp x2, x3, [x0, #16] + cbnz x19, 0f // skip cache invalidation if MMU is on dmb sy // needed before dc ivac with // MMU off add x1, x0, #0x20 // 4 x 8 bytes b dcache_inval_poc // tail call +0: str_l x19, mmu_enabled_at_boot, x0 + ret SYM_CODE_END(preserve_boot_args) SYM_FUNC_START_LOCAL(clear_page_tables)@@ -497,6 +518,7 @@ SYM_FUNC_START(init_kernel_el) SYM_INNER_LABEL(init_el1, SYM_L_LOCAL) mov_q x0, INIT_SCTLR_EL1_MMU_OFF + pre_disable_mmu_workaround msr sctlr_el1, x0 isb mov_q x0, INIT_PSTATE_EL1@@ -529,11 +551,13 @@ SYM_INNER_LABEL(init_el2, SYM_L_LOCAL) cbz x0, 1f /* Set a sane SCTLR_EL1, the VHE way */ + pre_disable_mmu_workaround msr_s SYS_SCTLR_EL12, x1 mov x2, #BOOT_CPU_FLAG_E2H b 2f 1: + pre_disable_mmu_workaround msr sctlr_el1, x1 mov x2, xzr 2:arch/arm64/kernel/image-vars.h +1
-1@@ -11,7 +11,7 @@#endif PROVIDE (__efistub_kernel_size = _edata - _text); -PROVIDE (__efistub_primary_entry_offset = primary_entry - _text); +PROVIDE (__efistub_primary_entry_offset = efi_primary_entry - _text);/* * The EFI stub has its own symbol namespace prefixed by __efistub_, to arch/arm64/kernel/setup.c +7
-3@@ -58,6 +58,7 @@ static int num_standard_resources;static struct resource *standard_resources;phys_addr_t __fdt_pointer__initdata ; +u64 mmu_enabled_at_boot__initdata ;/* * Standard memory resources @@ -332,8 +333,12 @@ void __init __no_sanitize_address setup_arch(char **cmdline_p)xen_early_init ();efi_init (); -if (!efi_enabled (EFI_BOOT ) && ((u64 )_text %MIN_KIMG_ALIGN ) !=0 ) -pr_warn (FW_BUG "Kernel image misaligned at boot, please fix your bootloader!" ); +if (!efi_enabled (EFI_BOOT )) { +if ((u64 )_text %MIN_KIMG_ALIGN ) +pr_warn (FW_BUG "Kernel image misaligned at boot, please fix your bootloader!" ); +WARN_TAINT (mmu_enabled_at_boot,TAINT_FIRMWARE_WORKAROUND , +FW_BUG "Booted with MMU enabled!" ); + }arm64_memblock_init (); --
Quoted 2 lines
2.35.1
_______________________________________________
linux-arm-kernel mailing list
[email protected]
http://lists.infradead.org/mailman/listinfo/linux-arm-kernel
Will Deacon
maintainerRe: [PATCH v6 3/6] arm64: head: record the MMU state at primary entry
Quoted 37 lines · Ard Biesheuvel
Quoted 7 lines · Ard Biesheuvel
Prepare for being able to deal with primary entry with the MMU and caches enabled, by recording whether or not we entered with the MMU on in register x19 and in a global variable. (Note that writing to this variable does not require cache invalidation, nor is it required for storing the bootargs in that case, so omit the cache maintenance).
Quoted 11 lines
Since boot with the MMU enabled is not permitted by the bare metal boot protocol, ensure that a diagnostic is emitted and a taint bit set if the MMU was found to be enabled on a non-EFI boot. We will make an exception for EFI boot later, which has strict requirements for the mapping of system memory, permitting us to relax the boot protocol and hand over from the EFI stub to the core kernel with MMU and caches left enabled. To reduce the likelihood that bare metal boot will violate this requirement, introduce a separate entry point for EFI boot, which is different from the one that is invoked by branching to offset #0 in the image.
This is actually much more fiddly than I though:
- the existing arm64 stub can link to any symbol so there, it does not make any difference
- the zboot stub needs to find the right entry point into the image, as the one pointed to by the header is no longer correct.I confused myself into thinking that we can just grab it from the PE/COFF header, but that is the header point *into* the EFI stub, not the one that the stub calls on its way out.
Grabbing stuff from the PE/COFF header is needed anyway, and can be done in a generic manner.
Grabbing a special EFI-only entry point means mangling the ELF to find the relative offset from the start of the binary representation of the image, and passing that information along with the compressed blob to the decompressor.
Quoted 2 lines
So the options are:
- go back to the old approach (and add apanic()in the right place)
I'm ok with that, but I'd like an Ack from Mark as well.
Cheers,
Will
_______________________________________________
linux-arm-kernel mailing list
[email protected]
http://lists.infradead.org/mailman/listinfo/linux-arm-kernel
Ard Biesheuvel
maintainerPATCH v6 4/6 arm64: head: avoid cache invalidation when entering with the MMU on
In reply to Ard BiesheuvelIf we enter with the MMU on, there is no need for explicit cache invalidation for stores to memory, as they will be coherent with the caches.
Let's take advantage of this, and create the ID map with the MMU still enabled if that is how we entered, and avoid any cache invalidation calls in that case.
--- arch/arm64/kernel/head.S | 5 +++-- 1 file changed, 3 insertions(+), 2 deletions(-)
arch/arm64/kernel/head.S +3 -2
@@ -93,9 +93,9 @@ SYM_CODE_START(efi_primary_entry) SYM_INNER_LABEL(primary_entry, SYM_L_LOCAL) mov x19, xzr // MMU must be off on bare metal boot 0: bl preserve_boot_args + bl create_idmap bl init_kernel_el // w0=cpu_boot_mode mov x20, x0 - bl create_idmap /* * The following calls CPU setup code, see arch/arm64/mm/proc.S for
@@ -381,12 +381,13 @@ SYM_FUNC_START_LOCAL(create_idmap) * accesses (MMU disabled), invalidate those tables again to * remove any speculatively loaded cache lines. */ + cbnz x19, 0f // skip cache invalidation if MMU is on dmb sy adrp x0, init_idmap_pg_dir adrp x1, init_idmap_pg_end bl dcache_inval_poc - ret x28 +0: ret x28 SYM_FUNC_END(create_idmap) SYM_FUNC_START_LOCAL(create_kernel_mapping)
Signature · 7 lines
2.35.1 _______________________________________________ linux-arm-kernel mailing list [email protected] http://lists.infradead.org/mailman/listinfo/linux-arm-kernel
Ard Biesheuvel
maintainerPATCH v6 5/6 arm64: head: Clean the ID map and the HYP text to the PoC if needed
In reply to Ard BiesheuvelIf we enter with the MMU and caches enabled, the bootloader may not have performed any cache maintenance to the PoC. So clean the ID mapped page to the PoC, to ensure that instruction and data accesses with the MMU off see the correct data. For similar reasons, clean all the HYP text to the PoC as well when entering at EL2 with the MMU and caches enabled.
Note that this means primary_entry() itself needs to be moved into the ID map as well, as we will return from init_kernel_el() with the MMU and caches off.
--- arch/arm64/kernel/head.S | 30 ++++++++++++++++++-- arch/arm64/kernel/sleep.S | 1 + 2 files changed, 28 insertions(+), 3 deletions(-)
arch/arm64/kernel/head.S +27 -3
@@ -90,10 +90,22 @@ SYM_CODE_START(efi_primary_entry) bl record_mmu_state b 0f + .section ".idmap.text","awx" SYM_INNER_LABEL(primary_entry, SYM_L_LOCAL) mov x19, xzr // MMU must be off on bare metal boot 0: bl preserve_boot_args bl create_idmap + + /* + * If we entered with the MMU and caches on, clean the ID mapped part + * of the primary boot code to the PoC so we can safely execute it with + * the MMU off. + */ + cbz x19, 1f + adrp x0, __idmap_text_start + adr_l x1, __idmap_text_end + bl dcache_clean_poc +1: mov x0, x19 bl init_kernel_el // w0=cpu_boot_mode mov x20, x0
@@ -115,6 +127,7 @@ SYM_INNER_LABEL(primary_entry, SYM_L_LOCAL) b __primary_switch SYM_CODE_END(primary_entry) + __INIT SYM_CODE_START_LOCAL(record_mmu_state) mrs x19, CurrentEL cmp x19, #CurrentEL_EL2
@@ -511,10 +524,12 @@ SYM_FUNC_END(__primary_switched) * Returns either BOOT_CPU_MODE_EL1 or BOOT_CPU_MODE_EL2 in x0 if * booted in EL1 or EL2 respectively, with the top 32 bits containing * potential context flags. These flags are *not* stored in __boot_cpu_mode. + * + * x0: whether we are being called from the primary boot path with the MMU on */ SYM_FUNC_START(init_kernel_el) - mrs x0, CurrentEL - cmp x0, #CurrentEL_EL2 + mrs x1, CurrentEL + cmp x1, #CurrentEL_EL2 b.eq init_el2 SYM_INNER_LABEL(init_el1, SYM_L_LOCAL)
@@ -529,6 +544,14 @@ SYM_INNER_LABEL(init_el1, SYM_L_LOCAL) eret SYM_INNER_LABEL(init_el2, SYM_L_LOCAL) + msr elr_el2, lr + + // clean all HYP code to the PoC if we booted at EL2 with the MMU on + cbz x0, 0f + adrp x0, __hyp_idmap_text_start + adr_l x1, __hyp_text_end + bl dcache_clean_poc +0: mov_q x0, HCR_HOST_NVHE_FLAGS msr hcr_el2, x0 isb
@@ -562,7 +585,6 @@ SYM_INNER_LABEL(init_el2, SYM_L_LOCAL) msr sctlr_el1, x1 mov x2, xzr 2: - msr elr_el2, lr mov w0, #BOOT_CPU_MODE_EL2 orr x0, x0, x2 eret
@@ -573,6 +595,7 @@ SYM_FUNC_END(init_kernel_el) * cores are held until we're ready for them to initialise. */ SYM_FUNC_START(secondary_holding_pen) + mov x0, xzr bl init_kernel_el // w0=cpu_boot_mode mrs x2, mpidr_el1 mov_q x1, MPIDR_HWID_BITMASK
@@ -590,6 +613,7 @@ SYM_FUNC_END(secondary_holding_pen) * be used where CPUs are brought online dynamically by the kernel. */ SYM_FUNC_START(secondary_entry) + mov x0, xzr bl init_kernel_el // w0=cpu_boot_mode b secondary_startup SYM_FUNC_END(secondary_entry)
arch/arm64/kernel/sleep.S +1 -0
@@ -99,6 +99,7 @@ SYM_FUNC_END(__cpu_suspend_enter) .pushsection ".idmap.text", "awx" SYM_CODE_START(cpu_resume) + mov x0, xzr bl init_kernel_el mov x19, x0 // preserve boot mode #if VA_BITS > 48
Signature · 7 lines
2.35.1 _______________________________________________ linux-arm-kernel mailing list [email protected] http://lists.infradead.org/mailman/listinfo/linux-arm-kernel
Ard Biesheuvel
maintainerPATCH v6 6/6 arm64: lds: reduce effective minimum image alignment to 64k
In reply to Ard BiesheuvelOur segment alignment is 64k for all configurations, and coincidentally, this is the largest alignment supported by the PE/COFF executable format used by EFI. This means that generally, there is no need to move the image around in memory after it has been loaded by the firmware, which can be advantageous as it also permits us to rely on the memory attributes set by the firmware (R-X for [_text, __inittext_end] and RW- for [__initdata_begin, _end].
However, the minimum alignment of the image is actually 128k on 64k pages configurations with CONFIG_VMAP_STACK=y, due to the existence of a single 128k aligned object in the image, which is the stack of the init task.
Let's work around this by adding some padding before the init stack allocation, so we can round down the stack pointer to a suitably aligned value if the image is not aligned to 128k in memory.
Note that this does not affect the boot protocol, which still requires 2 MiB alignment for bare metal boot, but is only part of the internal contract between the EFI stub and the kernel proper.
--- arch/arm64/include/asm/efi.h | 7 ------- arch/arm64/kernel/head.S | 3 +++ arch/arm64/kernel/vmlinux.lds.S | 11 ++++++++++- drivers/firmware/efi/libstub/arm64-stub.c | 2 +- include/linux/efi.h | 6 +----- 5 files changed, 15 insertions(+), 14 deletions(-)
arch/arm64/include/asm/efi.h +0 -7
@@ -62,13 +62,6 @@ efi_status_t __efi_rt_asm_wrapper(void *, const char *, ...);/* arch specific definitions used by the stub code */ -/* -* In some configurations (e.g. VMAP_STACK && 64K pages), stacks built into the -* kernel need greater alignment than we require the segments to be padded to. -*/ -#define EFI_KIMG_ALIGN \ - (SEGMENT_ALIGN > THREAD_ALIGN ? SEGMENT_ALIGN : THREAD_ALIGN) -/* * On arm64, we have to ensure that the initrd ends up in the linear region, * which is a 1 GB aligned region of size '1UL << (VA_BITS_MIN - 1)' that is
arch/arm64/kernel/head.S +3 -0
@@ -433,6 +433,9 @@ SYM_FUNC_END(create_kernel_mapping) msr sp_el0, \tsk ldr \tmp1, [\tsk, #TSK_STACK] +#if THREAD_ALIGN > SEGMENT_ALIGN + bic \tmp1, \tmp1, #THREAD_ALIGN - 1 +#endif add sp, \tmp1, #THREAD_SIZE sub sp, sp, #PT_REGS_SIZE
arch/arm64/kernel/vmlinux.lds.S +10 -1
@@ -287,7 +287,16 @@ SECTIONS _data = .; _sdata = .; - RW_DATA(L1_CACHE_BYTES, PAGE_SIZE, THREAD_ALIGN) +#if THREAD_ALIGN > SEGMENT_ALIGN + /* + * Add some padding for the init stack so we can fix up any potential + * misalignment at runtime. In practice, this can only occur on 64k + * pages configurations with CONFIG_VMAP_STACK=y. + */ + . += THREAD_ALIGN - SEGMENT_ALIGN; + ASSERT(. == init_stack, "init_stack not at start of RW_DATA as expected") +#endif + RW_DATA(L1_CACHE_BYTES, PAGE_SIZE, SEGMENT_ALIGN) /* * Data written with the MMU off but read with the MMU on requires
drivers/firmware/efi/libstub/arm64-stub.c +1 -1
@@ -97,7 +97,7 @@ efi_status_t handle_kernel_image(unsigned long *image_addr,* 2M alignment if KASLR was explicitly disabled, even if it was not * going to be activated to begin with. */ -u64 min_kimg_align = efi_nokaslr ?MIN_KIMG_ALIGN :EFI_KIMG_ALIGN ; +u64 min_kimg_align = efi_nokaslr ?MIN_KIMG_ALIGN :SEGMENT_ALIGN ;if (IS_ENABLED (CONFIG_RANDOMIZE_BASE )) {efi_guid_t li_fixed_proto =LINUX_EFI_LOADED_IMAGE_FIXED_GUID ;
include/linux/efi.h +1 -5
@@ -421,11 +421,7 @@ void efi_native_runtime_setup(void);/* * This GUID may be installed onto the kernel image's handle as a NULL protocol * to signal to the stub that the placement of the image should be respected, -* and moving the image in physical memory is undesirable. To ensure -* compatibility with 64k pages kernels with virtually mapped stacks, and to -* avoid defeating physical randomization, this protocol should only be -* installed if the image was placed at a randomized 128k aligned address in -* memory. +* and moving the image in physical memory is undesirable. */ #define LINUX_EFI_LOADED_IMAGE_FIXED_GUID EFI_GUID(0xf5a37b6d, 0x3344, 0x42a5, 0xb6, 0xbb, 0x97, 0x86, 0x48, 0xc1, 0x89, 0x0a)
Signature · 7 lines
2.35.1 _______________________________________________ linux-arm-kernel mailing list [email protected] http://lists.infradead.org/mailman/listinfo/linux-arm-kernel