arm64: efi: leave MMU and caches on at boot
Ard Biesheuvel
maintainercover arm64: efi: leave MMU and caches on at boot
Currently, booting via the EFI stub involves a fair amount of cache maintenance, as well as two attempts at setting up the page tables when KASLR is enabled, which runs with the MMU and caches off. This may be problematic for virtualization payloads, which sometimes have very tight boot time budgets, and run in a context where uncached accesses to memory are problematic, as they are incoherent with the accesses of the host VMM or the hypervisor.
While it makes sense for the boot protocol to be pedantic in this respect, EFI boot is tightly coupled with the core kernel, given that the EFI stub is the de facto bootloader, which is part of the same kernel image. It is therefore justified to cut some corners and simplify things substantially.
As UEFI guarantees that all memory is 1:1 mapped with inner/outer writeback cacheable attributes, there is really no need to clean the entire kernel image to the PoC, or do all the cache invalidation that is required to ensure that page tables and other state held in memory is not corrupted by inadvertent writeback of stale dirty cache lines.
Since we setup and teardown page tables redundantly for KASLR, fix this first, by permitting the KASLR seed to be passed via register X1 as well as the /chosen/kaslr-seed property. This permits us the run the page table setup code with the firmware's cached 1:1 mapping active, removing the need for any cache invalidation.
Then, ensure that the only piece of code we do execute with the MMU off in this case is cleaned to the PoC, which is all held in a single page. (the ID map)
Finally, drop all the cache maintenance from the EFI entry code when booting at EL1, and just branch to the kernel with the firmware's 1:1 cached mapping live.
Ard Biesheuvel (8):
arm64: kaslr: deal with init called with VA randomization enabled
arm64: head: record the MMU state at primary entry
arm64: head: take KASLR seed passed via x1 into account
arm64: head: avoid cache invalidation when entering with the MMU on
arm64: head: populate page tables before disabling the MMU
arm64: head: clean the ID map page to the PoC
arm64: lds: move idmap_pg_dir out of .rodata
arm64: efi: leave MMU and caches on when handing over to the core
kernel
arch/arm64/kernel/efi-entry.S | 20 ++--- arch/arm64/kernel/head.S | 82 +++++++++++++++++--- arch/arm64/kernel/kaslr.c | 8 +- arch/arm64/kernel/vmlinux.lds.S | 8 +- drivers/firmware/efi/libstub/fdt.c | 6 +- 5 files changed, 95 insertions(+), 29 deletions(-)
Signature · 7 lines
2.30.2 _______________________________________________ linux-arm-kernel mailing list [email protected] http://lists.infradead.org/mailman/listinfo/linux-arm-kernel
Ard Biesheuvel
maintainerRFC PATCH 1/8 arm64: kaslr: deal with init called with VA randomization enabled
We will pass the KASLR seed via x1 as well as via the device tree, so that the boot code does not need to create the kernel mapping twice.
This means we may enter kaslr_early_init() fully randomized, and the offset calculated by this function should be identical to the offset that has already been taken into account.
This means that taking the address of _end or _etext and adding offset to it produces the wrong value, given that _end and _etext references will have been fixed up already, and therefore already incorporate offset.
So instead of referring to these symbols directly, use their offsets relative to _text, which should produce values that depend on the size and layout of the Image only. Then, add KIMAGE_VADDR to obtain the unrandomized values.
--- arch/arm64/kernel/kaslr.c | 8 ++++++-- 1 file changed, 6 insertions(+), 2 deletions(-)
arch/arm64/kernel/kaslr.c +6 -2
@@ -145,6 +145,8 @@ u64 __init kaslr_early_init(void)return offset %SZ_2G ;if (IS_ENABLED (CONFIG_RANDOMIZE_MODULE_REGION_FULL )) { +u64 end = (u64 )_end - (u64 )_text +KIMAGE_VADDR ; +/* * Randomize the module region over a 2 GB window covering the * kernel. This reduces the risk of modules leaking information
@@ -154,9 +156,11 @@ u64 __init kaslr_early_init(void)* resolved normally.) */ module_range =SZ_2G - (u64)(_end - _stext); - module_alloc_base =max ((u64 )_end + offset -SZ_2G , + module_alloc_base =max (end + offset -SZ_2G , (u64 )MODULES_VADDR ); }else { +u64 end = (u64 )_etext - (u64 )_text +KIMAGE_VADDR ; +/* * Randomize the module region by setting module_alloc_base to * a PAGE_SIZE multiple in the range [_etext - MODULES_VSIZE,
@@ -167,7 +171,7 @@ u64 __init kaslr_early_init(void)* when ARM64_MODULE_PLTS is enabled. */ module_range =MODULES_VSIZE - (u64)(_etext - _stext); - module_alloc_base = (u64 )_etext + offset -MODULES_VSIZE ; + module_alloc_base = end + offset -MODULES_VSIZE ; }/* use the lower 21 bits to randomize the base of the module region */
Signature · 7 lines
2.30.2 _______________________________________________ linux-arm-kernel mailing list [email protected] http://lists.infradead.org/mailman/listinfo/linux-arm-kernel
Ard Biesheuvel
maintainerRFC PATCH 2/8 arm64: head: record the MMU state at primary entry
In reply to Ard BiesheuvelPrepare for being able to deal with primary entry with the MMU and caches enabled, by recording whether or not we entered at EL1 with the MMU on in register x25.
While at it, add the disable_mmu_workaround macro invocation to init_kernel_el, as its manipulation of SCTLR_EL1 may come down to disabling of the MMU after subsequent patches.
--- arch/arm64/kernel/head.S | 13 +++++++++++++ 1 file changed, 13 insertions(+)
arch/arm64/kernel/head.S +13 -0
@@ -87,8 +87,10 @@ * x28 __create_page_tables() callee preserved temp register * x19/x20 __primary_switch() callee preserved temp registers * x24 __primary_switch() .. relocate_kernel() current RELR displacement + * x25 primary_entry() .. start_kernel() whether we entered at EL1 with the MMU on */ SYM_CODE_START(primary_entry) + bl record_mmu_state bl preserve_boot_args bl init_kernel_el // w0=cpu_boot_mode adrp x23, __PHYS_OFFSET
@@ -105,6 +107,16 @@ SYM_CODE_START(primary_entry) b __primary_switch SYM_CODE_END(primary_entry) +SYM_CODE_START_LOCAL(record_mmu_state) + mrs x25, CurrentEL + cmp x25, #CurrentEL_EL2 + b.eq 0f + mrs x25, sctlr_el1 + tst x25, #SCTLR_ELx_M +0: cset w25, ne + ret +SYM_CODE_END(record_mmu_state) + /* * Preserve the arguments passed by the bootloader in x0 .. x3 */
@@ -500,6 +512,7 @@ SYM_FUNC_START(init_kernel_el) SYM_INNER_LABEL(init_el1, SYM_L_LOCAL) mov_q x0, INIT_SCTLR_EL1_MMU_OFF + pre_disable_mmu_workaround msr sctlr_el1, x0 isb mov_q x0, INIT_PSTATE_EL1
Signature · 7 lines
2.30.2 _______________________________________________ linux-arm-kernel mailing list [email protected] http://lists.infradead.org/mailman/listinfo/linux-arm-kernel
Ard Biesheuvel
maintainerRFC PATCH 3/8 arm64: head: take KASLR seed passed via x1 into account
In reply to Ard BiesheuvelIf the kernel was entered with the MMU on, use the value passed via register x1 as the seed for virtual randomization rather than the one passed via the /chosen/kaslr-seed DT property. This removes the need to map and unmap the kernel just to access the DT.
Note that kaslr_early_init() still needs to be called exactly once in this case, to set the other KASLR related state, so ensure that this happens when called with the MMU on.
--- arch/arm64/kernel/head.S | 28 ++++++++++++++++++-- 1 file changed, 26 insertions(+), 2 deletions(-)
arch/arm64/kernel/head.S +26 -2
@@ -91,10 +91,9 @@ */ SYM_CODE_START(primary_entry) bl record_mmu_state + bl record_kaslr_offset bl preserve_boot_args bl init_kernel_el // w0=cpu_boot_mode - adrp x23, __PHYS_OFFSET - and x23, x23, MIN_KIMG_ALIGN - 1 // KASLR offset, defaults to 0 bl set_cpu_boot_mode_flag bl __create_page_tables /*
@@ -117,6 +116,29 @@ SYM_CODE_START_LOCAL(record_mmu_state) ret SYM_CODE_END(record_mmu_state) +SYM_CODE_START_LOCAL(record_kaslr_offset) + adrp x23, __PHYS_OFFSET + and x23, x23, MIN_KIMG_ALIGN - 1 // KASLR offset, defaults to 0 + cbz x25, 0f // skip if MMU is disabled +#ifdef CONFIG_RANDOMIZE_BASE + /* + * Calculate the virtual randomization that will be applied based on + * the KASLR seed passed via x1, if provided. + */ + and x4, x1, ~(MIN_KIMG_ALIGN - 1) + and x4, x4, #(1 << (VA_BITS_MIN - 2)) - 1 + mov x5, #1 << (VA_BITS_MIN - 3) + add x4, x4, x5 +#if !defined(CONFIG_KASAN_VMALLOC) && \ + (defined(CONFIG_KASAN_GENERIC) || defined(CONFIG_KASAN_SW_TAGS)) + and x4, x4, #(1 << 31) - 1 // limit to 2G +#endif + orr x23, x23, x4 + mov x1, xzr // wipe the seed from x1 +#endif +0: ret +SYM_CODE_END(record_kaslr_offset) + /* * Preserve the arguments passed by the bootloader in x0 .. x3 */
@@ -467,8 +489,10 @@ SYM_FUNC_START_LOCAL(__primary_switched) bl init_feature_override // Parse cpu feature overrides #ifdef CONFIG_RANDOMIZE_BASE tst x23, ~(MIN_KIMG_ALIGN - 1) // already running randomized? + ccmp x25, #1, #4, ne // and we booted with the MMU off? b.ne 0f bl kaslr_early_init // parse FDT for KASLR options + cbnz x25, 0f // MMU was on? cbz x0, 0f // KASLR disabled? just proceed orr x23, x23, x0 // record KASLR offset ldp x29, x30, [sp], #16 // we must enable KASLR, return
Signature · 7 lines
2.30.2 _______________________________________________ linux-arm-kernel mailing list [email protected] http://lists.infradead.org/mailman/listinfo/linux-arm-kernel
Ard Biesheuvel
maintainerRFC PATCH 4/8 arm64: head: avoid cache invalidation when entering with the MMU on
In reply to Ard BiesheuvelThe primary entry code populates memory with the MMU and caches disabled, and therefore needs to go out of its way to prevent dirty but stale cachelines from potentially corrupting these memory contents inadvertently.
When entering with the MMU on, this is not needed, so skip it.
While at it, renumber some asm labels to avoid confusion.
--- arch/arm64/kernel/head.S | 23 ++++++++++++++------ 1 file changed, 16 insertions(+), 7 deletions(-)
arch/arm64/kernel/head.S +16 -7
@@ -149,11 +149,13 @@ SYM_CODE_START_LOCAL(preserve_boot_args) stp x21, x1, [x0] // x0 .. x3 at kernel entry stp x2, x3, [x0, #16] + cbnz x25, 0f // skip cache invalidation if MMU is on dmb sy // needed before dc ivac with // MMU off add x1, x0, #0x20 // 4 x 8 bytes b dcache_inval_poc // tail call +0: ret SYM_CODE_END(preserve_boot_args) /*
@@ -296,6 +298,8 @@ SYM_CODE_END(preserve_boot_args) SYM_FUNC_START_LOCAL(__create_page_tables) mov x28, lr + cbnz x25, 0f // skip cache invalidation if MMU is on + /* * Invalidate the init page tables to avoid potential dirty cache lines * being evicted. Other page tables are allocated in rodata as part of
@@ -309,7 +313,7 @@ SYM_FUNC_START_LOCAL(__create_page_tables) /* * Clear the init page tables. */ - adrp x0, init_pg_dir +0: adrp x0, init_pg_dir adrp x1, init_pg_end sub x1, x1, x0 1: stp xzr, xzr, [x0], #16
@@ -331,15 +335,16 @@ SYM_FUNC_START_LOCAL(__create_page_tables) mrs_s x6, SYS_ID_AA64MMFR2_EL1 and x6, x6, #(0xf << ID_AA64MMFR2_LVA_SHIFT) mov x5, #52 - cbnz x6, 1f + cbnz x6, 2f #endif mov x5, #VA_BITS_MIN -1: +2: adr_l x6, vabits_actual str x5, [x6] + cbnz x25, 3f // skip cache invalidation if MMU is on dmb sy dc ivac, x6 // Invalidate potentially stale cache line - +3: /* * VA_BITS may be too small to allow for an ID mapping to be created * that covers system RAM if that is located sufficiently high in the
@@ -355,12 +360,14 @@ SYM_FUNC_START_LOCAL(__create_page_tables) adrp x5, __idmap_text_end clz x5, x5 cmp x5, TCR_T0SZ(VA_BITS_MIN) // default T0SZ small enough? - b.ge 1f // .. then skip VA range extension + b.ge 5f // .. then skip VA range extension adr_l x6, idmap_t0sz str x5, [x6] + cbnz x25, 4f // skip cache invalidation if MMU is on dmb sy dc ivac, x6 // Invalidate potentially stale cache line +4: #if (VA_BITS < 48) #define EXTRA_SHIFT (PGDIR_SHIFT + PAGE_SHIFT - 3)
@@ -387,7 +394,7 @@ SYM_FUNC_START_LOCAL(__create_page_tables) mov x4, #1 << (PHYS_MASK_SHIFT - PGDIR_SHIFT) str_l x4, idmap_ptrs_per_pgd, x5 #endif -1: +5: ldr_l x4, idmap_ptrs_per_pgd adr_l x6, __idmap_text_end // __pa(__idmap_text_end)
@@ -407,6 +414,8 @@ SYM_FUNC_START_LOCAL(__create_page_tables) map_memory x0, x1, x5, x6, x7, x3, x4, x10, x11, x12, x13, x14 + cbnz x25, 4f // skip cache invalidation if MMU is on + /* * Since the page tables have been populated with non-cacheable * accesses (MMU disabled), invalidate those tables again to
@@ -422,7 +431,7 @@ SYM_FUNC_START_LOCAL(__create_page_tables) adrp x1, init_pg_end bl dcache_inval_poc - ret x28 +4: ret x28 SYM_FUNC_END(__create_page_tables) /*
Signature · 7 lines
2.30.2 _______________________________________________ linux-arm-kernel mailing list [email protected] http://lists.infradead.org/mailman/listinfo/linux-arm-kernel
Ard Biesheuvel
maintainerRFC PATCH 5/8 arm64: head: populate page tables before disabling the MMU
In reply to Ard BiesheuvelReorder the call to init_kernel_el() with the creation of the page tables so the latter code runs with MMU and caches enabled if that is how we entered.
--- arch/arm64/kernel/head.S | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-)
arch/arm64/kernel/head.S +1 -1
@@ -93,9 +93,9 @@ SYM_CODE_START(primary_entry) bl record_mmu_state bl record_kaslr_offset bl preserve_boot_args + bl __create_page_tables bl init_kernel_el // w0=cpu_boot_mode bl set_cpu_boot_mode_flag - bl __create_page_tables /* * The following calls CPU setup code, see arch/arm64/mm/proc.S for * details.
Signature · 7 lines
2.30.2 _______________________________________________ linux-arm-kernel mailing list [email protected] http://lists.infradead.org/mailman/listinfo/linux-arm-kernel
Ard Biesheuvel
maintainerRFC PATCH 6/8 arm64: head: clean the ID map page to the PoC
In reply to Ard BiesheuvelIf we enter with the MMU and caches enabled, the caller may not have performed any cache maintenance. So clean the ID mapped page to the PoC, and invalidate the I-cache so we can safely execute from it after disabling the MMU and caches.
Note that this means primary_entry() itself needs to be moved into the ID map as well, as we will return from init_kernel_el() with the MMU and caches off.
--- arch/arm64/kernel/head.S | 16 +++++++++++++++- 1 file changed, 15 insertions(+), 1 deletion(-)
arch/arm64/kernel/head.S +15 -1
@@ -75,7 +75,7 @@ __EFI_PE_HEADER - __INIT + .section ".idmap.text","awx" /* * The following callee saved general purpose registers are used on the
@@ -94,6 +94,19 @@ SYM_CODE_START(primary_entry) bl record_kaslr_offset bl preserve_boot_args bl __create_page_tables + + /* + * If we entered with the MMU and caches on, clean the ID mapped part + * of the primary boot code to the PoC and invalidate it from the + * I-cache so we can safely disable them. + */ + cbz x25, 0f + adrp x0, __idmap_text_start + adr_l x1, __idmap_text_end + sub x1, x1, x0 + bl dcache_clean_poc + ic ialluis +0: bl init_kernel_el // w0=cpu_boot_mode bl set_cpu_boot_mode_flag /*
@@ -106,6 +119,7 @@ SYM_CODE_START(primary_entry) b __primary_switch SYM_CODE_END(primary_entry) + __INIT SYM_CODE_START_LOCAL(record_mmu_state) mrs x25, CurrentEL cmp x25, #CurrentEL_EL2
Signature · 7 lines
2.30.2 _______________________________________________ linux-arm-kernel mailing list [email protected] http://lists.infradead.org/mailman/listinfo/linux-arm-kernel
Ard Biesheuvel
maintainerRFC PATCH 7/8 arm64: lds: move idmap_pg_dir out of .rodata
In reply to Ard BiesheuvelAfter future changes, the ID map may be set up by the boot entry code with the MMU and caches enabled, which means we will reuse the identity map set up by the firmware. This means that memory we describe as read-only in the PE/COFF header may not be writable, preventing us from creating the new identity map if the root level is located in such a region.
--- arch/arm64/kernel/vmlinux.lds.S | 8 ++++---- 1 file changed, 4 insertions(+), 4 deletions(-)
arch/arm64/kernel/vmlinux.lds.S +4 -4
@@ -194,10 +194,6 @@ SECTIONS HYPERVISOR_DATA_SECTIONS - idmap_pg_dir = .; - . += IDMAP_DIR_SIZE; - idmap_pg_end = .; - #ifdef CONFIG_UNMAP_KERNEL_AT_EL0 tramp_pg_dir = .; . += PAGE_SIZE;
@@ -298,6 +294,10 @@ SECTIONS BSS_SECTION(SBSS_ALIGN, 0, 0) . = ALIGN(PAGE_SIZE); + idmap_pg_dir = .; + . += IDMAP_DIR_SIZE; + idmap_pg_end = .; + init_pg_dir = .; . += INIT_DIR_SIZE; init_pg_end = .;
Signature · 7 lines
2.30.2 _______________________________________________ linux-arm-kernel mailing list [email protected] http://lists.infradead.org/mailman/listinfo/linux-arm-kernel
Ard Biesheuvel
maintainerRFC PATCH 8/8 arm64: efi: leave MMU and caches on when handing over to the core kernel
In reply to Ard BiesheuvelInstead of cleaning the entire kernel image to the PoC and enter with the MMU and caches disabled, leave them on and let the primary boot code deal with this if we are executing at EL1.
--- arch/arm64/kernel/efi-entry.S | 20 ++++++++++---------- drivers/firmware/efi/libstub/fdt.c | 6 ++++-- 2 files changed, 14 insertions(+), 12 deletions(-)
arch/arm64/kernel/efi-entry.S +10 -10
@@ -23,6 +23,10 @@ SYM_CODE_START(efi_enter_kernel) add x19, x0, x2 // relocated Image entrypoint mov x20, x1 // DTB address + mrs x0, CurrentEL + cmp x0, #CurrentEL_EL2 + b.ne 1f + /* * Clean the copied Image to the PoC, and ensure it is not shadowed by * stale icache entries from before relocation.
@@ -41,29 +45,25 @@ SYM_CODE_START(efi_enter_kernel) bl dcache_clean_poc 0: /* Turn off Dcache and MMU */ - mrs x0, CurrentEL - cmp x0, #CurrentEL_EL2 - b.ne 1f mrs x0, sctlr_el2 bic x0, x0, #1 << 0 // clear SCTLR.M bic x0, x0, #1 << 2 // clear SCTLR.C pre_disable_mmu_workaround msr sctlr_el2, x0 isb + mov x1, xzr b 2f 1: - mrs x0, sctlr_el1 - bic x0, x0, #1 << 0 // clear SCTLR.M - bic x0, x0, #1 << 2 // clear SCTLR.C - pre_disable_mmu_workaround - msr sctlr_el1, x0 - isb + ldr_l x1, kaslr_seed 2: /* Jump to kernel entry point */ mov x0, x20 - mov x1, xzr mov x2, xzr mov x3, xzr br x19 3: SYM_CODE_END(efi_enter_kernel) + + .section ".bss", "aw", %nobits + .align 3 +SYM_DATA(kaslr_seed, .quad 0x0)
drivers/firmware/efi/libstub/fdt.c +4 -2
@@ -137,11 +137,13 @@ static efi_status_t update_fdt(void *orig_fdt, unsigned long orig_fdt_size, goto fdt_set_fail;if (IS_ENABLED (CONFIG_RANDOMIZE_BASE ) && !efi_nokaslr) { +extern u64 kaslr_seed;efi_status_t efi_status; - efi_status =efi_get_random_bytes (sizeof (fdt_val64), - (u8 *)&fdt_val64); + efi_status =efi_get_random_bytes (sizeof (kaslr_seed), + (u8 *)&kaslr_seed);if (efi_status ==EFI_SUCCESS ) { + fdt_val64 =cpu_to_fdt64 (kaslr_seed); status =fdt_setprop_var (fdt, node,"kaslr-seed" , fdt_val64);if (status) goto fdt_set_fail;
Signature · 7 lines
2.30.2 _______________________________________________ linux-arm-kernel mailing list [email protected] http://lists.infradead.org/mailman/listinfo/linux-arm-kernel