Skip to contentSkip to conversation

arm64: Emit debug bootcodes for early kernel boot

RFC PATCH New Bartosz Szczepanek 1 participant 1 message Apr 23, 2025 arm64

lore
Share & tools
lore
#0 lore raw

RFC PATCH 1/1 arm64: Emit debug bootcodes for early kernel boot

Hi all,

I'm looking for your comments on the following change, that came out of our investigations of rare crashes in the very early phase of kernel boot while still in the assembly code. The change we came up with relies on contextidr_el2 which is used to record checkpoints (bootcodes), to be used later for narrowing down on the point of failure. To do so, one needs modified firmware or another tool capable of dumping contextidr_el2 after failure happens.

The usage of contextidr_el2 register comes from the fact that it's not used for any purpose within the kernel (contrary to contextidr_el1), and contents of it are unlikely to change even if any other code would be executed after the failure.

Primary alternative solution – debugging by printing – hardly applies to such early code, as page table mappings are not yet established and stable virtual address of console can't be relied upon.

On the other hand, bisecting through the code is not always feasible, especially if the failure rates are very low. Bootcodes help that case.

What I'd like to get your input on:
- Is there an alternative solution for post-mortem analysis of rare crashes in very early kernel boot that we missed?
- Would the introduction of bootcodes be welcome upstream, in this or any other shape? If so, how could we make it suitable for inclusion?

Kind regards
Bartosz

-------------------------------------------------
arm64: Emit debug bootcodes for early kernel boot

Reuse contextidr_el2 register which is unused in the early boot phase for the purpose of tracking bootcodes. This allows to identify point of failure in early boot code and helps investigations.

To make the change safe on targets without contextidr_el2, we're checking dfr0_el1 debug version bits. If it tells us we're below ARMV8.1, we bail out without writing to contextidr_el2.

To make use of the change, modified firmware or another tool capable of periodically dumping contextidr_el2 is necessary.

---
 arch/arm64/Kconfig.debug   |  6 ++++++
 arch/arm64/kernel/Makefile |  5 +++++
 arch/arm64/kernel/head.S   | 40 ++++++++++++++++++++++++++++++++++++++
 3 files changed, 51 insertions(+)
arch/arm64/Kconfig.debug +6 -0
index 265c4461031f..668bda096773 100644
@@ -20,4 +20,10 @@ config ARM64_RELOC_TEST
 	depends on m
 	tristate "Relocation testing module"
 
+config DEBUG_EARLY_BOOT
+	bool "Early boot debugging"
+	help
+	  Enable this option to use contextidr_el2 register for storing
+	  bootcodes from the early assembly code.
+
 source "drivers/hwtracing/coresight/Kconfig"
arch/arm64/kernel/Makefile +5 -0
index 71c29a2a2f19..72b38e8314d4 100644
@@ -84,5 +84,10 @@ ifeq ($(CONFIG_DEBUG_EFI),y)
 AFLAGS_head.o += -DVMLINUX_PATH="\"$(realpath $(objtree)/vmlinux)\""
 endif
 
+ifeq ($(CONFIG_DEBUG_EARLY_BOOT),y)
+AFLAGS_head.o += -march=armv8.1-a
+CFLAGS_setup.o += -march=armv8.1-a
+endif
+
 # for cleaning
 subdir- += vdso vdso32
arch/arm64/kernel/head.S +40 -0
index 2ce73525de2c..f15db7e83b6e 100644
@@ -42,6 +42,34 @@
 #error PAGE_OFFSET must be at least 2MB aligned
 #endif
 
+#ifdef CONFIG_DEBUG_EARLY_BOOT
+/*
+ * This debugging enhancement makes use of contextidr_el2 for the purpose
+ * of storing bootcodes.
+ */
+.macro _emit_bootcode val, t1
+	/* Check if in EL2 */
+	mrs	\t1, CurrentEL
+	ubfx    \t1, \t1, #2, #2
+	cmp	\t1, #2
+	b.ne	1f
+
+	/* Check if we're at least ARMv8.1 */
+	mrs	\t1, ID_AA64DFR0_EL1
+	ubfx	\t1, \t1, #0, #4
+	cmp	\t1, #7
+	b.lt	1f
+
+	mov	\t1, #\val
+	msr	contextidr_el2, \t1
+1:
+	/* Quit */
+.endm
+#define emit_bootcode(reg) _emit_bootcode __LINE__, reg
+#else
+#define emit_bootcode(reg)
+#endif
+
 /*
  * Kernel startup entry point.
  * ---------------------------
@@ -86,6 +114,7 @@ SYM_CODE_START(primary_entry)
 	bl	record_mmu_state
 	bl	preserve_boot_args
 
+	emit_bootcode(x22)
 	adrp	x1, early_init_stack
 	mov	sp, x1
 	mov	x29, xzr
@@ -101,6 +130,7 @@ SYM_CODE_START(primary_entry)
 	cbnz	x19, 0f
 	dmb     sy
 	mov	x1, x0				// end of used region
+	emit_bootcode(x22)
 	adrp    x0, init_idmap_pg_dir
 	adr_l	x2, dcache_inval_poc
 	blr	x2
@@ -126,7 +156,9 @@ SYM_CODE_START(primary_entry)
 	 * On return, the CPU will be ready for the MMU to be turned on and
 	 * the TCR will have been set.
 	 */
+	emit_bootcode(x22)
 	bl	__cpu_setup			// initialise processor
+	emit_bootcode(x22)
 	b	__primary_switch
 SYM_CODE_END(primary_entry)
 
@@ -220,10 +252,12 @@ SYM_CODE_END(preserve_boot_args)
 SYM_FUNC_START_LOCAL(__primary_switched)
 	adr_l	x4, init_task
 	init_cpu_task x4, x5, x6
+	emit_bootcode(x22)
 
 	adr_l	x8, vectors			// load VBAR_EL1 with virtual
 	msr	vbar_el1, x8			// vector table address
 	isb
+	emit_bootcode(x22)
 
 	stp	x29, x30, [sp, #-16]!
 	mov	x29, sp
@@ -236,6 +270,7 @@ SYM_FUNC_START_LOCAL(__primary_switched)
 
 	mov	x0, x20
 	bl	set_cpu_boot_mode_flag
+	emit_bootcode(x22)
 
 #if defined(CONFIG_KASAN_GENERIC) || defined(CONFIG_KASAN_SW_TAGS)
 	bl	kasan_early_init
@@ -243,6 +278,7 @@ SYM_FUNC_START_LOCAL(__primary_switched)
 	mov	x0, x20
 	bl	finalise_el2			// Prefer VHE if possible
 	ldp	x29, x30, [sp], #16
+	emit_bootcode(x22)
 	bl	start_kernel
 	ASM_BUG()
 SYM_FUNC_END(__primary_switched)
@@ -464,6 +500,7 @@ SYM_FUNC_START(__enable_mmu)
 	cmp     x3, #ID_AA64MMFR0_EL1_TGRAN_SUPPORTED_MAX
 	b.gt    __no_granule_support
 	phys_to_ttbr x2, x2
+	emit_bootcode(x22)
 	msr	ttbr0_el1, x2			// load TTBR0
 	load_ttbr1 x1, x1, x3
 
@@ -506,16 +543,19 @@ SYM_FUNC_START_LOCAL(__no_granule_support)
 SYM_FUNC_END(__no_granule_support)
 
 SYM_FUNC_START_LOCAL(__primary_switch)
+	emit_bootcode(x22)
 	adrp	x1, reserved_pg_dir
 	adrp	x2, init_idmap_pg_dir
 	bl	__enable_mmu
 
+	emit_bootcode(x22)
 	adrp	x1, early_init_stack
 	mov	sp, x1
 	mov	x29, xzr
 	mov	x0, x20				// pass the full boot status
 	mov	x1, x21				// pass the FDT
 	bl	__pi_early_map_kernel		// Map and relocate the kernel
+	emit_bootcode(x22)
 
 	ldr	x8, =__primary_switched
 	adrp	x0, KERNEL_START		// __pa(KERNEL_START)
Signature · 1 line
2.47.1