mainline/master bisection: baseline.login on meson-sm1-khadas-vim3l

* * * * * * * * * * * * * * * * * * * * * * * * * * * * * * * * *

* This automated bisection report was sent to you on the basis  *
* that you may be involved with the breaking commit it has      *
* found.  No manual investigation has been done to verify it,   *
* and the root cause of the problem may be somewhere else.      *
*                                                               *
* If you do send a fix, please include this trailer:            *
*   Reported-by: "kernelci.org bot" <[email protected]>          *
*                                                               *
* Hope this helps!                                              *

* * * * * * * * * * * * * * * * * * * * * * * * * * * * * * * * *

mainline/master bisection: baseline.login on meson-sm1-khadas-vim3l

Summary:

  Start:      31caf8b2a847 Merge branch 'linus' of git://git.kernel.org/pub/scm/linux/kernel/git/herbert/crypto-2.6
  Plain log:  https://storage.kernelci.org/mainline/master/v5.11-7579-g31caf8b2a847/arm64/defconfig+CONFIG_ARM64_64K_PAGES=y/clang-10/lab-baylibre/baseline-meson-sm1-khadas-vim3l.txt
  HTML log:   https://storage.kernelci.org/mainline/master/v5.11-7579-g31caf8b2a847/arm64/defconfig+CONFIG_ARM64_64K_PAGES=y/clang-10/lab-baylibre/baseline-meson-sm1-khadas-vim3l.html
  Result:     0c93df9622d4 arm64: Initialise as nVHE before switching to VHE

Checks:
revert: PASS
verify: PASS

Parameters:

  Tree:       mainline
  URL:        https://git.kernel.org/pub/scm/linux/kernel/git/torvalds/linux.git
  Branch:     master
  Target:     meson-sm1-khadas-vim3l
  CPU arch:   arm64
  Lab:        lab-baylibre
  Compiler:   clang-10
  Config:     defconfig+CONFIG_ARM64_64K_PAGES=y
  Test case:  baseline.login

Breaking commit found:

-------------------------------------------------------------------------------
commit 0c93df9622d4d921bcd0dc83f71fed9e98f5119f

Author: Marc Zyngier <[email protected]>
Date:   Mon Feb 8 09:57:14 2021 +0000

arm64: Initialise as nVHE before switching to VHE

As we are aiming to be able to control whether we enable VHE or not, let's always drop down to EL1 first, and only then upgrade to VHE if at all possible.

This means that if the kernel is booted at EL2, we always start with a nVHE init, drop to EL1 to initialise the the kernel, and only then upgrade the kernel EL to EL2 if possible (the process is obviously shortened for secondary CPUs).

The resume path is handled similarly to a secondary CPU boot.

Signed-off-by Marc Zyngier <[email protected]>

Acked-by David Brazdil <[email protected]>

Acked-by Catalin Marinas <[email protected]>

Link https://lore.kernel.org/r/[email protected] open here

[will: Avoid calling switch_to_vhe twice on kaslr path]

Signed-off-by Will Deacon <[email protected]>

arch/arm64/kernel/head.S +2 -36

index 28e9735302df..ec66dc061b0c 100644
@@ -447,6 +447,7 @@ SYM_FUNC_START_LOCAL(__primary_switched)
 	ret					// to __primary_switch()
 0:
 #endif
+	bl	switch_to_vhe			// Prefer VHE if possible
 	add	sp, sp, #16
 	mov	x29, #0
 	mov	x30, #0

@@ -493,42 +494,6 @@ SYM_INNER_LABEL(init_el1, SYM_L_LOCAL)
 	eret
 
 SYM_INNER_LABEL(init_el2, SYM_L_LOCAL)
-#ifdef CONFIG_ARM64_VHE
-	/*
-	 * Check for VHE being present. x2 being non-zero indicates that we
-	 * do have VHE, and that the kernel is intended to run at EL2.
-	 */
-	mrs	x2, id_aa64mmfr1_el1
-	ubfx	x2, x2, #ID_AA64MMFR1_VHE_SHIFT, #4
-#else
-	mov	x2, xzr
-#endif
-	cbz	x2, init_el2_nvhe
-
-	/*
-	 * When VHE _is_ in use, EL1 will not be used in the host and
-	 * requires no configuration, and all non-hyp-specific EL2 setup
-	 * will be done via the _EL1 system register aliases in __cpu_setup.
-	 */
-	mov_q	x0, HCR_HOST_VHE_FLAGS
-	msr	hcr_el2, x0
-	isb
-
-	init_el2_state vhe
-
-	isb
-
-	mov_q	x0, INIT_PSTATE_EL2
-	msr	spsr_el2, x0
-	msr	elr_el2, lr
-	mov	w0, #BOOT_CPU_MODE_EL2
-	eret
-
-SYM_INNER_LABEL(init_el2_nvhe, SYM_L_LOCAL)
-	/*
-	 * When VHE is not in use, early init of EL2 and EL1 needs to be
-	 * done here.
-	 */
 	mov_q	x0, INIT_SCTLR_EL1_MMU_OFF
 	msr	sctlr_el1, x0
 

@@ -623,6 +588,7 @@ SYM_FUNC_START_LOCAL(secondary_startup)
 	/*
 	 * Common entry point for secondary CPUs.
 	 */
+	bl	switch_to_vhe
 	bl	__cpu_secondary_check52bitva
 	bl	__cpu_setup			// initialise processor
 	adrp	x1, swapper_pg_dir

arch/arm64/kernel/hyp-stub.S +24 -0

index 3f3dbbe8914d..373ed2213e1d 100644
@@ -190,3 +190,27 @@ SYM_FUNC_START(__hyp_reset_vectors)
 	hvc	#0
 	ret
 SYM_FUNC_END(__hyp_reset_vectors)
+
+/*
+ * Entry point to switch to VHE if deemed capable
+ */
+SYM_FUNC_START(switch_to_vhe)
+#ifdef CONFIG_ARM64_VHE
+	// Need to have booted at EL2
+	adr_l	x1, __boot_cpu_mode
+	ldr	w0, [x1]
+	cmp	w0, #BOOT_CPU_MODE_EL2
+	b.ne	1f
+
+	// and still be at EL1
+	mrs	x0, CurrentEL
+	cmp	x0, #CurrentEL_EL1
+	b.ne	1f
+
+	// Turn the world upside down
+	mov	x0, #HVC_VHE_RESTART
+	hvc	#0
+1:
+#endif
+	ret
+SYM_FUNC_END(switch_to_vhe)

arch/arm64/kernel/sleep.S +1 -1

index 6bdef7362c0e..5bfd9b87f85d 100644
@@ -100,6 +100,7 @@ SYM_FUNC_END(__cpu_suspend_enter)
 	.pushsection ".idmap.text", "awx"
 SYM_CODE_START(cpu_resume)
 	bl	init_kernel_el
+	bl	switch_to_vhe
 	bl	__cpu_setup
 	/* enable the MMU early - so we can access sleep_save_stash by va */
 	adrp	x1, swapper_pg_dir
-------------------------------------------------------------------------------

Git bisection log:

-------------------------------------------------------------------------------
git bisect start
# good: [55f62bc873477dae2c45bbbc30b86cf3e0982f3b] Merge tag 'pnp-5.12-rc1' of git://git.kernel.org/pub/scm/linux/kernel/git/rafael/linux-pm
git bisect good 55f62bc873477dae2c45bbbc30b86cf3e0982f3b
# bad: [31caf8b2a847214be856f843e251fc2ed2cd1075] Merge branch 'linus' of git://git.kernel.org/pub/scm/linux/kernel/git/herbert/crypto-2.6
git bisect bad 31caf8b2a847214be856f843e251fc2ed2cd1075
# bad: [de1617578849acab8e16c9ffdce39b91fb50639d] Merge tag 'media/v5.12-1' of git://git.kernel.org/pub/scm/linux/kernel/git/mchehab/linux-media
git bisect bad de1617578849acab8e16c9ffdce39b91fb50639d
# good: [657bd90c93146a929c69cd43addf2804eb70c926] Merge tag 'sched-core-2021-02-17' of git://git.kernel.org/pub/scm/linux/kernel/git/tip/tip
git bisect good 657bd90c93146a929c69cd43addf2804eb70c926
# bad: [3e10585335b7967326ca7b4118cada0d2d00a2ab] Merge tag 'for-linus' of git://git.kernel.org/pub/scm/virt/kvm/kvm
git bisect bad 3e10585335b7967326ca7b4118cada0d2d00a2ab
# bad: [08179b47e1fdf288e5d59f90e5ce31513bb019c3] Merge branch 'parisc-5.12-1' of git://git.kernel.org/pub/scm/linux/kernel/git/deller/parisc-linux
git bisect bad 08179b47e1fdf288e5d59f90e5ce31513bb019c3
# bad: [99ca0edb41aabd888ca1548fa0391a4975740a83] Merge tag 'arm64-upstream' of git://git.kernel.org/pub/scm/linux/kernel/git/arm64/linux
git bisect bad 99ca0edb41aabd888ca1548fa0391a4975740a83
# bad: [1d32854ea76331d10d376ed3ac67831b94466ae9] Merge branch 'for-next/misc' into for-next/core
git bisect bad 1d32854ea76331d10d376ed3ac67831b94466ae9
# bad: [90eb8c9d94fa7f9969792584c376b435bf8a035c] Merge branch 'for-next/errata' into for-next/core
git bisect bad 90eb8c9d94fa7f9969792584c376b435bf8a035c
# bad: [41fac42c25338f4ea295b58106c26683d893a1c6] arm64: Honor VHE being disabled from the command-line
git bisect bad 41fac42c25338f4ea295b58106c26683d893a1c6
# bad: [19e87e131915a2389a08874092a82fe5aa0f8952] arm64: Move VHE-specific SPE setup to mutate_to_vhe()
git bisect bad 19e87e131915a2389a08874092a82fe5aa0f8952
# good: [8cc8a32415364e475c25277b507f06f67c47ca9a] arm64: Turn the MMU-on sequence into a macro
git bisect good 8cc8a32415364e475c25277b507f06f67c47ca9a
# bad: [0c93df9622d4d921bcd0dc83f71fed9e98f5119f] arm64: Initialise as nVHE before switching to VHE
git bisect bad 0c93df9622d4d921bcd0dc83f71fed9e98f5119f
# good: [f359182291c757cdf77bcd014c025d1ed6b87662] arm64: Provide an 'upgrade to VHE' stub hypercall
git bisect good f359182291c757cdf77bcd014c025d1ed6b87662

# first bad commit: [0c93df9622d4d921bcd0dc83f71fed9e98f5119f] arm64: Initialise as nVHE before switching to VHE
-------------------------------------------------------------------------------

-=-=-=-=-=-=-=-=-=-=-=-
Groups.io Links: You receive all messages sent to this group.
View/Reply Online (#7239): https://groups.io/g/kernelci-results/message/7239
Mute This Topic: https://groups.io/mt/80821202/924702
Group Owner: [email protected]
Unsubscribe: https://groups.io/g/kernelci-results/unsub [[email protected]]
-=-=-=-=-=-=-=-=-=-=-=-

Hello Marc,

Please see the bisection report below about a boot failure on meson-sm1-khadas-vim3l on mainline. It seems to only be affecting kernels built with CONFIG_ARM64_64K_PAGES=y.

Reports aren't automatically sent to the public while we're trialing new bisection features on kernelci.org but this one looks valid.

There's no output in the log, so the kernel is most likely crashing early. Some more details can be found here:

https://kernelci.org/test/case/id/6034bed3b344e2860daddcc8/

Please let us know if you need any help to debug the issue or try a fix on this platform.

Hi Guillaume,

Quoted 11 lines · Guillaume Tucker

Hello Marc,

Please see the bisection report below about a boot failure on meson-sm1-khadas-vim3l on mainline. It seems to only be affecting kernels built with CONFIG_ARM64_64K_PAGES=y.

Reports aren't automatically sent to the public while we're trialing new bisection features on kernelci.org but this one looks valid.

Quoted 7 lines

There's no output in the log, so the kernel is most likely crashing early. Some more details can be found here:

https://kernelci.org/test/case/id/6034bed3b344e2860daddcc8/

Please let us know if you need any help to debug the issue or try a fix on this platform.

Thanks for the heads up.

There is actually a fundamental problem with the patch you bisected
to: it provides no guarantee that the point where we enable the EL2 MMU is in the idmap and, as it turns out, the code we're running from disappears from under our feet, leading to a translation fault we're not prepared to handle.

How does it work with 4kB pages? Luck.

Do you mind giving the patch below a go? It does work on my vim3l and on a FVP, so odds are that it will solve it for you too.

Thanks,

	M.

arch/arm64/kernel/hyp-stub.S +22 -9

index 678cd2c618ee..fbd2543b8f7d 100644
@@ -96,8 +96,10 @@ SYM_CODE_START_LOCAL(mutate_to_vhe)
 	cmp	x1, xzr
 	and	x2, x2, x1
 	csinv	x2, x2, xzr, ne
-	cbz	x2, 1f
+	cbnz	x2, 2f
 
+1:	eret
+2:
 	// Engage the VHE magic!
 	mov_q	x0, HCR_HOST_VHE_FLAGS
 	msr	hcr_el2, x0

@@ -131,11 +133,29 @@ SYM_CODE_START_LOCAL(mutate_to_vhe)
 	msr	mair_el1, x0
 	isb
 
+	// Hack the exception return to stay at EL2
+	mrs	x0, spsr_el1
+	and	x0, x0, #~PSR_MODE_MASK
+	mov	x1, #PSR_MODE_EL2h
+	orr	x0, x0, x1
+	msr	spsr_el1, x0
+
+	b	enter_vhe
+SYM_CODE_END(mutate_to_vhe)
+
+	// At the point where we reach enter_vhe(), we run with
+	// the MMU off (which is enforced by mutate_to_vhe()).
+	// We thus need to be in the idmap, or everything will
+	// explode when enabling the MMU.
+
+	.pushsection	.idmap.text, "ax"
+
+SYM_CODE_START_LOCAL(enter_vhe)
+	// Enable the EL2 S1 MMU, as set up from EL1
 	// Invalidate TLBs before enabling the MMU
 	tlbi	vmalle1
 	dsb	nsh
 
-	// Enable the EL2 S1 MMU, as set up from EL1
 	mrs_s	x0, SYS_SCTLR_EL12
 	set_sctlr_el1	x0
 

@@ -143,17 +163,12 @@ SYM_CODE_START_LOCAL(mutate_to_vhe)
 	mov_q	x0, INIT_SCTLR_EL1_MMU_OFF
 	msr_s	SYS_SCTLR_EL12, x0
 
-	// Hack the exception return to stay at EL2
-	mrs	x0, spsr_el1
-	and	x0, x0, #~PSR_MODE_MASK
-	mov	x1, #PSR_MODE_EL2h
-	orr	x0, x0, x1
-	msr	spsr_el1, x0
-
 	mov	x0, xzr

Quoted 31 lines · Marc Zyngier

Hi Guillaume,

Quoted 11 lines · Guillaume Tucker

Hello Marc,

Please see the bisection report below about a boot failure on meson-sm1-khadas-vim3l on mainline. It seems to only be affecting kernels built with CONFIG_ARM64_64K_PAGES=y.

Reports aren't automatically sent to the public while we're trialing new bisection features on kernelci.org but this one looks valid.

Quoted 7 lines

There's no output in the log, so the kernel is most likely crashing early. Some more details can be found here:

https://kernelci.org/test/case/id/6034bed3b344e2860daddcc8/

Please let us know if you need any help to debug the issue or try a fix on this platform.

Thanks for the heads up.

There is actually a fundamental problem with the patch you bisected
to: it provides no guarantee that the point where we enable the EL2 MMU is in the idmap and, as it turns out, the code we're running from disappears from under our feet, leading to a translation fault we're not prepared to handle.

Quoted 1 line

How does it work with 4kB pages? Luck.

There may be a fascinating explanation for it, but luck works too. It really seems to be booting happily with 4k pages:

https://kernelci.org/test/plan/id/60347b358de339d1b7addcc5/

Quoted 3 lines

Do you mind giving the patch below a go? It does work on my vim3l and on a FVP, so odds are that it will solve it for you too.

Sure, and that worked here as well:

http://lava.baylibre.com:10080/scheduler/job/752416

and here's the test branch where I applied your fix, for completeness:

https://gitlab.collabora.com/gtucker/linux/-/commits/v5.11-vim3l-vhe/

As always, if you do send a patch with the fix, please give some credit to the bot:

Reported-by "kernelci.org bot" <[email protected]>

Quoted 38 lines · Guillaume Tucker

Quoted 25 lines · Marc Zyngier

Hi Guillaume,

Quoted 11 lines · Guillaume Tucker

Hello Marc,

Please see the bisection report below about a boot failure on meson-sm1-khadas-vim3l on mainline. It seems to only be affecting kernels built with CONFIG_ARM64_64K_PAGES=y.

Reports aren't automatically sent to the public while we're trialing new bisection features on kernelci.org but this one looks valid.

Quoted 7 lines

There's no output in the log, so the kernel is most likely crashing early. Some more details can be found here:

https://kernelci.org/test/case/id/6034bed3b344e2860daddcc8/

Please let us know if you need any help to debug the issue or try a fix on this platform.

Thanks for the heads up.

Quoted 7 lines

There is actually a fundamental problem with the patch you bisected
to: it provides no guarantee that the point where we enable the EL2 MMU is in the idmap and, as it turns out, the code we're running from disappears from under our feet, leading to a translation fault we're not prepared to handle.

How does it work with 4kB pages? Luck.

There may be a fascinating explanation for it, but luck works too. It really seems to be booting happily with 4k pages:

Quoted 1 line

https://kernelci.org/test/plan/id/60347b358de339d1b7addcc5/

Oh, I know it boots fine with 4k, that's what I used everywhere. We're just lucky that the bit of code that deals with the MMU happens to *also* be in the idmap. With 64k pages, it gets pushed further down the line, and bad things happen. Short of explicit statements in the code, luck rules.

Quoted 11 lines

Quoted 2 lines

Do you mind giving the patch below a go? It does work on my vim3l and on a FVP, so odds are that it will solve it for you too.

Sure, and that worked here as well:

http://lava.baylibre.com:10080/scheduler/job/752416

and here's the test branch where I applied your fix, for completeness:

https://gitlab.collabora.com/gtucker/linux/-/commits/v5.11-vim3l-vhe/

Awesome. thanks for having tested it.

Quoted 4 lines

As always, if you do send a patch with the fix, please give some credit to the bot:

Reported-by "kernelci.org bot" <[email protected]>

Will do. Mind if I credit you too for the testing?