arm64: entry: Simplify KPTI trampoline exit code
v2 cover arm64: entry: Simplify KPTI trampoline exit code
Changes since v1:
- add a separate patch that merges the native and compat code paths
- improve commit log on patch #2
Ard Biesheuvel (2):
arm64: entry: Preserve/restore X29 even for compat tasks
arm64: entry: Simplify tramp_alias macro and tramp_exit routine
arch/arm64/kernel/entry.S | 57 +++++++------------- 1 file changed, 20 insertions(+), 37 deletions(-)
Signature · 7 lines
2.39.2 _______________________________________________ linux-arm-kernel mailing list [email protected] http://lists.infradead.org/mailman/listinfo/linux-arm-kernel
PATCH v2 1/2 arm64: entry: Preserve/restore X29 even for compat tasks
Currently, the KPTI trampoline code for returning to user space takes care to only preserve X29 into FAR_EL1 for native tasks, as compat tasks don't have access to this register anyway, and so preserving it is not necessary. It also means it does not need to be restored, and so we have two code paths for returning back to user space: the native one that restores X29 from FAR_EL1, and the compat one that leaves X29 clobbered, containing the value of TTBR1_EL1, which carries a physical address pointing somewhere into the kernel image.
This is needlessly complex, and given that FAR_EL1 becomes UNKNOWN after an exception return anway, the only benefit of avoiding the preserve and restore is that we can skip the system register write and read.
So let's simplify this, and collapse the two code paths into one that always preserves X29 into FAR_EL1, and always restores it again after the TTBR switch.
--- arch/arm64/kernel/entry.S | 10 ---------- 1 file changed, 10 deletions(-)
arch/arm64/kernel/entry.S +0 -10
@@ -435,13 +435,9 @@ alternative_if_not ARM64_UNMAP_KERNEL_AT_EL0 eret alternative_else_nop_endif #ifdef CONFIG_UNMAP_KERNEL_AT_EL0 - bne 4f msr far_el1, x29 tramp_alias x30, tramp_exit_native, x29 br x30 -4: - tramp_alias x30, tramp_exit_compat, x29 - br x30 #endif .else ldr lr, [sp, #S_LR]
@@ -740,9 +736,7 @@ alternative_else_nop_endif msr vbar_el1, x30 ldr lr, [sp, #S_LR] tramp_unmap_kernel x29 - .if \regsize == 64 mrs x29, far_el1 - .endif add sp, sp, #PT_REGS_SIZE // restore sp eret sb
@@ -780,10 +774,6 @@ SYM_CODE_END(tramp_vectors) SYM_CODE_START(tramp_exit_native) tramp_exit SYM_CODE_END(tramp_exit_native) - -SYM_CODE_START(tramp_exit_compat) - tramp_exit 32 -SYM_CODE_END(tramp_exit_compat) .popsection // .entry.tramp.text #endif /* CONFIG_UNMAP_KERNEL_AT_EL0 */
Signature · 7 lines
2.39.2 _______________________________________________ linux-arm-kernel mailing list [email protected] http://lists.infradead.org/mailman/listinfo/linux-arm-kernel
PATCH v2 2/2 arm64: entry: Simplify tramp_alias macro and tramp_exit routine
In reply to Ard BiesheuvelThe tramp_alias macro constructs the virtual alias of a symbol in the trampoline text mapping, based on its kernel text address, and does so in a way that is more convoluted than necessary. So let's simplify that.
Also, now that the address of the vector table is kept in a per-CPU variable, there is no need to defer the load and the assignment of VBAR_EL1 to tramp_exit(). This means we can use a PC-relative reference to the per-CPU variable instead of storing its absolute address in a global variable in the trampoline rodata.
And given that tramp_alias no longer needs a temp register, this means we can restore X30 earlier as well, and only leave X29 for tramp_exit() to restore.
While at it, give some related symbols static linkage, considering that they are only referenced from the object file that defines them.
--- arch/arm64/kernel/entry.S | 47 +++++++++----------- 1 file changed, 20 insertions(+), 27 deletions(-)
arch/arm64/kernel/entry.S +20 -27
@@ -101,12 +101,11 @@ .org .Lventry_start\@ + 128 // Did we overflow the ventry slot? .endm - .macro tramp_alias, dst, sym, tmp - mov_q \dst, TRAMP_VALIAS - adr_l \tmp, \sym - add \dst, \dst, \tmp - adr_l \tmp, .entry.tramp.text - sub \dst, \dst, \tmp + .macro tramp_alias, dst, sym + .set .Lalias\@, TRAMP_VALIAS + \sym - .entry.tramp.text + movz \dst, :abs_g2_s:.Lalias\@ + movk \dst, :abs_g1_nc:.Lalias\@ + movk \dst, :abs_g0_nc:.Lalias\@ .endm /*
@@ -436,8 +435,13 @@ alternative_if_not ARM64_UNMAP_KERNEL_AT_EL0 alternative_else_nop_endif #ifdef CONFIG_UNMAP_KERNEL_AT_EL0 msr far_el1, x29 - tramp_alias x30, tramp_exit_native, x29 - br x30 + + ldr_this_cpu x30, this_cpu_vector, x29 + tramp_alias x29, tramp_exit + msr vbar_el1, x30 // install vector table + ldr lr, [sp, #S_LR] // restore x30 + add sp, sp, #PT_REGS_SIZE // restore sp + br x29 #endif .else ldr lr, [sp, #S_LR]
@@ -728,20 +732,6 @@ alternative_else_nop_endif .org 1b + 128 // Did we overflow the ventry slot? .endm - .macro tramp_exit, regsize = 64 - tramp_data_read_var x30, this_cpu_vector - get_this_cpu_offset x29 - ldr x30, [x30, x29] - - msr vbar_el1, x30 - ldr lr, [sp, #S_LR] - tramp_unmap_kernel x29 - mrs x29, far_el1 - add sp, sp, #PT_REGS_SIZE // restore sp - eret - sb - .endm - .macro generate_tramp_vector, kpti, bhb .Lvector_start\@: .space 0x400
@@ -762,7 +752,7 @@ alternative_else_nop_endif */ .pushsection ".entry.tramp.text", "ax" .align 11 -SYM_CODE_START_NOALIGN(tramp_vectors) +SYM_CODE_START_LOCAL_NOALIGN(tramp_vectors) #ifdef CONFIG_MITIGATE_SPECTRE_BRANCH_HISTORY generate_tramp_vector kpti=1, bhb=BHB_MITIGATION_LOOP generate_tramp_vector kpti=1, bhb=BHB_MITIGATION_FW
@@ -771,9 +761,12 @@ SYM_CODE_START_NOALIGN(tramp_vectors) generate_tramp_vector kpti=1, bhb=BHB_MITIGATION_NONE SYM_CODE_END(tramp_vectors) -SYM_CODE_START(tramp_exit_native) - tramp_exit -SYM_CODE_END(tramp_exit_native) +SYM_CODE_START_LOCAL(tramp_exit) + tramp_unmap_kernel x29 + mrs x29, far_el1 // restore x29 + eret + sb +SYM_CODE_END(tramp_exit) .popsection // .entry.tramp.text #endif /* CONFIG_UNMAP_KERNEL_AT_EL0 */
@@ -1067,7 +1060,7 @@ alternative_if_not ARM64_UNMAP_KERNEL_AT_EL0 alternative_else_nop_endif #ifdef CONFIG_UNMAP_KERNEL_AT_EL0 - tramp_alias dst=x5, sym=__sdei_asm_exit_trampoline, tmp=x3 + tramp_alias dst=x5, sym=__sdei_asm_exit_trampoline br x5 #endif SYM_CODE_END(__sdei_asm_handler)
Signature · 7 lines
2.39.2 _______________________________________________ linux-arm-kernel mailing list [email protected] http://lists.infradead.org/mailman/listinfo/linux-arm-kernel
Quoted 10 lines · Ard Biesheuvel
Changes since v1:
- add a separate patch that merges the native and compat code paths
- improve commit log on patch #2Ard Biesheuvel (2):
arm64: entry: Preserve/restore X29 even for compat tasks
arm64: entry: Simplifytramp_aliasmacro andtramp_exitroutinearch/arm64/kernel/entry.S | 57 +++++++------------- 1 file changed, 20 insertions(+), 37 deletions(-)
For both patches:
Will
_______________________________________________
linux-arm-kernel mailing list
[email protected]
http://lists.infradead.org/mailman/listinfo/linux-arm-kernel
Quoted 9 lines · Ard Biesheuvel
Changes since v1:
- add a separate patch that merges the native and compat code paths
- improve commit log on patch #2Ard Biesheuvel (2):
arm64: entry: Preserve/restore X29 even for compat tasks
arm64: entry: Simplifytramp_aliasmacro andtramp_exitroutine[...]
Applied to arm64 (for-next/kpti), thanks!
[1/2] arm64: entry: Preserve/restore X29 even for compat tasks
https://git.kernel.org/arm64/c/0936243cabf0 [2/2] arm64: entry: Simplify tramp_alias macro and tramp_exit routine https://git.kernel.org/arm64/c/211ceca377f4
Signature · 7 lines
Catalin _______________________________________________ linux-arm-kernel mailing list [email protected] http://lists.infradead.org/mailman/listinfo/linux-arm-kernel