arm64: mte: avoid TFSR related operations unless in async mode
PATCH v2 1/1 arm64: mte: avoid TFSR related operations unless in async mode
There is no reason to touch TFSR nor issue a DSB unless our task is in asynchronous mode. Since these operations (especially the DSB) may be expensive on certain microarchitectures, only perform them if necessary.
Furthermore, stop clearing TFSR on entry because it will be cleared on exit and it is not necessary to have any particular value in TFSR between entry and exit.
---
v2:
- avoid clearing TFSR on entry
arch/arm64/kernel/entry.S | 28 +++++++++++++++++++--------- 1 file changed, 19 insertions(+), 9 deletions(-)
arch/arm64/kernel/entry.S +19 -9
@@ -133,29 +133,37 @@ alternative_cb_end .endm /* Check for MTE asynchronous tag check faults */ - .macro check_mte_async_tcf, tmp, ti_flags + .macro check_mte_async_tcf, tmp, ti_flags, thread_sctlr #ifdef CONFIG_ARM64_MTE .arch_extension lse alternative_if_not ARM64_MTE b 1f alternative_else_nop_endif + /* + * Asynchronous tag check faults are only possible in ASYNC (2) or + * ASYM (3) modes. In each of these modes bit 1 of SCTLR_EL1.TCF0 is + * set, so skip the check if it is unset. + */ + tbz \thread_sctlr, #(SCTLR_EL1_TCF0_SHIFT + 1), 1f mrs_s \tmp, SYS_TFSRE0_EL1 tbz \tmp, #SYS_TFSR_EL1_TF0_SHIFT, 1f /* Asynchronous TCF occurred for TTBR0 access, set the TI flag */ mov \tmp, #_TIF_MTE_ASYNC_FAULT add \ti_flags, tsk, #TSK_TI_FLAGS stset \tmp, [\ti_flags] - msr_s SYS_TFSRE0_EL1, xzr 1: #endif .endm /* Clear the MTE asynchronous tag check faults */ - .macro clear_mte_async_tcf + .macro clear_mte_async_tcf thread_sctlr #ifdef CONFIG_ARM64_MTE alternative_if ARM64_MTE + /* See comment in check_mte_async_tcf above. */ + tbz \thread_sctlr, #(SCTLR_EL1_TCF0_SHIFT + 1), 1f dsb ish msr_s SYS_TFSRE0_EL1, xzr +1: alternative_else_nop_endif #endif .endm
@@ -231,8 +239,8 @@ alternative_else_nop_endif disable_step_tsk x19, x20 /* Check for asynchronous tag check faults in user space */ - check_mte_async_tcf x22, x23 - apply_ssbd 1, x22, x23 + ldr x0, [tsk, THREAD_SCTLR_USER] + check_mte_async_tcf x22, x23, x0 #ifdef CONFIG_ARM64_PTR_AUTH alternative_if ARM64_HAS_ADDRESS_AUTH
@@ -245,7 +253,6 @@ alternative_if ARM64_HAS_ADDRESS_AUTH * was disabled on kernel exit then we would have left the kernel IA * installed so there is no need to install it again. */ - ldr x0, [tsk, THREAD_SCTLR_USER] tbz x0, SCTLR_ELx_ENIA_SHIFT, 1f __ptrauth_keys_install_kernel_nosync tsk, x20, x22, x23 b 2f
@@ -258,6 +265,8 @@ alternative_if ARM64_HAS_ADDRESS_AUTH alternative_else_nop_endif #endif + apply_ssbd 1, x22, x23 + mte_set_kernel_gcr x22, x23 scs_load tsk
@@ -362,6 +371,10 @@ alternative_else_nop_endif 3: scs_save tsk + /* Ignore asynchronous tag check faults in the uaccess routines */ + ldr x0, [tsk, THREAD_SCTLR_USER] + clear_mte_async_tcf x0 + #ifdef CONFIG_ARM64_PTR_AUTH alternative_if ARM64_HAS_ADDRESS_AUTH /*
@@ -371,7 +384,6 @@ alternative_if ARM64_HAS_ADDRESS_AUTH * * No kernel C function calls after this. */ - ldr x0, [tsk, THREAD_SCTLR_USER] tbz x0, SCTLR_ELx_ENIA_SHIFT, 1f __ptrauth_keys_install_user tsk, x0, x1, x2 b 2f
@@ -599,8 +611,6 @@ SYM_CODE_START_LOCAL(ret_to_user) cbnz x2, work_pending finish_ret_to_user: user_enter_irqoff - /* Ignore asynchronous tag check faults in the uaccess routines */ - clear_mte_async_tcf enable_step_tsk x19, x2 #ifdef CONFIG_GCC_PLUGIN_STACKLEAK bl stackleak_erase
Signature · 7 lines
2.32.0.93.g670b81a890-goog _______________________________________________ linux-arm-kernel mailing list [email protected] http://lists.infradead.org/mailman/listinfo/linux-arm-kernel
Catalin Marinas
maintainerQuoted 8 lines · Peter Collingbourne
There is no reason to touch TFSR nor issue a DSB unless our task is in asynchronous mode. Since these operations (especially the DSB) may be expensive on certain microarchitectures, only perform them if necessary.
Furthermore, stop clearing TFSR on entry because it will be cleared on exit and it is not necessary to have any particular value in TFSR between entry and exit.
Nitpick: this should be clearer that it's about TFSRE0_EL1 not the kernel's TFSR_EL1.
Anyway, the patch looks fine, no need to repost. I can change the comment.
Thanks.
Signature · 6 lines
Catalin _______________________________________________ linux-arm-kernel mailing list [email protected] http://lists.infradead.org/mailman/listinfo/linux-arm-kernel
Quoted 8 lines · Peter Collingbourne
There is no reason to touch TFSR nor issue a DSB unless our task is in asynchronous mode. Since these operations (especially the DSB) may be expensive on certain microarchitectures, only perform them if necessary.
Furthermore, stop clearing TFSR on entry because it will be cleared on exit and it is not necessary to have any particular value in TFSR between entry and exit.
Applied to arm64 (for-next/mte), thanks!
[1/1] arm64: mte: avoid TFSR related operations unless in async mode
https://git.kernel.org/arm64/c/42b6b10a54f0
Signature · 7 lines
Catalin _______________________________________________ linux-arm-kernel mailing list [email protected] http://lists.infradead.org/mailman/listinfo/linux-arm-kernel