arm64/irqflags: __always_inline the arch_local_irq_*() helpers
PATCH v2 1/1 arm64/irqflags: __always_inline the arch_local_irq_*() helpers
The arch_local_irq_*() wrappers in <asm/irqflags.h> dispatch between two underlying primitives: the __daif_* path on most systems, and the __pmr_* path on builds that use GIC PMR-based masking (Pseudo-NMI). The leaf primitives are already __always_inline, but the wrappers themselves are plain "static inline".
That is unsafe for noinstr callers: nothing prevents the compiler from emitting an out-of-line copy of e.g. arch_local_irq_disable(), and an out-of-line copy can be instrumented (ftrace, kcov, sanitizers), which breaks the noinstr contract on the entry/idle paths that rely on these helpers.
x86 hit and fixed exactly this class of bug in commit 7a745be1cc90 ("x86/entry: __always_inline irqflags for noinstr").
Force-inline all of the arch_local_irq_*() wrappers so they cannot be emitted out-of-line:
- arch_local_irq_enable()
- arch_local_irq_disable()
- arch_local_save_flags()
- arch_irqs_disabled_flags()
- arch_irqs_disabled()
- arch_local_irq_save()
- arch_local_irq_restore()
The primary motivation is noinstr safety. There is a useful side effect for fleet-wide profiling: when the wrapper is emitted out-of-line, samples taken inside it during the post-WFI IRQ unmask in default_idle_call() are attributed to arch_local_irq_enable rather than default_idle_call(), and the FP-unwinder loses default_idle_call() from the chain.
---
Changes in v2:
- Expand the functions that uses always_inline in arm64
- Link to v1: https://patch.msgid.link/[email protected]
--- arch/arm64/include/asm/irqflags.h | 14 +++++++------- 1 file changed, 7 insertions(+), 7 deletions(-)
arch/arm64/include/asm/irqflags.h +7 -7
@@ -40,7 +40,7 @@ static __always_inline void __pmr_local_irq_enable(void)barrier (); } -static inline void arch_local_irq_enable (void ) +static __always_inline void arch_local_irq_enable(void) {if (system_uses_irq_prio_masking ()) {__pmr_local_irq_enable ();
@@ -68,7 +68,7 @@ static __always_inline void __pmr_local_irq_disable(void)barrier (); } -static inline void arch_local_irq_disable (void ) +static __always_inline void arch_local_irq_disable(void) {if (system_uses_irq_prio_masking ()) {__pmr_local_irq_disable ();
@@ -90,7 +90,7 @@ static __always_inline unsigned long __pmr_local_save_flags(void)/* * Save the current interrupt enable state. */ -static inline unsigned long arch_local_save_flags (void ) +static __always_inline unsigned long arch_local_save_flags (void ) {if (system_uses_irq_prio_masking ()) {return __pmr_local_save_flags ();
@@ -109,7 +109,7 @@ static __always_inline bool __pmr_irqs_disabled_flags(unsigned long flags)return flags !=GIC_PRIO_IRQON ; } -static inline bool arch_irqs_disabled_flags (unsigned long flags) +static __always_inline bool arch_irqs_disabled_flags(unsigned long flags) {if (system_uses_irq_prio_masking ()) {return __pmr_irqs_disabled_flags (flags);
@@ -128,7 +128,7 @@ static __always_inline bool __pmr_irqs_disabled(void)return __pmr_irqs_disabled_flags (__pmr_local_save_flags ()); } -static inline bool arch_irqs_disabled (void ) +static __always_inline bool arch_irqs_disabled(void) {if (system_uses_irq_prio_masking ()) {return __pmr_irqs_disabled ();
@@ -160,7 +160,7 @@ static __always_inline unsigned long __pmr_local_irq_save(void)return flags; } -static inline unsigned long arch_local_irq_save (void ) +static __always_inline unsigned long arch_local_irq_save (void ) {if (system_uses_irq_prio_masking ()) {return __pmr_local_irq_save ();
@@ -187,7 +187,7 @@ static __always_inline void __pmr_local_irq_restore(unsigned long flags)/* * restore saved IRQ state */ -static inline void arch_local_irq_restore (unsigned long flags) +static __always_inline void arch_local_irq_restore(unsigned long flags) {if (system_uses_irq_prio_masking ()) {__pmr_local_irq_restore (flags);
---
base-commit: bee6ea30c48788e18348309f891ed8afbf7702ac
change-id: 20260420-arm64_always_inline-6bc9dd3c17e6
Best regards,
Signature · 1 line
Breno Leitao <[email protected]>
Quoted 114 lines · Breno Leitao
The arch_local_irq_*() wrappers in <
asm/irqflags.h> dispatch between two underlying primitives: the __daif_* path on most systems, and the __pmr_* path on builds that use GIC PMR-based masking (Pseudo-NMI). The leaf primitives are already__always_inline, but the wrappers themselves are plain "static inline".That is unsafe for noinstr callers: nothing prevents the compiler from emitting an out-of-line copy of e.g.
arch_local_irq_disable(), and an out-of-line copy can be instrumented (ftrace, kcov, sanitizers), which breaks the noinstr contract on the entry/idle paths that rely on these helpers.x86 hit and fixed exactly this class of bug in commit
7a745be1cc90("x86/entry:__always_inlineirqflags for noinstr").Force-inline all of the arch_local_irq_*() wrappers so they cannot be emitted out-of-line:
-
arch_local_irq_enable()
-arch_local_irq_disable()
-arch_local_save_flags()
-arch_irqs_disabled_flags()
-arch_irqs_disabled()
-arch_local_irq_save()
-arch_local_irq_restore()The primary motivation is noinstr safety. There is a useful side effect for fleet-wide profiling: when the wrapper is emitted out-of-line, samples taken inside it during the post-WFI IRQ unmask in
default_idle_call()are attributed toarch_local_irq_enablerather thandefault_idle_call(), and the FP-unwinder losesdefault_idle_call()from the chain.Signed-off-by Breno Leitao <[email protected]>---
Changes in v2:
- Expand the functions that usesalways_inlinein arm64
- Link to v1: https://patch.msgid.link/[email protected]--- arch/arm64/include/asm/irqflags.h | 14 +++++++------- 1 file changed, 7 insertions(+), 7 deletions(-)arch/arm64/include/asm/irqflags.h +7
-8@@ -40,7 +40,7 @@ static __always_inline void __pmr_local_irq_enable(void)barrier (); } -static inline void arch_local_irq_enable (void ) +static __always_inline void arch_local_irq_enable(void) {if (system_uses_irq_prio_masking ()) {__pmr_local_irq_enable ();@@ -68,7 +68,7 @@ static __always_inline void __pmr_local_irq_disable(void)barrier (); } -static inline void arch_local_irq_disable (void ) +static __always_inline void arch_local_irq_disable(void) {if (system_uses_irq_prio_masking ()) {__pmr_local_irq_disable ();@@ -90,7 +90,7 @@ static __always_inline unsigned long __pmr_local_save_flags(void)/* * Save the current interrupt enable state. */ -static inline unsigned long arch_local_save_flags (void ) +static __always_inline unsigned long arch_local_save_flags (void ) {if (system_uses_irq_prio_masking ()) {return __pmr_local_save_flags ();@@ -109,7 +109,7 @@ static __always_inline bool __pmr_irqs_disabled_flags(unsigned long flags)return flags !=GIC_PRIO_IRQON ; } -static inline bool arch_irqs_disabled_flags (unsigned long flags) +static __always_inline bool arch_irqs_disabled_flags(unsigned long flags) {if (system_uses_irq_prio_masking ()) {return __pmr_irqs_disabled_flags (flags);@@ -128,7 +128,7 @@ static __always_inline bool __pmr_irqs_disabled(void)return __pmr_irqs_disabled_flags (__pmr_local_save_flags ()); } -static inline bool arch_irqs_disabled (void ) +static __always_inline bool arch_irqs_disabled(void) {if (system_uses_irq_prio_masking ()) {return __pmr_irqs_disabled ();@@ -160,7 +160,7 @@ static __always_inline unsigned long __pmr_local_irq_save(void)return flags; } -static inline unsigned long arch_local_irq_save (void ) +static __always_inline unsigned long arch_local_irq_save (void ) {if (system_uses_irq_prio_masking ()) {return __pmr_local_irq_save ();@@ -187,7 +187,7 @@ static __always_inline void __pmr_local_irq_restore(unsigned long flags)/* * restore saved IRQ state */ -static inline void arch_local_irq_restore (unsigned long flags) +static __always_inline void arch_local_irq_restore(unsigned long flags) {if (system_uses_irq_prio_masking ()) {__pmr_local_irq_restore (flags); ---base-commit:
bee6ea30c48788e18348309f891ed8afbf7702ac
change-id: 20260420-arm64_always_inline-6bc9dd3c17e6
Quoted 4 lines
Best regards,
Signature · 1 line
Breno Leitao <[email protected]>
Looks correct to what was discussed in V1. FWIW:
Quoted 26 lines · Breno Leitao
The arch_local_irq_*() wrappers in <
asm/irqflags.h> dispatch between two underlying primitives: the __daif_* path on most systems, and the __pmr_* path on builds that use GIC PMR-based masking (Pseudo-NMI). The leaf primitives are already__always_inline, but the wrappers themselves are plain "static inline".That is unsafe for noinstr callers: nothing prevents the compiler from emitting an out-of-line copy of e.g.
arch_local_irq_disable(), and an out-of-line copy can be instrumented (ftrace, kcov, sanitizers), which breaks the noinstr contract on the entry/idle paths that rely on these helpers.x86 hit and fixed exactly this class of bug in commit
7a745be1cc90("x86/entry:__always_inlineirqflags for noinstr").Force-inline all of the arch_local_irq_*() wrappers so they cannot be emitted out-of-line:
-
arch_local_irq_enable()
-arch_local_irq_disable()
-arch_local_save_flags()
-arch_irqs_disabled_flags()
-arch_irqs_disabled()
-arch_local_irq_save()
-arch_local_irq_restore()
Quoted 6 lines
The primary motivation is noinstr safety. There is a useful side effect for fleet-wide profiling: when the wrapper is emitted out-of-line, samples taken inside it during the post-WFI IRQ unmask in
default_idle_call()are attributed toarch_local_irq_enablerather thandefault_idle_call(), and the FP-unwinder losesdefault_idle_call()from the chain.
FWIW I run scripts/bloat-o-meter on the kernel with and without the patch, and the the code size is mostly the same. here is the result:
add/remove: 4/12 grow/shrink: 40/0 up/down: 1684/-652 (1032)
Function old new delta
__schedule 8892 9024 +132
irqentry_exit 816 892 +76
lockdep_hardirqs_off 396 452 +56
lock_is_held_type 412 468 +56
ct_idle_exit 76 132 +56
cpu_idle_poll 304 360 +56
arch_stack_walk_reliable 1152 1196 +44
arch_stack_walk 1184 1228 +44
arch_bpf_stack_walk 996 1040 +44
lockdep_hardirqs_on 464 504 +40
el0_watchpt 576 616 +40
el0_undef 560 600 +40
el0_sys 560 600 +40
el0_sve_acc 560 600 +40
el0_svc 600 640 +40
el0_sp 564 604 +40
el0_softstp 728 768 +40
el0_sme_acc 560 600 +40
el0_pc 740 780 +40
el0_mops 560 600 +40
el0_inv 564 604 +40
el0_interrupt 656 696 +40
el0_ia 716 756 +40
el0_gcs 560 600 +40
el0_fpsimd_exc 560 600 +40
el0_fpsimd_acc 560 600 +40
el0_fpac 560 600 +40
el0_da 568 608 +40
el0_bti 552 592 +40
el0_brk64 560 600 +40
el0_breakpt 720 760 +40
asm_exit_to_user_mode 416 456 +40
__el0_error_handler_common 592 632 +40
cpuidle_enter_state 1220 1248 +28
check_preemption_disabled 228 252 +24
default_idle_call 252 272 +20
ct_kernel_enter 388 404 +16
ct_idle_enter 52 68 +16
look_up_lock_class 364 376 +12
check_flags 492 504 +12
__CortexA53843419_FFFF800081146000 - 8 +8
__CortexA53843419_FFFF8000809C3004 - 8 +8
__CortexA53843419_FFFF8000809AE000 - 8 +8
__CortexA53843419_FFFF800080248004 - 8 +8
__CortexA53843419_FFFF80008100C000 8 - -8
__CortexA53843419_FFFF8000809A9000 8 - -8
__CortexA53843419_FFFF8000809A8004 8 - -8
__CortexA53843419_FFFF800080448008 8 - -8
__CortexA53843419_FFFF8000801EE000 8 - -8
arch_local_irq_restore 48 - -48
arch_local_save_flags 80 - -80
arch_local_irq_save 80 - -80
arch_local_irq_enable 84 - -84
arch_local_irq_disable 96 - -96
arch_irqs_disabled_flags 96 - -96
arch_irqs_disabled 128 - -128
Total: Before=163062863, After=163063895, chg +0.00%
Catalin Marinas
maintainerQuoted 10 lines · Breno Leitao
Force-inline all of the arch_local_irq_*() wrappers so they cannot be emitted out-of-line:
-
arch_local_irq_enable()
-arch_local_irq_disable()
-arch_local_save_flags()
-arch_irqs_disabled_flags()
-arch_irqs_disabled()
-arch_local_irq_save()
-arch_local_irq_restore()
I'll queue this, thanks!
I think we should also do local_daif_{mask,restore,inherit} as they seem to be called from noinstr locations in entry-common.c.
Signature · 1 line
Catalin
Mark Rutland
reviewerQuoted 12 lines · Catalin Marinas
Quoted 10 lines · Breno Leitao
Force-inline all of the arch_local_irq_*() wrappers so they cannot be emitted out-of-line:
-
arch_local_irq_enable()
-arch_local_irq_disable()
-arch_local_save_flags()
-arch_irqs_disabled_flags()
-arch_irqs_disabled()
-arch_local_irq_save()
-arch_local_irq_restore()
Quoted 4 lines
I'll queue this, thanks!
I think we should also do local_daif_{mask,restore,inherit} as they seem to be called from noinstr locations in
entry-common.c.
I agree we probably should mark those as __always_inline, but I beleive they're safe as-is. For their current usage in entry-common.c they're only called between between enter_from_*_mode() and exit_to_*_mode(), in a period where instrumentation is safe.
Mark.
PATCH 1/1 arm64/daifflags: Make local_daif_*() helpers __always_inline
Make sure those helpers are always inlined and instrumentation safe.
--- arch/arm64/include/asm/daifflags.h | 10 +++++----- 1 file changed, 5 insertions(+), 5 deletions(-)
arch/arm64/include/asm/daifflags.h +5 -5
@@ -12,66 +12,66 @@#include <asm/cpufeature.h> #include <asm/ptrace.h> #define DAIF_PROCCTX 0#define DAIF_PROCCTX_NOIRQ (PSR_I_BIT | PSR_F_BIT)#define DAIF_ERRCTX (PSR_A_BIT | PSR_I_BIT | PSR_F_BIT)#define DAIF_MASK (PSR_D_BIT | PSR_A_BIT | PSR_I_BIT | PSR_F_BIT)/* mask/save/unmask/restore all exceptions, including interrupts. */ -static inline void local_daif_mask (void ) +static __always_inline void local_daif_mask(void) {WARN_ON (system_has_prio_mask_debugging () && (read_sysreg_s (SYS_ICC_PMR_EL1 ) == (GIC_PRIO_IRQOFF |GIC_PRIO_PSR_I_SET ))); asmvolatile ("msr daifset, #0xf // local_daif_mask\n" : : :"memory" );/* Don't really care for a dsb here, we don't intend to enable IRQs */ if (system_uses_irq_prio_masking ())gic_write_pmr (GIC_PRIO_IRQON |GIC_PRIO_PSR_I_SET );trace_hardirqs_off (); } -static inline unsigned long local_daif_save_flags (void ) +static __always_inline unsigned long local_daif_save_flags (void ) {unsigned long flags; flags =read_sysreg (daif);if (system_uses_irq_prio_masking ()) {/* If IRQs are masked with PMR, reflect it in the flags */ if (read_sysreg_s (SYS_ICC_PMR_EL1 ) !=GIC_PRIO_IRQON ) flags |=PSR_I_BIT |PSR_F_BIT ; }return flags; } -static inline unsigned long local_daif_save (void ) +static __always_inline unsigned long local_daif_save (void ) {unsigned long flags; flags =local_daif_save_flags ();local_daif_mask ();return flags; } -static inline void local_daif_restore (unsigned long flags) +static __always_inline void local_daif_restore(unsigned long flags) {bool irq_disabled = flags &PSR_I_BIT ;WARN_ON (system_has_prio_mask_debugging () && (read_sysreg (daif) & (PSR_I_BIT |PSR_F_BIT )) != (PSR_I_BIT |PSR_F_BIT ));if (!irq_disabled) {trace_hardirqs_on ();if (system_uses_irq_prio_masking ()) {
@@ -117,21 +117,21 @@ static inline void local_daif_restore(unsigned long flags)write_sysreg (flags, daif);if (irq_disabled)trace_hardirqs_off (); }/* * Called by synchronous exception handlers to restore the DAIF bits that were * modified by taking an exception. */ -static inline void local_daif_inherit (struct pt_regs *regs) +static __always_inline void local_daif_inherit(struct pt_regs *regs) {unsigned long flags = regs->pstate &DAIF_MASK ;if (!regs_irqs_disabled (regs))trace_hardirqs_on ();if (system_uses_irq_prio_masking ())gic_write_pmr (regs->pmr);/*
Signature · 1 line
2.54.0
Will Deacon
maintainerRe: [PATCH] arm64/daifflags: Make local_daif_*() helpers __always_inline
Quoted 6 lines · Leonardo Bras
Make sure those helpers are always inlined and instrumentation safe.
Signed-off-by Leonardo Bras <[email protected]>--- arch/arm64/include/asm/daifflags.h | 10 +++++----- 1 file changed, 5 insertions(+), 5 deletions(-)
b4 really doesn't like this patch being a reply to another one, so I'm just replying to say that I've picked it up manually.
Cheers,
Will
Re: [PATCH] arm64/daifflags: Make local_daif_*() helpers __always_inline
Quoted 8 lines · Will Deacon
Quoted 6 lines · Leonardo Bras
Make sure those helpers are always inlined and instrumentation safe.
Signed-off-by Leonardo Bras <[email protected]>--- arch/arm64/include/asm/daifflags.h | 10 +++++----- 1 file changed, 5 insertions(+), 5 deletions(-)
Quoted 6 lines
b4 really doesn't like this patch being a reply to another one, so I'm just replying to say that I've picked it up manually.
Cheers,
Will
Hi Will,
Noted. I will avoid doing this in the future.
Thanks!
Leo
Quoted 13 lines · Mark Rutland
Quoted 12 lines · Catalin Marinas
Quoted 10 lines · Breno Leitao
Force-inline all of the arch_local_irq_*() wrappers so they cannot be emitted out-of-line:
-
arch_local_irq_enable()
-arch_local_irq_disable()
-arch_local_save_flags()
-arch_irqs_disabled_flags()
-arch_irqs_disabled()
-arch_local_irq_save()
-arch_local_irq_restore()
Quoted 7 lines
Quoted 4 lines
I'll queue this, thanks!
I think we should also do local_daif_{mask,restore,inherit} as they seem to be called from noinstr locations in
entry-common.c.I agree we probably should mark those as
__always_inline, but I beleive they're safe as-is.
Returning to this topic, I've also observed arch_counter_get_cntpct() appearing in backtraces during lock spinning, which obscures the actual code location and makes debugging less straightforward.
This scenario is common at Meta when a remote function call targets a stuck CPU, resulting in an infinite loop, and dying at arch_counter_get_cntpc().
Is it worth making arch_counter_get_cntpc() always inline as well?
[ T18] Sending NMI from CPU 1 to CPUs 18: [ C18] NMI backtrace for cpu 18 [ C18] CPU: 18 UID: 0 PID: 7467 Comm: dynoKernelMon Kdump: loaded Not tainted 6.16.1-0_fbk3_0_gd6c130b80483 #1 NONE [ C18] pstate: 63401009 (nZCv daif +PAN -UAO +TCO +DIT +SSBS BTYPE=--) [ C18] pc : arch_counter_get_cntpct+0x14/0x18 [ C18] lr : ktime_get_mono_fast_ns+0x5c/0x1a8 [ C18] sp : ffff8000d43afa30 [ C18] x29: ffff8000d43afa30 x28: 00000000000f4240 x27: 0000000000000000 [ C18] x26: 0000028ac8fe4369 x25: 0000000000001388 x24: 000000000054fb54 [ C18] x23: ffff800082fbd000 x22: 0000028f22c480dd x21: ffff800081b60000 [ C18] x20: ffff800080c71058 x19: ffff800082fb8848 x18: 0000000000000018 [ C18] x17: 0000000000000058 x16: 00000000ffffffff x15: 00000000ffff0000 [ C18] x14: 0000000000000002 x13: 0000000000000003 x12: 000000000000ffd8 [ C18] x11: ffff8000d43affd8 x10: 0000000000000038 x9 : 0000000000000000 [ C18] x8 : ffff8000d43afa30 x7 : ff7fff7f7f7f7f7f x6 : 000000000000000a [ C18] x5 : ffff8000832772ba x4 : 0000000000000000 x3 : 0000000000000000 [ C18] x2 : 0000000000000000 x1 : 000000000014b4c0 x0 : 000002a87573b7a0 [ C18] Call trace: [ C18] arch_counter_get_cntpct+0x14/0x18 (P) [ C18] smp_call_function_single+0x2d0/0xd90 [ C18] event_function_call+0x80/0x2e0 [ C18] _perf_event_disable+0x5c/0xf0 [ C18] perf_ioctl+0x170/0x890 [ C18] __arm64_sys_ioctl+0xbb0/0xd28 [ C18] invoke_syscall+0x4c/0xd0 [ C18] do_el0_svc+0x80/0xb8 [ C18] el0_svc+0x30/0xf0 [ C18] el0t_64_sync_handler+0x70/0x100 [ C18] el0t_64_sync+0x17c/0x180
Thanks,
--breno
Quoted 6 lines · Breno Leitao
The arch_local_irq_*() wrappers in <
asm/irqflags.h> dispatch between two underlying primitives: the __daif_* path on most systems, and the __pmr_* path on builds that use GIC PMR-based masking (Pseudo-NMI). The leaf primitives are already__always_inline, but the wrappers themselves are plain "static inline".
Quoted 7 lines
That is unsafe for noinstr callers: nothing prevents the compiler from emitting an out-of-line copy of e.g.
arch_local_irq_disable(), and an out-of-line copy can be instrumented (ftrace, kcov, sanitizers), which breaks the noinstr contract on the entry/idle paths that rely on these helpers.[...]
Applied to arm64 (for-next/fixes), thanks!
[1/1] arm64/irqflags: __always_inline the arch_local_irq_*() helpers
https://git.kernel.org/arm64/c/caecde119e34