Skip to contentSkip to conversation

arm64/irqflags: __always_inline the arch_local_irq_*() helpers

PATCH Applied Breno Leitao 5 participants 10 messages Apr 21 – Jun 15 arm64

lore
Share & tools
lore
#0 lore raw

PATCH v2 1/1 arm64/irqflags: __always_inline the arch_local_irq_*() helpers

The arch_local_irq_*() wrappers in <asm/irqflags.h> dispatch between two underlying primitives: the __daif_* path on most systems, and the __pmr_* path on builds that use GIC PMR-based masking (Pseudo-NMI). The leaf primitives are already __always_inline, but the wrappers themselves are plain "static inline".

That is unsafe for noinstr callers: nothing prevents the compiler from emitting an out-of-line copy of e.g. arch_local_irq_disable(), and an out-of-line copy can be instrumented (ftrace, kcov, sanitizers), which breaks the noinstr contract on the entry/idle paths that rely on these helpers.

x86 hit and fixed exactly this class of bug in commit 7a745be1cc90 ("x86/entry: __always_inline irqflags for noinstr").

Force-inline all of the arch_local_irq_*() wrappers so they cannot be emitted out-of-line:

- arch_local_irq_enable()
- arch_local_irq_disable()
- arch_local_save_flags()
- arch_irqs_disabled_flags()
- arch_irqs_disabled()
- arch_local_irq_save()
- arch_local_irq_restore()

The primary motivation is noinstr safety. There is a useful side effect for fleet-wide profiling: when the wrapper is emitted out-of-line, samples taken inside it during the post-WFI IRQ unmask in default_idle_call() are attributed to arch_local_irq_enable rather than default_idle_call(), and the FP-unwinder loses default_idle_call() from the chain.

---
Changes in v2:
- Expand the functions that uses always_inline in arm64
- Link to v1: https://patch.msgid.link/[email protected]

---
 arch/arm64/include/asm/irqflags.h | 14 +++++++-------
 1 file changed, 7 insertions(+), 7 deletions(-)
arch/arm64/include/asm/irqflags.h +7 -7
index d4d7451c2c129..a8cb5a5c93b78 100644
@@ -40,7 +40,7 @@ static __always_inline void __pmr_local_irq_enable(void)
 	barrier();
 }
 
-static inline void arch_local_irq_enable(void)
+static __always_inline void arch_local_irq_enable(void)
 {
 	if (system_uses_irq_prio_masking()) {
 		__pmr_local_irq_enable();
@@ -68,7 +68,7 @@ static __always_inline void __pmr_local_irq_disable(void)
 	barrier();
 }
 
-static inline void arch_local_irq_disable(void)
+static __always_inline void arch_local_irq_disable(void)
 {
 	if (system_uses_irq_prio_masking()) {
 		__pmr_local_irq_disable();
@@ -90,7 +90,7 @@ static __always_inline unsigned long __pmr_local_save_flags(void)
 /*
  * Save the current interrupt enable state.
  */
-static inline unsigned long arch_local_save_flags(void)
+static __always_inline unsigned long arch_local_save_flags(void)
 {
 	if (system_uses_irq_prio_masking()) {
 		return __pmr_local_save_flags();
@@ -109,7 +109,7 @@ static __always_inline bool __pmr_irqs_disabled_flags(unsigned long flags)
 	return flags != GIC_PRIO_IRQON;
 }
 
-static inline bool arch_irqs_disabled_flags(unsigned long flags)
+static __always_inline bool arch_irqs_disabled_flags(unsigned long flags)
 {
 	if (system_uses_irq_prio_masking()) {
 		return __pmr_irqs_disabled_flags(flags);
@@ -128,7 +128,7 @@ static __always_inline bool __pmr_irqs_disabled(void)
 	return __pmr_irqs_disabled_flags(__pmr_local_save_flags());
 }
 
-static inline bool arch_irqs_disabled(void)
+static __always_inline bool arch_irqs_disabled(void)
 {
 	if (system_uses_irq_prio_masking()) {
 		return __pmr_irqs_disabled();
@@ -160,7 +160,7 @@ static __always_inline unsigned long __pmr_local_irq_save(void)
 	return flags;
 }
 
-static inline unsigned long arch_local_irq_save(void)
+static __always_inline unsigned long arch_local_irq_save(void)
 {
 	if (system_uses_irq_prio_masking()) {
 		return __pmr_local_irq_save();
@@ -187,7 +187,7 @@ static __always_inline void __pmr_local_irq_restore(unsigned long flags)
 /*
  * restore saved IRQ state
  */
-static inline void arch_local_irq_restore(unsigned long flags)
+static __always_inline void arch_local_irq_restore(unsigned long flags)
 {
 	if (system_uses_irq_prio_masking()) {
 		__pmr_local_irq_restore(flags);

---
base-commit: bee6ea30c48788e18348309f891ed8afbf7702ac
change-id: 20260420-arm64_always_inline-6bc9dd3c17e6

Best regards,

Signature · 1 line
Breno Leitao <[email protected]>
Reviewed-by on v2 1/1
#1 lore raw
Quoted 114 lines · Breno Leitao

The arch_local_irq_*() wrappers in <asm/irqflags.h> dispatch between two underlying primitives: the __daif_* path on most systems, and the __pmr_* path on builds that use GIC PMR-based masking (Pseudo-NMI). The leaf primitives are already __always_inline, but the wrappers themselves are plain "static inline".

That is unsafe for noinstr callers: nothing prevents the compiler from emitting an out-of-line copy of e.g. arch_local_irq_disable(), and an out-of-line copy can be instrumented (ftrace, kcov, sanitizers), which breaks the noinstr contract on the entry/idle paths that rely on these helpers.

x86 hit and fixed exactly this class of bug in commit 7a745be1cc90 ("x86/entry: __always_inline irqflags for noinstr").

Force-inline all of the arch_local_irq_*() wrappers so they cannot be emitted out-of-line:

- arch_local_irq_enable()
- arch_local_irq_disable()
- arch_local_save_flags()
- arch_irqs_disabled_flags()
- arch_irqs_disabled()
- arch_local_irq_save()
- arch_local_irq_restore()

The primary motivation is noinstr safety. There is a useful side effect for fleet-wide profiling: when the wrapper is emitted out-of-line, samples taken inside it during the post-WFI IRQ unmask in default_idle_call() are attributed to arch_local_irq_enable rather than default_idle_call(), and the FP-unwinder loses default_idle_call() from the chain.

---
Changes in v2:
- Expand the functions that uses always_inline in arm64
- Link to v1: https://patch.msgid.link/[email protected]

---
 arch/arm64/include/asm/irqflags.h | 14 +++++++-------
 1 file changed, 7 insertions(+), 7 deletions(-)
arch/arm64/include/asm/irqflags.h +7 -8
index d4d7451c2c129..a8cb5a5c93b78 100644
@@ -40,7 +40,7 @@ static __always_inline void __pmr_local_irq_enable(void)
 	barrier();
 }
 
-static inline void arch_local_irq_enable(void)
+static __always_inline void arch_local_irq_enable(void)
 {
 	if (system_uses_irq_prio_masking()) {
 		__pmr_local_irq_enable();
@@ -68,7 +68,7 @@ static __always_inline void __pmr_local_irq_disable(void)
 	barrier();
 }
 
-static inline void arch_local_irq_disable(void)
+static __always_inline void arch_local_irq_disable(void)
 {
 	if (system_uses_irq_prio_masking()) {
 		__pmr_local_irq_disable();
@@ -90,7 +90,7 @@ static __always_inline unsigned long __pmr_local_save_flags(void)
 /*
  * Save the current interrupt enable state.
  */
-static inline unsigned long arch_local_save_flags(void)
+static __always_inline unsigned long arch_local_save_flags(void)
 {
 	if (system_uses_irq_prio_masking()) {
 		return __pmr_local_save_flags();
@@ -109,7 +109,7 @@ static __always_inline bool __pmr_irqs_disabled_flags(unsigned long flags)
 	return flags != GIC_PRIO_IRQON;
 }
 
-static inline bool arch_irqs_disabled_flags(unsigned long flags)
+static __always_inline bool arch_irqs_disabled_flags(unsigned long flags)
 {
 	if (system_uses_irq_prio_masking()) {
 		return __pmr_irqs_disabled_flags(flags);
@@ -128,7 +128,7 @@ static __always_inline bool __pmr_irqs_disabled(void)
 	return __pmr_irqs_disabled_flags(__pmr_local_save_flags());
 }
 
-static inline bool arch_irqs_disabled(void)
+static __always_inline bool arch_irqs_disabled(void)
 {
 	if (system_uses_irq_prio_masking()) {
 		return __pmr_irqs_disabled();
@@ -160,7 +160,7 @@ static __always_inline unsigned long __pmr_local_irq_save(void)
 	return flags;
 }
 
-static inline unsigned long arch_local_irq_save(void)
+static __always_inline unsigned long arch_local_irq_save(void)
 {
 	if (system_uses_irq_prio_masking()) {
 		return __pmr_local_irq_save();
@@ -187,7 +187,7 @@ static __always_inline void __pmr_local_irq_restore(unsigned long flags)
 /*
  * restore saved IRQ state
  */
-static inline void arch_local_irq_restore(unsigned long flags)
+static __always_inline void arch_local_irq_restore(unsigned long flags)
 {
 	if (system_uses_irq_prio_masking()) {
 		__pmr_local_irq_restore(flags);
 
---

base-commit: bee6ea30c48788e18348309f891ed8afbf7702ac
change-id: 20260420-arm64_always_inline-6bc9dd3c17e6

Quoted 4 lines

Best regards,

Signature · 1 line
Breno Leitao <[email protected]>

Looks correct to what was discussed in V1. FWIW:

#2 lore raw
In reply to Breno Leitao
Quoted 26 lines · Breno Leitao

The arch_local_irq_*() wrappers in <asm/irqflags.h> dispatch between two underlying primitives: the __daif_* path on most systems, and the __pmr_* path on builds that use GIC PMR-based masking (Pseudo-NMI). The leaf primitives are already __always_inline, but the wrappers themselves are plain "static inline".

That is unsafe for noinstr callers: nothing prevents the compiler from emitting an out-of-line copy of e.g. arch_local_irq_disable(), and an out-of-line copy can be instrumented (ftrace, kcov, sanitizers), which breaks the noinstr contract on the entry/idle paths that rely on these helpers.

x86 hit and fixed exactly this class of bug in commit 7a745be1cc90 ("x86/entry: __always_inline irqflags for noinstr").

Force-inline all of the arch_local_irq_*() wrappers so they cannot be emitted out-of-line:

- arch_local_irq_enable()
- arch_local_irq_disable()
- arch_local_save_flags()
- arch_irqs_disabled_flags()
- arch_irqs_disabled()
- arch_local_irq_save()
- arch_local_irq_restore()

Quoted 6 lines

The primary motivation is noinstr safety. There is a useful side effect for fleet-wide profiling: when the wrapper is emitted out-of-line, samples taken inside it during the post-WFI IRQ unmask in default_idle_call() are attributed to arch_local_irq_enable rather than default_idle_call(), and the FP-unwinder loses default_idle_call() from the chain.

FWIW I run scripts/bloat-o-meter on the kernel with and without the patch, and the the code size is mostly the same. here is the result:

add/remove: 4/12 grow/shrink: 40/0 up/down: 1684/-652 (1032)
Function old new delta
__schedule 8892 9024 +132
irqentry_exit 816 892 +76
lockdep_hardirqs_off 396 452 +56
lock_is_held_type 412 468 +56
ct_idle_exit 76 132 +56
cpu_idle_poll 304 360 +56
arch_stack_walk_reliable 1152 1196 +44
arch_stack_walk 1184 1228 +44
arch_bpf_stack_walk 996 1040 +44
lockdep_hardirqs_on 464 504 +40
el0_watchpt 576 616 +40
el0_undef 560 600 +40
el0_sys 560 600 +40
el0_sve_acc 560 600 +40
el0_svc 600 640 +40
el0_sp 564 604 +40
el0_softstp 728 768 +40
el0_sme_acc 560 600 +40
el0_pc 740 780 +40
el0_mops 560 600 +40
el0_inv 564 604 +40
el0_interrupt 656 696 +40
el0_ia 716 756 +40
el0_gcs 560 600 +40
el0_fpsimd_exc 560 600 +40
el0_fpsimd_acc 560 600 +40
el0_fpac 560 600 +40
el0_da 568 608 +40
el0_bti 552 592 +40
el0_brk64 560 600 +40
el0_breakpt 720 760 +40
asm_exit_to_user_mode 416 456 +40
__el0_error_handler_common 592 632 +40
cpuidle_enter_state 1220 1248 +28
check_preemption_disabled 228 252 +24
default_idle_call 252 272 +20
ct_kernel_enter 388 404 +16
ct_idle_enter 52 68 +16
look_up_lock_class 364 376 +12
check_flags 492 504 +12
__CortexA53843419_FFFF800081146000 - 8 +8
__CortexA53843419_FFFF8000809C3004 - 8 +8
__CortexA53843419_FFFF8000809AE000 - 8 +8
__CortexA53843419_FFFF800080248004 - 8 +8
__CortexA53843419_FFFF80008100C000 8 - -8
__CortexA53843419_FFFF8000809A9000 8 - -8
__CortexA53843419_FFFF8000809A8004 8 - -8
__CortexA53843419_FFFF800080448008 8 - -8
__CortexA53843419_FFFF8000801EE000 8 - -8
arch_local_irq_restore 48 - -48
arch_local_save_flags 80 - -80
arch_local_irq_save 80 - -80
arch_local_irq_enable 84 - -84
arch_local_irq_disable 96 - -96
arch_irqs_disabled_flags 96 - -96
arch_irqs_disabled 128 - -128
Total: Before=163062863, After=163063895, chg +0.00%

maintainer
#3 lore raw
In reply to Breno Leitao
Quoted 10 lines · Breno Leitao

Force-inline all of the arch_local_irq_*() wrappers so they cannot be emitted out-of-line:

- arch_local_irq_enable()
- arch_local_irq_disable()
- arch_local_save_flags()
- arch_irqs_disabled_flags()
- arch_irqs_disabled()
- arch_local_irq_save()
- arch_local_irq_restore()

I'll queue this, thanks!

I think we should also do local_daif_{mask,restore,inherit} as they seem to be called from noinstr locations in entry-common.c.

Signature · 1 line
Catalin
reviewer
#4 lore raw
Quoted 12 lines · Catalin Marinas
Quoted 10 lines · Breno Leitao

Force-inline all of the arch_local_irq_*() wrappers so they cannot be emitted out-of-line:

- arch_local_irq_enable()
- arch_local_irq_disable()
- arch_local_save_flags()
- arch_irqs_disabled_flags()
- arch_irqs_disabled()
- arch_local_irq_save()
- arch_local_irq_restore()

Quoted 4 lines

I'll queue this, thanks!

I think we should also do local_daif_{mask,restore,inherit} as they seem to be called from noinstr locations in entry-common.c.

I agree we probably should mark those as __always_inline, but I beleive they're safe as-is. For their current usage in entry-common.c they're only called between between enter_from_*_mode() and exit_to_*_mode(), in a period where instrumentation is safe.

Mark.

#5 lore raw

PATCH 1/1 arm64/daifflags: Make local_daif_*() helpers __always_inline

Make sure those helpers are always inlined and instrumentation safe.

---
 arch/arm64/include/asm/daifflags.h | 10 +++++-----
 1 file changed, 5 insertions(+), 5 deletions(-)
arch/arm64/include/asm/daifflags.h +5 -5
index 5fca48009043..795b35128467 100644
@@ -12,66 +12,66 @@
 #include <asm/cpufeature.h>
 #include <asm/ptrace.h>
 
 #define DAIF_PROCCTX		0
 #define DAIF_PROCCTX_NOIRQ	(PSR_I_BIT | PSR_F_BIT)
 #define DAIF_ERRCTX		(PSR_A_BIT | PSR_I_BIT | PSR_F_BIT)
 #define DAIF_MASK		(PSR_D_BIT | PSR_A_BIT | PSR_I_BIT | PSR_F_BIT)
 
 
 /* mask/save/unmask/restore all exceptions, including interrupts. */
-static inline void local_daif_mask(void)
+static __always_inline void local_daif_mask(void)
 {
 	WARN_ON(system_has_prio_mask_debugging() &&
 		(read_sysreg_s(SYS_ICC_PMR_EL1) == (GIC_PRIO_IRQOFF |
 						    GIC_PRIO_PSR_I_SET)));
 
 	asm volatile(
 		"msr	daifset, #0xf		// local_daif_mask\n"
 		:
 		:
 		: "memory");
 
 	/* Don't really care for a dsb here, we don't intend to enable IRQs */
 	if (system_uses_irq_prio_masking())
 		gic_write_pmr(GIC_PRIO_IRQON | GIC_PRIO_PSR_I_SET);
 
 	trace_hardirqs_off();
 }
 
-static inline unsigned long local_daif_save_flags(void)
+static __always_inline unsigned long local_daif_save_flags(void)
 {
 	unsigned long flags;
 
 	flags = read_sysreg(daif);
 
 	if (system_uses_irq_prio_masking()) {
 		/* If IRQs are masked with PMR, reflect it in the flags */
 		if (read_sysreg_s(SYS_ICC_PMR_EL1) != GIC_PRIO_IRQON)
 			flags |= PSR_I_BIT | PSR_F_BIT;
 	}
 
 	return flags;
 }
 
-static inline unsigned long local_daif_save(void)
+static __always_inline unsigned long local_daif_save(void)
 {
 	unsigned long flags;
 
 	flags = local_daif_save_flags();
 
 	local_daif_mask();
 
 	return flags;
 }
 
-static inline void local_daif_restore(unsigned long flags)
+static __always_inline void local_daif_restore(unsigned long flags)
 {
 	bool irq_disabled = flags & PSR_I_BIT;
 
 	WARN_ON(system_has_prio_mask_debugging() &&
 		(read_sysreg(daif) & (PSR_I_BIT | PSR_F_BIT)) != (PSR_I_BIT | PSR_F_BIT));
 
 	if (!irq_disabled) {
 		trace_hardirqs_on();
 
 		if (system_uses_irq_prio_masking()) {
@@ -117,21 +117,21 @@ static inline void local_daif_restore(unsigned long flags)
 	write_sysreg(flags, daif);
 
 	if (irq_disabled)
 		trace_hardirqs_off();
 }
 
 /*
  * Called by synchronous exception handlers to restore the DAIF bits that were
  * modified by taking an exception.
  */
-static inline void local_daif_inherit(struct pt_regs *regs)
+static __always_inline void local_daif_inherit(struct pt_regs *regs)
 {
 	unsigned long flags = regs->pstate & DAIF_MASK;
 
 	if (!regs_irqs_disabled(regs))
 		trace_hardirqs_on();
 
 	if (system_uses_irq_prio_masking())
 		gic_write_pmr(regs->pmr);
 
 	/*
Signature · 1 line
2.54.0
maintainer
#6 lore raw

Re: [PATCH] arm64/daifflags: Make local_daif_*() helpers __always_inline

Quoted 6 lines · Leonardo Bras

Make sure those helpers are always inlined and instrumentation safe.

---
 arch/arm64/include/asm/daifflags.h | 10 +++++-----
 1 file changed, 5 insertions(+), 5 deletions(-)

b4 really doesn't like this patch being a reply to another one, so I'm just replying to say that I've picked it up manually.

Cheers,

Will

#7 lore raw

Re: [PATCH] arm64/daifflags: Make local_daif_*() helpers __always_inline

Quoted 8 lines · Will Deacon
Quoted 6 lines · Leonardo Bras

Make sure those helpers are always inlined and instrumentation safe.

---
 arch/arm64/include/asm/daifflags.h | 10 +++++-----
 1 file changed, 5 insertions(+), 5 deletions(-)
Quoted 6 lines

b4 really doesn't like this patch being a reply to another one, so I'm just replying to say that I've picked it up manually.

Cheers,

Will

Hi Will,

Noted. I will avoid doing this in the future.

Thanks!
Leo

#8 lore raw
In reply to Mark Rutland
Quoted 13 lines · Mark Rutland
Quoted 12 lines · Catalin Marinas
Quoted 10 lines · Breno Leitao

Force-inline all of the arch_local_irq_*() wrappers so they cannot be emitted out-of-line:

- arch_local_irq_enable()
- arch_local_irq_disable()
- arch_local_save_flags()
- arch_irqs_disabled_flags()
- arch_irqs_disabled()
- arch_local_irq_save()
- arch_local_irq_restore()

Quoted 7 lines
Quoted 4 lines

I'll queue this, thanks!

I think we should also do local_daif_{mask,restore,inherit} as they seem to be called from noinstr locations in entry-common.c.

I agree we probably should mark those as __always_inline, but I beleive they're safe as-is.

Returning to this topic, I've also observed arch_counter_get_cntpct() appearing in backtraces during lock spinning, which obscures the actual code location and makes debugging less straightforward.

This scenario is common at Meta when a remote function call targets a stuck CPU, resulting in an infinite loop, and dying at arch_counter_get_cntpc().

Is it worth making arch_counter_get_cntpc() always inline as well?

 [     T18] Sending NMI from CPU 1 to CPUs 18:
 [     C18] NMI backtrace for cpu 18
 [     C18] CPU: 18 UID: 0 PID: 7467 Comm: dynoKernelMon Kdump: loaded Not tainted 6.16.1-0_fbk3_0_gd6c130b80483 #1 NONE
 [     C18] pstate: 63401009 (nZCv daif +PAN -UAO +TCO +DIT +SSBS BTYPE=--)
 [     C18] pc : arch_counter_get_cntpct+0x14/0x18
 [     C18] lr : ktime_get_mono_fast_ns+0x5c/0x1a8
 [     C18] sp : ffff8000d43afa30
 [     C18] x29: ffff8000d43afa30 x28: 00000000000f4240 x27: 0000000000000000
 [     C18] x26: 0000028ac8fe4369 x25: 0000000000001388 x24: 000000000054fb54
 [     C18] x23: ffff800082fbd000 x22: 0000028f22c480dd x21: ffff800081b60000
 [     C18] x20: ffff800080c71058 x19: ffff800082fb8848 x18: 0000000000000018
 [     C18] x17: 0000000000000058 x16: 00000000ffffffff x15: 00000000ffff0000
 [     C18] x14: 0000000000000002 x13: 0000000000000003 x12: 000000000000ffd8
 [     C18] x11: ffff8000d43affd8 x10: 0000000000000038 x9 : 0000000000000000
 [     C18] x8 : ffff8000d43afa30 x7 : ff7fff7f7f7f7f7f x6 : 000000000000000a
 [     C18] x5 : ffff8000832772ba x4 : 0000000000000000 x3 : 0000000000000000
 [     C18] x2 : 0000000000000000 x1 : 000000000014b4c0 x0 : 000002a87573b7a0
 [     C18] Call trace:
 [     C18]  arch_counter_get_cntpct+0x14/0x18 (P)
 [     C18]  smp_call_function_single+0x2d0/0xd90
 [     C18]  event_function_call+0x80/0x2e0
 [     C18]  _perf_event_disable+0x5c/0xf0
 [     C18]  perf_ioctl+0x170/0x890
 [     C18]  __arm64_sys_ioctl+0xbb0/0xd28
 [     C18]  invoke_syscall+0x4c/0xd0
 [     C18]  do_el0_svc+0x80/0xb8
 [     C18]  el0_svc+0x30/0xf0
 [     C18]  el0t_64_sync_handler+0x70/0x100
 [     C18]  el0t_64_sync+0x17c/0x180

Thanks,
--breno

maintainer Applied
#9 lore raw
In reply to Breno Leitao
Quoted 6 lines · Breno Leitao

The arch_local_irq_*() wrappers in <asm/irqflags.h> dispatch between two underlying primitives: the __daif_* path on most systems, and the __pmr_* path on builds that use GIC PMR-based masking (Pseudo-NMI). The leaf primitives are already __always_inline, but the wrappers themselves are plain "static inline".

Quoted 7 lines

That is unsafe for noinstr callers: nothing prevents the compiler from emitting an out-of-line copy of e.g. arch_local_irq_disable(), and an out-of-line copy can be instrumented (ftrace, kcov, sanitizers), which breaks the noinstr contract on the entry/idle paths that rely on these helpers.

[...]

Applied to arm64 (for-next/fixes), thanks!

[1/1] arm64/irqflags: __always_inline the arch_local_irq_*() helpers

      https://git.kernel.org/arm64/c/caecde119e34