arm64: Make the kprobes debug exception path noinstr
Subsystems (6)
cover arm64: Make the kprobes debug exception path noinstr
From: Hongyan Xia <[email protected]>
Commit 879a6754d3d1 ("arm64: kprobes: Only handle faults originating from XOL slot") and 23f851ac0078 ("arm64: kprobes: Allow reentering kprobes while single-stepping") band-aided two corner cases in the arm64 kprobe path: Unrelated page faults inside Kprobe and nested Kprobe SS handling. Although the two patches were taken, Will asked whether we should keep patching things like this each time we find a corner case, and whether making Kprobe noinstr is the correct answer [1].
This series attempts to do exactly that: Making the Kprobe path noinstr. The kprobe handlers become noinstr end to end, and everything that is inherently instrumentable is confined to instrumentation_begin/end() windows, similar to the x86 side. After this series, the two corner-cases mentioned above are also gone, hence the revert in 7/9 and 8/9. Some of the instrumentation_begin/end() sections are suggested by AI, so they may need careful review.
Note that arm64 has no objtool to validate noinstr, so I wrote a small script finding bl/blr and validated that their calls are confined within noinstr. This is exactly what found the Clang outline compile problem that motivated patch 9/9.
One question is whether other BRK paths get the same treatment. Currently they stay instrumentable inside a big window.
[1] https://lore.kernel.org/all/alpuL10h7-OK2hFb@willie-the-truck/
Hongyan Xia (9):
arm64/entry: Bound certain debug exception paths in instrumentation
windows
arm64/entry: Make debug_exception_enter/exit() noinstr arm64/debug-monitors: Make do_el1_brk64()/do_el1_softstep() noinstr arm64/kprobes: Make the single-step machinery noinstr arm64/kprobes: Invoke pre/post handlers inside instrumentation arm64/kprobes: Make kprobe_fault_handler() noinstr arm64/kprobes: Drop the KPROBE_HIT_SS reentry special case arm64/kprobes: Drop the XOL single-step fault PC check arm64/debug: Mark debug exception helpers __always_inline
arch/arm64/include/asm/esr.h | 2 +- arch/arm64/include/asm/kprobes.h | 15 +---- arch/arm64/include/asm/percpu.h | 2 +- arch/arm64/include/asm/preempt.h | 4 +- arch/arm64/include/asm/ptrace.h | 4 +- arch/arm64/kernel/debug-monitors.c | 76 ++++++++++++----------- arch/arm64/kernel/entry-common.c | 43 +++++++++---- arch/arm64/kernel/probes/kprobes.c | 96 ++++++++++++++---------------- include/linux/kprobes.h | 8 +-- 9 files changed, 131 insertions(+), 119 deletions(-)
Signature · 1 line
2.47.3
RFC PATCH 1/9 arm64/entry: Bound certain debug exception paths in instrumentation windows
From: Hongyan Xia <[email protected]>
The el1 debug exception handlers are noinstr, but their payload calls (do_breakpoint(), do_watchpoint(), try_step_suspended_breakpoints() and the Cortex-A76 erratum handler) are ordinary instrumentable code.
Mark the boundaries explicitly with instrumentation_begin()/end(), in the same style as x86 exception entries.
The BRK handlers (do_el1_brk64(), do_el1_softstep()) will be dealt with in later patches.
--- arch/arm64/kernel/entry-common.c | 19 +++++++++++++++++-- 1 file changed, 17 insertions(+), 2 deletions(-)
arch/arm64/kernel/entry-common.c +17 -2
@@ -6,6 +6,7 @@*/ #include <linux/context_tracking.h> +#include <linux/instrumentation.h> #include <linux/irq-entry-common.h> #include <linux/kasan.h> #include <linux/linkage.h>
@@ -380,7 +381,9 @@ static void noinstr el1_breakpt(struct pt_regs *regs, unsigned long esr) state =arm64_enter_el1_dbg (regs);debug_exception_enter (regs); +instrumentation_begin ();do_breakpoint (esr, regs); +instrumentation_end ();debug_exception_exit (regs);arm64_exit_el1_dbg (regs, state); }
@@ -388,9 +391,15 @@ static void noinstr el1_breakpt(struct pt_regs *regs, unsigned long esr) static void noinstr el1_softstp(struct pt_regs *regs, unsigned long esr) {irqentry_state_t state; +bool handled; state =arm64_enter_el1_dbg (regs); -if (!cortex_a76_erratum_1463225_debug_handler (regs)) { + +instrumentation_begin (); + handled =cortex_a76_erratum_1463225_debug_handler (regs); +instrumentation_end (); + +if (!handled) {debug_exception_enter (regs);/* * After handling a breakpoint, we suspend the breakpoint
@@ -398,7 +407,11 @@ static void noinstr el1_softstp(struct pt_regs *regs, unsigned long esr)* If we are stepping a suspended breakpoint there's nothing more to do: * the single-step is complete. */ -if (!try_step_suspended_breakpoints (regs)) +instrumentation_begin (); + handled =try_step_suspended_breakpoints (regs); +instrumentation_end (); + +if (!handled)do_el1_softstep (esr, regs);debug_exception_exit (regs); }
@@ -413,7 +426,9 @@ static void noinstr el1_watchpt(struct pt_regs *regs, unsigned long esr) state =arm64_enter_el1_dbg (regs);debug_exception_enter (regs); +instrumentation_begin ();do_watchpoint (far, esr, regs); +instrumentation_end ();debug_exception_exit (regs);arm64_exit_el1_dbg (regs, state); }
Signature · 1 line
2.47.3
Mark Rutland
reviewerRe: [RFC PATCH 1/9] arm64/entry: Bound certain debug exception paths in instrumentation windows
Quoted 6 lines · Hongyan Xia
From: Hongyan Xia <[email protected]>
The el1 debug exception handlers are noinstr, but their payload calls (
do_breakpoint(),do_watchpoint(),try_step_suspended_breakpoints()and the Cortex-A76 erratum handler) are ordinary instrumentable code.
Quoted 7 lines
Mark the boundaries explicitly with instrumentation_begin()/end(), in the same style as x86 exception entries.
The BRK handlers (
do_el1_brk64(),do_el1_softstep()) will be dealt with in later patches.Signed-off-by Hongyan Xia <[email protected]>
Please don't do this.
The instrumentation_{begin,end}() markers don't do anything useful on arm64 (since we don't use objtoo, etc), and adding them to a subset of arm64's exception entry code gives the misleading impression that these cases are different from other exception entry cases.
Mark.
Quoted 69 lines
--- arch/arm64/kernel/entry-common.c | 19 +++++++++++++++++-- 1 file changed, 17 insertions(+), 2 deletions(-)arch/arm64/kernel/entry-common.c +17
-2@@ -6,6 +6,7 @@*/ #include <linux/context_tracking.h> +#include <linux/instrumentation.h> #include <linux/irq-entry-common.h> #include <linux/kasan.h> #include <linux/linkage.h> @@ -380,7 +381,9 @@ static void noinstr el1_breakpt(struct pt_regs *regs, unsigned long esr) state =arm64_enter_el1_dbg (regs);debug_exception_enter (regs); +instrumentation_begin ();do_breakpoint (esr, regs); +instrumentation_end ();debug_exception_exit (regs);arm64_exit_el1_dbg (regs, state); }@@ -388,9 +391,15 @@ static void noinstr el1_breakpt(struct pt_regs *regs, unsigned long esr) static void noinstr el1_softstp(struct pt_regs *regs, unsigned long esr) {irqentry_state_t state; +bool handled; state =arm64_enter_el1_dbg (regs); -if (!cortex_a76_erratum_1463225_debug_handler (regs)) { + +instrumentation_begin (); + handled =cortex_a76_erratum_1463225_debug_handler (regs); +instrumentation_end (); + +if (!handled) {debug_exception_enter (regs);/* * After handling a breakpoint, we suspend the breakpoint @@ -398,7 +407,11 @@ static void noinstr el1_softstp(struct pt_regs *regs, unsigned long esr)* If we are stepping a suspended breakpoint there's nothing more to do: * the single-step is complete. */ -if (!try_step_suspended_breakpoints (regs)) +instrumentation_begin (); + handled =try_step_suspended_breakpoints (regs); +instrumentation_end (); + +if (!handled)do_el1_softstep (esr, regs);debug_exception_exit (regs); }@@ -413,7 +426,9 @@ static void noinstr el1_watchpt(struct pt_regs *regs, unsigned long esr) state =arm64_enter_el1_dbg (regs);debug_exception_enter (regs); +instrumentation_begin ();do_watchpoint (far, esr, regs); +instrumentation_end ();debug_exception_exit (regs);arm64_exit_el1_dbg (regs, state); }Signature · 1 line
2.47.3
Re: [RFC PATCH 1/9] arm64/entry: Bound certain debug exception paths in instrumentation windows
Quoted 13 lines · Mark Rutland
Quoted 12 lines · Hongyan Xia
From: Hongyan Xia <[email protected]>
The el1 debug exception handlers are noinstr, but their payload calls (
do_breakpoint(),do_watchpoint(),try_step_suspended_breakpoints()and the Cortex-A76 erratum handler) are ordinary instrumentable code.Mark the boundaries explicitly with instrumentation_begin()/end(), in the same style as x86 exception entries.
The BRK handlers (
do_el1_brk64(),do_el1_softstep()) will be dealt with in later patches.
Quoted 8 lines
Quoted 1 line
Signed-off-by Hongyan Xia <[email protected]>Please don't do this.
The instrumentation_{begin,end}() markers don't do anything useful on arm64 (since we don't use objtoo, etc), and adding them to a subset of arm64's exception entry code gives the misleading impression that these cases are different from other exception entry cases.
Okay. I think it's a bit unclear what the scope of this series is. This series started out trying to make the whole debug_exception() path noinstr, not just Kprobe. With my recent involvement in Kprobe, the Kprobe part is handled carefully while other paths are hit with a giant instructiontation_{begin,end}() hammer.
I can see this is confusing. I'll limit the series to just Kprobe noinstr and make the commit messages clear. This should also drop a few patches in this series.
Quoted 72 lines
Mark.
Quoted 69 lines
--- arch/arm64/kernel/entry-common.c | 19 +++++++++++++++++-- 1 file changed, 17 insertions(+), 2 deletions(-)arch/arm64/kernel/entry-common.c +17
-3@@ -6,6 +6,7 @@*/ #include <linux/context_tracking.h> +#include <linux/instrumentation.h> #include <linux/irq-entry-common.h> #include <linux/kasan.h> #include <linux/linkage.h> @@ -380,7 +381,9 @@ static void noinstr el1_breakpt(struct pt_regs *regs, unsigned long esr) state =arm64_enter_el1_dbg (regs);debug_exception_enter (regs); +instrumentation_begin ();do_breakpoint (esr, regs); +instrumentation_end ();debug_exception_exit (regs);arm64_exit_el1_dbg (regs, state); }@@ -388,9 +391,15 @@ static void noinstr el1_breakpt(struct pt_regs *regs, unsigned long esr) static void noinstr el1_softstp(struct pt_regs *regs, unsigned long esr) {irqentry_state_t state; +bool handled; state =arm64_enter_el1_dbg (regs); -if (!cortex_a76_erratum_1463225_debug_handler (regs)) { + +instrumentation_begin (); + handled =cortex_a76_erratum_1463225_debug_handler (regs); +instrumentation_end (); + +if (!handled) {debug_exception_enter (regs);/* * After handling a breakpoint, we suspend the breakpoint @@ -398,7 +407,11 @@ static void noinstr el1_softstp(struct pt_regs *regs, unsigned long esr)* If we are stepping a suspended breakpoint there's nothing more to do: * the single-step is complete. */ -if (!try_step_suspended_breakpoints (regs)) +instrumentation_begin (); + handled =try_step_suspended_breakpoints (regs); +instrumentation_end (); + +if (!handled)do_el1_softstep (esr, regs);debug_exception_exit (regs); }@@ -413,7 +426,9 @@ static void noinstr el1_watchpt(struct pt_regs *regs, unsigned long esr) state =arm64_enter_el1_dbg (regs);debug_exception_enter (regs); +instrumentation_begin ();do_watchpoint (far, esr, regs); +instrumentation_end ();debug_exception_exit (regs);arm64_exit_el1_dbg (regs, state); } --2.47.3
RFC PATCH 2/9 arm64/entry: Make debug_exception_enter/exit() noinstr
In reply to Hongyan XiaFrom: Hongyan Xia <[email protected]>
Commit 879a6754d3d11e30af24b7dc486f561510d62641 ran into a crash because debug_exception_enter/exit() triggered page faults caused by perf dwarf call graph tracing. That patch was a band-aid on top. Instead of trying to band-aid all possible paths that can happen during instrumentation or perf tracing, simply make both functions noinstr.
Drop the RCU_LOCKDEP_WARN(): arm64_enter_el1_dbg() runs first and enters NMI context via ct_nmi_enter(), so RCU is always watching by the time debug_exception_enter() runs.
--- arch/arm64/kernel/entry-common.c | 24 +++++++++++++++--------- 1 file changed, 15 insertions(+), 9 deletions(-)
arch/arm64/kernel/entry-common.c +15 -9
@@ -293,20 +293,26 @@ static __always_inline void fpsimd_syscall_exit(void)* accidentally schedule in exception context and it will force a warning * if we somehow manage to schedule by accident. */ -static void debug_exception_enter (struct pt_regs *regs) +static void noinstr debug_exception_enter(struct pt_regs *regs) { -preempt_disable (); - -/* This code is a bit fragile. Test it. */ -RCU_LOCKDEP_WARN (!rcu_is_watching (),"exception_enter didn't work" ); +/* +* debug_exception_enter/exit() can be quite delicate. The normal +* preempt_disable/enable() can be instrumented or traced by perf, +* which leads to a can of worms including triggering page faults. +* +* Instead of trying to make all of them work properly here, just +* open-code the simpler version of preempt_disable/enable() and make +* enter/exit() both noinstr to avoid the entire complexity. +*/ +__preempt_count_inc (); +barrier (); } -NOKPROBE_SYMBOL (debug_exception_enter); -static void debug_exception_exit (struct pt_regs *regs) +static void noinstr debug_exception_exit(struct pt_regs *regs) { -preempt_enable_no_resched (); +barrier (); +__preempt_count_dec (); } -NOKPROBE_SYMBOL (debug_exception_exit); UNHANDLED(el1t, 64, sync)UNHANDLED (el1t,64 , irq)
Signature · 1 line
2.47.3
Mark Rutland
reviewerRe: [RFC PATCH 2/9] arm64/entry: Make debug_exception_enter/exit() noinstr
Quoted 35 lines · Hongyan Xia
From: Hongyan Xia <[email protected]>
Commit
879a6754d3d11e30af24b7dc486f561510d62641ran into a crash because debug_exception_enter/exit() triggered page faults caused by perf dwarf call graph tracing. That patch was a band-aid on top. Instead of trying to band-aid all possible paths that can happen during instrumentation or perf tracing, simply make both functions noinstr.Drop the
RCU_LOCKDEP_WARN():arm64_enter_el1_dbg()runs first and enters NMI context viact_nmi_enter(), so RCU is always watching by the timedebug_exception_enter()runs.Signed-off-by Hongyan Xia <[email protected]>--- arch/arm64/kernel/entry-common.c | 24 +++++++++++++++--------- 1 file changed, 15 insertions(+), 9 deletions(-)arch/arm64/kernel/entry-common.c +4
-5@@ -293,20 +293,26 @@ static __always_inline void fpsimd_syscall_exit(void)* accidentally schedule in exception context and it will force a warning * if we somehow manage to schedule by accident. */ -static void debug_exception_enter (struct pt_regs *regs) +static void noinstr debug_exception_enter(struct pt_regs *regs) { -preempt_disable (); - -/* This code is a bit fragile. Test it. */ -RCU_LOCKDEP_WARN (!rcu_is_watching (),"exception_enter didn't work" ); +/* +* debug_exception_enter/exit() can be quite delicate. The normal +* preempt_disable/enable() can be instrumented or traced by perf,
arch/arm64/kernel/entry-common.c static __always_inline void fpsimd_syscall_exit(void) · line 303
Quoted 10 lines
arch/arm64/kernel/entry-common.c +8
-1+* which leads to a can of worms including triggering page faults. +* +* Instead of trying to make all of them work properly here, just +* open-code the simpler version of preempt_disable/enable() and make +* enter/exit() both noinstr to avoid the entire complexity. +*/ +__preempt_count_inc (); +barrier (); } -NOKPROBE_SYMBOL (debug_exception_enter);
This is an open-coded version of preempt_disable_notrace(), and there's no reason the existing code needs to be out-of-line in the first place. Please make this:
static __always_inline void debug_exception_enter(struct pt_regs *regs)
{
preempt_disable_notrace();
}arch/arm64/kernel/entry-common.c static __always_inline void fpsimd_syscall_exit(void) · line 309
Quoted 8 lines
arch/arm64/kernel/entry-common.c +3
-3-static void debug_exception_exit (struct pt_regs *regs) +static void noinstr debug_exception_exit(struct pt_regs *regs) { -preempt_enable_no_resched (); +barrier (); +__preempt_count_dec (); } -NOKPROBE_SYMBOL (debug_exception_exit);
Similarly, this is an open-coded copy of preempt_enable_no_resched_notrace(). Please make this:
static __always_inline void debug_exception_exit(struct pt_regs *regs)
{
preempt_enable_no_resched_notrace();
}In both cases I'm happy with keeping the regs argument for now.
Mark.
RFC PATCH 3/9 arm64/debug-monitors: Make do_el1_brk64()/do_el1_softstep() noinstr
In reply to Hongyan XiaFrom: Hongyan Xia <[email protected]>
Convert do_el1_brk64(), do_el1_softstep() and call_el1_break_hook() to noinstr. The kprobe and kretprobe BRK handlers (converted to noinstr in the following patches) are dispatched directly. Every other BRK handler are ordinary instrumentable code and now run bounded by instrumentation_begin()/end().
With this, everything on the el1 debug exception path from the vectors down to the kprobe handlers is noinstr, and instrumentation only runs inside explicit instrumentation windows.
--- arch/arm64/kernel/debug-monitors.c | 74 +++++++++++++++++------------- 1 file changed, 41 insertions(+), 33 deletions(-)
arch/arm64/kernel/debug-monitors.c +41 -33
@@ -11,6 +11,7 @@#include <linux/debugfs.h> #include <linux/hardirq.h> #include <linux/init.h> +#include <linux/instrumentation.h> #include <linux/ptrace.h> #include <linux/kprobes.h> #include <linux/stat.h>
@@ -193,59 +194,65 @@ void do_el0_softstep(unsigned long esr, struct pt_regs *regs)user_rewind_single_step (current); } -void do_el1_softstep (unsigned long esr,struct pt_regs *regs) +void noinstr do_el1_softstep(unsigned long esr, struct pt_regs *regs) { -if (kgdb_single_step_handler (regs, esr) ==DBG_HOOK_HANDLED ) +int handled; + +instrumentation_begin (); + handled =kgdb_single_step_handler (regs, esr); +instrumentation_end (); + +if (handled ==DBG_HOOK_HANDLED )return ; +instrumentation_begin ();pr_warn ("Unexpected kernel single-step exception at EL1\n" ); +instrumentation_end ();/* * Re-enable stepping since we know that we will be * returning to regs. */ set_regs_spsr_ss (regs); } -NOKPROBE_SYMBOL (do_el1_softstep); -static int call_el1_break_hook (struct pt_regs *regs,unsigned long esr) +static int noinstr call_el1_break_hook(struct pt_regs *regs, unsigned long esr) { -if (esr_brk_comment (esr) ==BUG_BRK_IMM ) -return bug_brk_handler (regs, esr); - -if (IS_ENABLED (CONFIG_CFI ) &&esr_is_cfi_brk (esr)) -return cfi_brk_handler (regs, esr); - -if (esr_brk_comment (esr) ==FAULT_BRK_IMM ) -return reserved_fault_brk_handler (regs, esr); - -if (IS_ENABLED (CONFIG_KASAN_SW_TAGS ) && - (esr_brk_comment (esr) & ~KASAN_BRK_MASK ) ==KASAN_BRK_IMM ) -return kasan_brk_handler (regs, esr); - -if (IS_ENABLED (CONFIG_UBSAN_TRAP ) &&esr_is_ubsan_brk (esr)) -return ubsan_brk_handler (regs, esr); - -if (IS_ENABLED (CONFIG_KGDB )) { -if (esr_brk_comment (esr) ==KGDB_DYN_DBG_BRK_IMM ) -return kgdb_brk_handler (regs, esr); -if (esr_brk_comment (esr) ==KGDB_COMPILED_DBG_BRK_IMM ) -return kgdb_compiled_brk_handler (regs, esr); - } +unsigned long comment =esr_brk_comment (esr); +int ret =DBG_HOOK_ERROR ;if (IS_ENABLED (CONFIG_KPROBES )) { -if (esr_brk_comment (esr) ==KPROBES_BRK_IMM ) +if (comment ==KPROBES_BRK_IMM )return kprobe_brk_handler (regs, esr); -if (esr_brk_comment (esr) ==KPROBES_BRK_SS_IMM ) +if (comment ==KPROBES_BRK_SS_IMM )return kprobe_ss_brk_handler (regs, esr); }if (IS_ENABLED (CONFIG_KRETPROBES ) && -esr_brk_comment (esr) ==KRETPROBES_BRK_IMM ) + comment ==KRETPROBES_BRK_IMM )return kretprobe_brk_handler (regs, esr); -return DBG_HOOK_ERROR ; +instrumentation_begin (); +if (comment ==BUG_BRK_IMM ) + ret =bug_brk_handler (regs, esr); +else if (IS_ENABLED (CONFIG_CFI ) &&esr_is_cfi_brk (esr)) + ret =cfi_brk_handler (regs, esr); +else if (comment ==FAULT_BRK_IMM ) + ret =reserved_fault_brk_handler (regs, esr); +else if (IS_ENABLED (CONFIG_KASAN_SW_TAGS ) && + (comment & ~KASAN_BRK_MASK ) ==KASAN_BRK_IMM ) + ret =kasan_brk_handler (regs, esr); +else if (IS_ENABLED (CONFIG_UBSAN_TRAP ) &&esr_is_ubsan_brk (esr)) + ret =ubsan_brk_handler (regs, esr); +else if (IS_ENABLED (CONFIG_KGDB )) { +if (comment ==KGDB_DYN_DBG_BRK_IMM ) + ret =kgdb_brk_handler (regs, esr); +else if (comment ==KGDB_COMPILED_DBG_BRK_IMM ) + ret =kgdb_compiled_brk_handler (regs, esr); + } +instrumentation_end (); + +return ret; } -NOKPROBE_SYMBOL (call_el1_break_hook);/* * We have already unmasked interrupts and enabled preemption
@@ -261,14 +268,15 @@ void do_el0_brk64(unsigned long esr, struct pt_regs *regs)send_user_sigtrap (TRAP_BRKPT ); } -void do_el1_brk64 (unsigned long esr,struct pt_regs *regs) +void noinstr do_el1_brk64(unsigned long esr, struct pt_regs *regs) {if (call_el1_break_hook (regs, esr) ==DBG_HOOK_HANDLED )return ; +instrumentation_begin ();die ("Oops - BRK" , regs, esr); +instrumentation_end (); } -NOKPROBE_SYMBOL (do_el1_brk64);#ifdef CONFIG_COMPAT void do_bkpt32 (unsigned long esr,struct pt_regs *regs)
Signature · 1 line
2.47.3
Mark Rutland
reviewerRe: [RFC PATCH 3/9] arm64/debug-monitors: Make do_el1_brk64()/do_el1_softstep() noinstr
Quoted 7 lines · Hongyan Xia
From: Hongyan Xia <[email protected]>
Convert
do_el1_brk64(),do_el1_softstep()andcall_el1_break_hook()to noinstr. The kprobe and kretprobe BRK handlers (converted to noinstr in the following patches) are dispatched directly. Every other BRK handler are ordinary instrumentable code and now run bounded by instrumentation_begin()/end().
Why is it necessary to change do_el1_softstep()?
Neither kprobes nor kretprobes uses software stepping since commit:
7ee31a3aa8f4 ("arm64: kprobes: Use BRK instead of single-step when executing instructions out-of-line")
... so either that shouldn't be necessary, or there's a problem that needs to be described in this commit message.
Quoted 38 lines
With this, everything on the el1 debug exception path from the vectors down to the kprobe handlers is noinstr, and instrumentation only runs inside explicit instrumentation windows.
Signed-off-by Hongyan Xia <[email protected]>--- arch/arm64/kernel/debug-monitors.c | 74 +++++++++++++++++------------- 1 file changed, 41 insertions(+), 33 deletions(-)arch/arm64/kernel/debug-monitors.c +9
-2@@ -11,6 +11,7 @@#include <linux/debugfs.h> #include <linux/hardirq.h> #include <linux/init.h> +#include <linux/instrumentation.h> #include <linux/ptrace.h> #include <linux/kprobes.h> #include <linux/stat.h> @@ -193,59 +194,65 @@ void do_el0_softstep(unsigned long esr, struct pt_regs *regs)user_rewind_single_step (current); } -void do_el1_softstep (unsigned long esr,struct pt_regs *regs) +void noinstr do_el1_softstep(unsigned long esr, struct pt_regs *regs) { -if (kgdb_single_step_handler (regs, esr) ==DBG_HOOK_HANDLED ) +int handled; + +instrumentation_begin (); + handled =kgdb_single_step_handler (regs, esr); +instrumentation_end (); + +if (handled ==DBG_HOOK_HANDLED )return ;
arch/arm64/kernel/debug-monitors.c void do_el0_softstep(unsigned long esr, struct pt_regs *regs) · line 208
Quoted 10 lines
arch/arm64/kernel/debug-monitors.c +2
-1+instrumentation_begin ();pr_warn ("Unexpected kernel single-step exception at EL1\n" ); +instrumentation_end ();/* * Re-enable stepping since we know that we will be * returning to regs. */ set_regs_spsr_ss (regs); } -NOKPROBE_SYMBOL (do_el1_softstep);
As above, I don't think it's necessary to change do_el1_softstep(), but I might be missing something that you haven't described in the commit message.
Is the existing NOKPROBE_SYMBOL() annotation actually necessary? It looks like that dates from before commit 7ee31a3aa8f4, and I suspect we can delete it even without making this noinstr.
I don't think you need to make structural changes here. Given the first thing the function does is an unconditional call to an instrumented function, we're not gaining anything by litering this with instrumentation_{begin,end}().
Quoted 56 lines
arch/arm64/kernel/debug-monitors.c +19
-27-static int call_el1_break_hook (struct pt_regs *regs,unsigned long esr) +static int noinstr call_el1_break_hook(struct pt_regs *regs, unsigned long esr) { -if (esr_brk_comment (esr) ==BUG_BRK_IMM ) -return bug_brk_handler (regs, esr); - -if (IS_ENABLED (CONFIG_CFI ) &&esr_is_cfi_brk (esr)) -return cfi_brk_handler (regs, esr); - -if (esr_brk_comment (esr) ==FAULT_BRK_IMM ) -return reserved_fault_brk_handler (regs, esr); - -if (IS_ENABLED (CONFIG_KASAN_SW_TAGS ) && - (esr_brk_comment (esr) & ~KASAN_BRK_MASK ) ==KASAN_BRK_IMM ) -return kasan_brk_handler (regs, esr); - -if (IS_ENABLED (CONFIG_UBSAN_TRAP ) &&esr_is_ubsan_brk (esr)) -return ubsan_brk_handler (regs, esr); - -if (IS_ENABLED (CONFIG_KGDB )) { -if (esr_brk_comment (esr) ==KGDB_DYN_DBG_BRK_IMM ) -return kgdb_brk_handler (regs, esr); -if (esr_brk_comment (esr) ==KGDB_COMPILED_DBG_BRK_IMM ) -return kgdb_compiled_brk_handler (regs, esr); - } +unsigned long comment =esr_brk_comment (esr); +int ret =DBG_HOOK_ERROR ;if (IS_ENABLED (CONFIG_KPROBES )) { -if (esr_brk_comment (esr) ==KPROBES_BRK_IMM ) +if (comment ==KPROBES_BRK_IMM )return kprobe_brk_handler (regs, esr); -if (esr_brk_comment (esr) ==KPROBES_BRK_SS_IMM ) +if (comment ==KPROBES_BRK_SS_IMM )return kprobe_ss_brk_handler (regs, esr); }if (IS_ENABLED (CONFIG_KRETPROBES ) && -esr_brk_comment (esr) ==KRETPROBES_BRK_IMM ) + comment ==KRETPROBES_BRK_IMM )return kretprobe_brk_handler (regs, esr); -return DBG_HOOK_ERROR ; +instrumentation_begin (); +if (comment ==BUG_BRK_IMM ) + ret =bug_brk_handler (regs, esr); +else if (IS_ENABLED (CONFIG_CFI ) &&esr_is_cfi_brk (esr)) + ret =cfi_brk_handler (regs, esr); +else if (comment ==FAULT_BRK_IMM ) + ret =reserved_fault_brk_handler (regs, esr); +else if (IS_ENABLED (CONFIG_KASAN_SW_TAGS ) && + (comment & ~KASAN_BRK_MASK ) ==KASAN_BRK_IMM ) + ret =kasan_brk_handler (regs, esr); +else if (IS_ENABLED (CONFIG_UBSAN_TRAP ) &&esr_is_ubsan_brk (esr)) + ret =ubsan_brk_handler (regs, esr); +else if (IS_ENABLED (CONFIG_KGDB )) {
arch/arm64/kernel/debug-monitors.c void do_el0_softstep(unsigned long esr, struct pt_regs *regs) · line 248
Quoted 10 lines
arch/arm64/kernel/debug-monitors.c +8
-1+if (comment ==KGDB_DYN_DBG_BRK_IMM ) + ret =kgdb_brk_handler (regs, esr); +else if (comment ==KGDB_COMPILED_DBG_BRK_IMM ) + ret =kgdb_compiled_brk_handler (regs, esr); + } +instrumentation_end (); + +return ret; } -NOKPROBE_SYMBOL (call_el1_break_hook);
I don't think you need to make any structural changes to call_el1_break_hook(). Just mark it as noinstr, and remove the NOKPROBE_SYMBOL() annotation. The existing control flow will be safe.
Quoted 7 lines
arch/arm64/kernel/debug-monitors.c +0
-1/* * We have already unmasked interrupts and enabled preemption @@ -261,14 +268,15 @@ void do_el0_brk64(unsigned long esr, struct pt_regs *regs)send_user_sigtrap (TRAP_BRKPT ); } -void do_el1_brk64 (unsigned long esr,struct pt_regs *regs)
arch/arm64/kernel/debug-monitors.c void do_el0_brk64(unsigned long esr, struct pt_regs *regs) · line 271
Quoted 10 lines
arch/arm64/kernel/debug-monitors.c +3
-1+void noinstr do_el1_brk64(unsigned long esr, struct pt_regs *regs) {if (call_el1_break_hook (regs, esr) ==DBG_HOOK_HANDLED )return ; +instrumentation_begin ();die ("Oops - BRK" , regs, esr); +instrumentation_end (); } -NOKPROBE_SYMBOL (do_el1_brk64);
Likewise, just mark do_el1_brk64() as noinstr and remove the NOKPROBE_SYMBOL() annotation, without the instrumentation_{begin,end}() calls.
Mark.
Re: [RFC PATCH 3/9] arm64/debug-monitors: Make do_el1_brk64()/do_el1_softstep() noinstr
Quoted 58 lines · Mark Rutland
[Some people who received this message don't often get email from [email protected]. Learn why this is important at https://aka.ms/LearnAboutSenderIdentification ]
Quoted 7 lines · Hongyan Xia
From: Hongyan Xia <[email protected]>
Convert
do_el1_brk64(),do_el1_softstep()andcall_el1_break_hook()to noinstr. The kprobe and kretprobe BRK handlers (converted to noinstr in the following patches) are dispatched directly. Every other BRK handler are ordinary instrumentable code and now run bounded by instrumentation_begin()/end().Why is it necessary to change
do_el1_softstep()?Neither kprobes nor kretprobes uses software stepping since commit:
7ee31a3aa8f4("arm64: kprobes: Use BRK instead of single-step when executing instructions out-of-line")... so either that shouldn't be necessary, or there's a problem that needs to be described in this commit message.
Quoted 38 lines
With this, everything on the el1 debug exception path from the vectors down to the kprobe handlers is noinstr, and instrumentation only runs inside explicit instrumentation windows.
Signed-off-by Hongyan Xia <[email protected]>--- arch/arm64/kernel/debug-monitors.c | 74 +++++++++++++++++------------- 1 file changed, 41 insertions(+), 33 deletions(-)arch/arm64/kernel/debug-monitors.c +9
-2@@ -11,6 +11,7 @@#include <linux/debugfs.h> #include <linux/hardirq.h> #include <linux/init.h> +#include <linux/instrumentation.h> #include <linux/ptrace.h> #include <linux/kprobes.h> #include <linux/stat.h> @@ -193,59 +194,65 @@ void do_el0_softstep(unsigned long esr, struct pt_regs *regs)user_rewind_single_step (current); } -void do_el1_softstep (unsigned long esr,struct pt_regs *regs) +void noinstr do_el1_softstep(unsigned long esr, struct pt_regs *regs) { -if (kgdb_single_step_handler (regs, esr) ==DBG_HOOK_HANDLED ) +int handled; + +instrumentation_begin (); + handled =kgdb_single_step_handler (regs, esr); +instrumentation_end (); + +if (handled ==DBG_HOOK_HANDLED )return ;
Quoted 14 lines
arch/arm64/kernel/debug-monitors.c void do_el0_softstep(unsigned long esr, struct pt_regs *regs) · line 208
Quoted 10 lines
arch/arm64/kernel/debug-monitors.c +2
-1+instrumentation_begin ();pr_warn ("Unexpected kernel single-step exception at EL1\n" ); +instrumentation_end ();/* * Re-enable stepping since we know that we will be * returning to regs. */ set_regs_spsr_ss (regs); } -NOKPROBE_SYMBOL (do_el1_softstep);As above, I don't think it's necessary to change
do_el1_softstep(), but I might be missing something that you haven't described in the commit message.
We have a noinstr verifier locally. To make it happy I started out this series trying to make the whole debug_exception() noinstr, but I see it has caused enough confusion (like in 1/9) and I'll limit the scope to only Kprobe noinstr in the next revision.
Quoted 92 lines
Is the existing
NOKPROBE_SYMBOL()annotation actually necessary? It looks like that dates from before commit7ee31a3aa8f4, and I suspect we can delete it even without making this noinstr.I don't think you need to make structural changes here. Given the first thing the function does is an unconditional call to an instrumented function, we're not gaining anything by litering this with instrumentation_{begin,end}().
Quoted 56 lines
arch/arm64/kernel/debug-monitors.c +19
-27-static int call_el1_break_hook (struct pt_regs *regs,unsigned long esr) +static int noinstr call_el1_break_hook(struct pt_regs *regs, unsigned long esr) { -if (esr_brk_comment (esr) ==BUG_BRK_IMM ) -return bug_brk_handler (regs, esr); - -if (IS_ENABLED (CONFIG_CFI ) &&esr_is_cfi_brk (esr)) -return cfi_brk_handler (regs, esr); - -if (esr_brk_comment (esr) ==FAULT_BRK_IMM ) -return reserved_fault_brk_handler (regs, esr); - -if (IS_ENABLED (CONFIG_KASAN_SW_TAGS ) && - (esr_brk_comment (esr) & ~KASAN_BRK_MASK ) ==KASAN_BRK_IMM ) -return kasan_brk_handler (regs, esr); - -if (IS_ENABLED (CONFIG_UBSAN_TRAP ) &&esr_is_ubsan_brk (esr)) -return ubsan_brk_handler (regs, esr); - -if (IS_ENABLED (CONFIG_KGDB )) { -if (esr_brk_comment (esr) ==KGDB_DYN_DBG_BRK_IMM ) -return kgdb_brk_handler (regs, esr); -if (esr_brk_comment (esr) ==KGDB_COMPILED_DBG_BRK_IMM ) -return kgdb_compiled_brk_handler (regs, esr); - } +unsigned long comment =esr_brk_comment (esr); +int ret =DBG_HOOK_ERROR ;if (IS_ENABLED (CONFIG_KPROBES )) { -if (esr_brk_comment (esr) ==KPROBES_BRK_IMM ) +if (comment ==KPROBES_BRK_IMM )return kprobe_brk_handler (regs, esr); -if (esr_brk_comment (esr) ==KPROBES_BRK_SS_IMM ) +if (comment ==KPROBES_BRK_SS_IMM )return kprobe_ss_brk_handler (regs, esr); }if (IS_ENABLED (CONFIG_KRETPROBES ) && -esr_brk_comment (esr) ==KRETPROBES_BRK_IMM ) + comment ==KRETPROBES_BRK_IMM )return kretprobe_brk_handler (regs, esr); -return DBG_HOOK_ERROR ; +instrumentation_begin (); +if (comment ==BUG_BRK_IMM ) + ret =bug_brk_handler (regs, esr); +else if (IS_ENABLED (CONFIG_CFI ) &&esr_is_cfi_brk (esr)) + ret =cfi_brk_handler (regs, esr); +else if (comment ==FAULT_BRK_IMM ) + ret =reserved_fault_brk_handler (regs, esr); +else if (IS_ENABLED (CONFIG_KASAN_SW_TAGS ) && + (comment & ~KASAN_BRK_MASK ) ==KASAN_BRK_IMM ) + ret =kasan_brk_handler (regs, esr); +else if (IS_ENABLED (CONFIG_UBSAN_TRAP ) &&esr_is_ubsan_brk (esr)) + ret =ubsan_brk_handler (regs, esr); +else if (IS_ENABLED (CONFIG_KGDB )) {arch/arm64/kernel/debug-monitors.c void do_el0_softstep(unsigned long esr, struct pt_regs *regs) · line 248
Quoted 10 lines
arch/arm64/kernel/debug-monitors.c +8
-1+if (comment ==KGDB_DYN_DBG_BRK_IMM ) + ret =kgdb_brk_handler (regs, esr); +else if (comment ==KGDB_COMPILED_DBG_BRK_IMM ) + ret =kgdb_compiled_brk_handler (regs, esr); + } +instrumentation_end (); + +return ret; } -NOKPROBE_SYMBOL (call_el1_break_hook);I don't think you need to make any structural changes to
call_el1_break_hook(). Just mark it as noinstr, and remove theNOKPROBE_SYMBOL()annotation. The existing control flow will be safe.Quoted 12 lines
arch/arm64/kernel/debug-monitors.c +1
-1/* * We have already unmasked interrupts and enabled preemption @@ -261,14 +268,15 @@ void do_el0_brk64(unsigned long esr, struct pt_regs *regs)send_user_sigtrap (TRAP_BRKPT ); } -void do_el1_brk64 (unsigned long esr,struct pt_regs *regs) +void noinstr do_el1_brk64(unsigned long esr, struct pt_regs *regs) {if (call_el1_break_hook (regs, esr) ==DBG_HOOK_HANDLED )return ;
Quoted 9 lines
arch/arm64/kernel/debug-monitors.c void do_el0_brk64(unsigned long esr, struct pt_regs *regs) · line 271
Quoted 5 lines
arch/arm64/kernel/debug-monitors.c +2
-1+instrumentation_begin ();die ("Oops - BRK" , regs, esr); +instrumentation_end (); } -NOKPROBE_SYMBOL (do_el1_brk64);Likewise, just mark
do_el1_brk64()as noinstr and remove theNOKPROBE_SYMBOL()annotation, without the instrumentation_{begin,end}() calls.
I'll just drop this patch and get Kprobe properly done first.
Quoted 2 lines
Mark.
RFC PATCH 4/9 arm64/kprobes: Make the single-step machinery noinstr
In reply to Hongyan XiaFrom: Hongyan Xia <[email protected]>
Convert the core of the arm64 kprobe single-step flow to noinstr:save_previous_kprobe(), restore_previous_kprobe(), set_current_kprobe(), kprobes_save_local_irqflag(), kprobes_restore_local_irqflag(), setup_singlestep(), reenter_kprobe() and post_kprobe_handler().
These functions only touch per-cpu state, pt_regs and DAIF, except for:
- kprobes_inc_nmissed_count(), which is generic and instrumentable; add an arm64-local wrapper that calls it bounded by instrumentation_begin()/end();
- the instruction simulation path in setup_singlestep(), whose decode handlers are instrumentable; bound arch_simulate_insn() with an instrumentation window;
- the unrecoverable-reentry pr_warn()/dump_kprobe()/BUG() path and the default WARN_ON(), both bounded with instrumentation windows.
The __kprobes attribute (notrace + .kprobes.text) is replaced by noinstr, which is a strict superset for these functions.
--- arch/arm64/kernel/probes/kprobes.c | 39 ++++++++++++++++++------------ 1 file changed, 24 insertions(+), 15 deletions(-)
arch/arm64/kernel/probes/kprobes.c +24 -15
@@ -14,6 +14,7 @@#include <linux/extable.h> #include <linux/kasan.h> #include <linux/kernel.h> +#include <linux/instrumentation.h> #include <linux/kprobes.h> #include <linux/sched/debug.h> #include <linux/set_memory.h>
@@ -39,7 +40,7 @@ DEFINE_PER_CPU(struct kprobe *, current_kprobe) = NULL; DEFINE_PER_CPU(struct kprobe_ctlblk, kprobe_ctlblk); -static void __kprobes +static void noinstrpost_kprobe_handler (struct kprobe *,struct kprobe_ctlblk *,struct pt_regs *);void *alloc_insn_page (void )
@@ -170,7 +171,7 @@ void __kprobes arch_remove_kprobe(struct kprobe *p) } } -static void __kprobes save_previous_kprobe (struct kprobe_ctlblk *kcb) +static void noinstr save_previous_kprobe(struct kprobe_ctlblk *kcb) { kcb->prev_kprobe.kp =kprobe_running (); kcb->prev_kprobe.status = kcb->kprobe_status;
@@ -184,7 +185,7 @@ static void __kprobes save_previous_kprobe(struct kprobe_ctlblk *kcb) kcb->prev_kprobe.saved_irqflag = kcb->saved_irqflag; } -static void __kprobes restore_previous_kprobe (struct kprobe_ctlblk *kcb) +static void noinstr restore_previous_kprobe(struct kprobe_ctlblk *kcb) {__this_cpu_write (current_kprobe, kcb->prev_kprobe.kp); kcb->kprobe_status = kcb->prev_kprobe.status;
@@ -197,7 +198,7 @@ static void __kprobes restore_previous_kprobe(struct kprobe_ctlblk *kcb) kcb->saved_irqflag = kcb->prev_kprobe.saved_irqflag; } -static void __kprobes set_current_kprobe (struct kprobe *p) +static void noinstr set_current_kprobe(struct kprobe *p) {__this_cpu_write (current_kprobe, p); }
@@ -207,23 +208,23 @@ static void __kprobes set_current_kprobe(struct kprobe *p)* simple and avoid nesting exceptions. Interrupts do have to be disabled since * the kprobe state is per-CPU and doesn't get migrated. */ -static void __kprobes kprobes_save_local_irqflag (struct kprobe_ctlblk *kcb, -struct pt_regs *regs) +static void noinstr kprobes_save_local_irqflag(struct kprobe_ctlblk *kcb, +struct pt_regs *regs) { kcb->saved_irqflag = regs->pstate &DAIF_MASK ; regs->pstate |=DAIF_MASK ; } -static void __kprobes kprobes_restore_local_irqflag (struct kprobe_ctlblk *kcb, -struct pt_regs *regs) +static void noinstr kprobes_restore_local_irqflag(struct kprobe_ctlblk *kcb, +struct pt_regs *regs) { regs->pstate &= ~DAIF_MASK ; regs->pstate |= kcb->saved_irqflag; } -static void __kprobes setup_singlestep (struct kprobe *p, -struct pt_regs *regs, -struct kprobe_ctlblk *kcb,int reenter) +static void noinstr setup_singlestep(struct kprobe *p, +struct pt_regs *regs, +struct kprobe_ctlblk *kcb,int reenter) {unsigned long slot;
@@ -244,13 +245,15 @@ static void __kprobes setup_singlestep(struct kprobe *p,instruction_pointer_set (regs, slot); }else {/* insn simulation */ +instrumentation_begin ();arch_simulate_insn (p, regs); +instrumentation_end (); } } -static int __kprobes reenter_kprobe (struct kprobe *p, -struct pt_regs *regs, -struct kprobe_ctlblk *kcb) +static int noinstr reenter_kprobe(struct kprobe *p, +struct pt_regs *regs, +struct kprobe_ctlblk *kcb) {switch (kcb->kprobe_status) {case KPROBE_HIT_SSDONE :
@@ -262,23 +265,29 @@ static int __kprobes reenter_kprobe(struct kprobe *p,* recoverable one-level reentry, so handle it in the same way as * reentry from KPROBE_HIT_ACTIVE or KPROBE_HIT_SSDONE. */ +instrumentation_begin ();kprobes_inc_nmissed_count (p); +instrumentation_end ();setup_singlestep (p, regs, kcb,1 );break ;case KPROBE_REENTER : +instrumentation_begin ();pr_warn ("Failed to recover from reentered kprobes.\n" );dump_kprobe (p);BUG (); +instrumentation_end ();break ;default : +instrumentation_begin ();WARN_ON (1 ); +instrumentation_end ();return 0 ; }return 1 ; } -static void __kprobes +static void noinstrpost_kprobe_handler (struct kprobe *cur,struct kprobe_ctlblk *kcb,struct pt_regs *regs) {/* return addr restore if non-branching insn */
Signature · 1 line
2.47.3
Mark Rutland
reviewerRe: [RFC PATCH 4/9] arm64/kprobes: Make the single-step machinery noinstr
Quoted 7 lines · Hongyan Xia
From: Hongyan Xia <[email protected]>
Convert the core of the arm64 kprobe single-step flow to noinstr:
save_previous_kprobe(),restore_previous_kprobe(),set_current_kprobe(),kprobes_save_local_irqflag(),kprobes_restore_local_irqflag(),setup_singlestep(),reenter_kprobe()andpost_kprobe_handler().
Quoted 5 lines
These functions only touch per-cpu state,
pt_regsand DAIF, except for:-
kprobes_inc_nmissed_count(), which is generic and instrumentable; add an arm64-local wrapper that calls it bounded by instrumentation_begin()/end();
Why is it safe to call kprobes_inc_nmissed_count() if it can be instrumented?
How does that work for other architectures?
Quoted 3 lines
- the instruction simulation path in
setup_singlestep(), whose decode handlers are instrumentable; boundarch_simulate_insn()with an instrumentation window;
I don't think that's sufficient. If we can instrument that code, then we can have unbounded recursion, unless I'm missing something?
I think we need to fix that code to be noinstr safe.
Ada Cc'd was looking into making the insn code generally noinstr-safe, but that's a big job. Maybe it's possible to clean up the subset that's necessary for arch_simulate_insn()?
Quoted 2 lines
- the unrecoverable-reentry pr_warn()/dump_kprobe()/BUG() path and the default
WARN_ON(), both bounded with instrumentation windows.
That sounds fine.
Mark.
Quoted 140 lines
The
__kprobesattribute (notrace + .kprobes.text) is replaced by noinstr, which is a strict superset for these functions.Signed-off-by Hongyan Xia <[email protected]>--- arch/arm64/kernel/probes/kprobes.c | 39 ++++++++++++++++++------------ 1 file changed, 24 insertions(+), 15 deletions(-)arch/arm64/kernel/probes/kprobes.c +24
-15@@ -14,6 +14,7 @@#include <linux/extable.h> #include <linux/kasan.h> #include <linux/kernel.h> +#include <linux/instrumentation.h> #include <linux/kprobes.h> #include <linux/sched/debug.h> #include <linux/set_memory.h> @@ -39,7 +40,7 @@ DEFINE_PER_CPU(struct kprobe *, current_kprobe) = NULL; DEFINE_PER_CPU(struct kprobe_ctlblk, kprobe_ctlblk); -static void __kprobes +static void noinstrpost_kprobe_handler (struct kprobe *,struct kprobe_ctlblk *,struct pt_regs *);void *alloc_insn_page (void )@@ -170,7 +171,7 @@ void __kprobes arch_remove_kprobe(struct kprobe *p) } } -static void __kprobes save_previous_kprobe (struct kprobe_ctlblk *kcb) +static void noinstr save_previous_kprobe(struct kprobe_ctlblk *kcb) { kcb->prev_kprobe.kp =kprobe_running (); kcb->prev_kprobe.status = kcb->kprobe_status;@@ -184,7 +185,7 @@ static void __kprobes save_previous_kprobe(struct kprobe_ctlblk *kcb) kcb->prev_kprobe.saved_irqflag = kcb->saved_irqflag; } -static void __kprobes restore_previous_kprobe (struct kprobe_ctlblk *kcb) +static void noinstr restore_previous_kprobe(struct kprobe_ctlblk *kcb) {__this_cpu_write (current_kprobe, kcb->prev_kprobe.kp); kcb->kprobe_status = kcb->prev_kprobe.status;@@ -197,7 +198,7 @@ static void __kprobes restore_previous_kprobe(struct kprobe_ctlblk *kcb) kcb->saved_irqflag = kcb->prev_kprobe.saved_irqflag; } -static void __kprobes set_current_kprobe (struct kprobe *p) +static void noinstr set_current_kprobe(struct kprobe *p) {__this_cpu_write (current_kprobe, p); }@@ -207,23 +208,23 @@ static void __kprobes set_current_kprobe(struct kprobe *p)* simple and avoid nesting exceptions. Interrupts do have to be disabled since * the kprobe state is per-CPU and doesn't get migrated. */ -static void __kprobes kprobes_save_local_irqflag (struct kprobe_ctlblk *kcb, -struct pt_regs *regs) +static void noinstr kprobes_save_local_irqflag(struct kprobe_ctlblk *kcb, +struct pt_regs *regs) { kcb->saved_irqflag = regs->pstate &DAIF_MASK ; regs->pstate |=DAIF_MASK ; } -static void __kprobes kprobes_restore_local_irqflag (struct kprobe_ctlblk *kcb, -struct pt_regs *regs) +static void noinstr kprobes_restore_local_irqflag(struct kprobe_ctlblk *kcb, +struct pt_regs *regs) { regs->pstate &= ~DAIF_MASK ; regs->pstate |= kcb->saved_irqflag; } -static void __kprobes setup_singlestep (struct kprobe *p, -struct pt_regs *regs, -struct kprobe_ctlblk *kcb,int reenter) +static void noinstr setup_singlestep(struct kprobe *p, +struct pt_regs *regs, +struct kprobe_ctlblk *kcb,int reenter) {unsigned long slot;@@ -244,13 +245,15 @@ static void __kprobes setup_singlestep(struct kprobe *p,instruction_pointer_set (regs, slot); }else {/* insn simulation */ +instrumentation_begin ();arch_simulate_insn (p, regs); +instrumentation_end (); } } -static int __kprobes reenter_kprobe (struct kprobe *p, -struct pt_regs *regs, -struct kprobe_ctlblk *kcb) +static int noinstr reenter_kprobe(struct kprobe *p, +struct pt_regs *regs, +struct kprobe_ctlblk *kcb) {switch (kcb->kprobe_status) {case KPROBE_HIT_SSDONE :@@ -262,23 +265,29 @@ static int __kprobes reenter_kprobe(struct kprobe *p,* recoverable one-level reentry, so handle it in the same way as * reentry from KPROBE_HIT_ACTIVE or KPROBE_HIT_SSDONE. */ +instrumentation_begin ();kprobes_inc_nmissed_count (p); +instrumentation_end ();setup_singlestep (p, regs, kcb,1 );break ;case KPROBE_REENTER : +instrumentation_begin ();pr_warn ("Failed to recover from reentered kprobes.\n" );dump_kprobe (p);BUG (); +instrumentation_end ();break ;default : +instrumentation_begin ();WARN_ON (1 ); +instrumentation_end ();return 0 ; }return 1 ; } -static void __kprobes +static void noinstrpost_kprobe_handler (struct kprobe *cur,struct kprobe_ctlblk *kcb,struct pt_regs *regs) {/* return addr restore if non-branching insn */ Signature · 1 line
2.47.3
Re: [RFC PATCH 4/9] arm64/kprobes: Make the single-step machinery noinstr
Quoted 8 lines · Mark Rutland
Quoted 7 lines · Hongyan Xia
From: Hongyan Xia <[email protected]>
Convert the core of the arm64 kprobe single-step flow to noinstr:
save_previous_kprobe(),restore_previous_kprobe(),set_current_kprobe(),kprobes_save_local_irqflag(),kprobes_restore_local_irqflag(),setup_singlestep(),reenter_kprobe()andpost_kprobe_handler().
Quoted 8 lines
Quoted 5 lines
These functions only touch per-cpu state,
pt_regsand DAIF, except for:-
kprobes_inc_nmissed_count(), which is generic and instrumentable; add an arm64-local wrapper that calls it bounded by instrumentation_begin()/end();Why is it safe to call
kprobes_inc_nmissed_count()if it can be instrumented?
There is list_for_each_entry_rcu() in that function. With RCU debugging or proving configs on, messy things like lockdep get in the way. When I looked at this issue before, my conclusion was it was not possible to make the kprobes_inc_nmissed_count() path noinstr.
Quoted 2 lines
How does that work for other architectures?
I think (other maintainers please correct me if I'm very wrong on this) arm64 is the first arch to actually attempt Kprobe noinstr. For example, x86 does Kprobe in a big int3 instrumentation window, so they genuinely need KPROBE_SS re-entry support and Kprobe page fault check, because instrumentation can happen. This series is in the other direction:
Instead of patching all the possibilities each time we find one, make Kprobe noinstr to avoid the whole complexity.
Quoted 4 lines
Quoted 3 lines
- the instruction simulation path in
setup_singlestep(), whose decode handlers are instrumentable; boundarch_simulate_insn()with an instrumentation window;
Quoted 8 lines
I don't think that's sufficient. If we can instrument that code, then we can have unbounded recursion, unless I'm missing something?
I think we need to fix that code to be noinstr safe.
Ada Cc'd was looking into making the insn code generally noinstr-safe, but that's a big job. Maybe it's possible to clean up the subset that's necessary for
arch_simulate_insn()?
That would be fantastic. Yes, I believe this is the correct way to do noinstr here.
I'll investigate what that subset could be.
Quoted 147 lines
Quoted 2 lines
- the unrecoverable-reentry pr_warn()/dump_kprobe()/BUG() path and the default
WARN_ON(), both bounded with instrumentation windows.That sounds fine.
Mark.
Quoted 140 lines
The
__kprobesattribute (notrace + .kprobes.text) is replaced by noinstr, which is a strict superset for these functions.Signed-off-by Hongyan Xia <[email protected]>--- arch/arm64/kernel/probes/kprobes.c | 39 ++++++++++++++++++------------ 1 file changed, 24 insertions(+), 15 deletions(-)arch/arm64/kernel/probes/kprobes.c +24
-16@@ -14,6 +14,7 @@#include <linux/extable.h> #include <linux/kasan.h> #include <linux/kernel.h> +#include <linux/instrumentation.h> #include <linux/kprobes.h> #include <linux/sched/debug.h> #include <linux/set_memory.h> @@ -39,7 +40,7 @@ DEFINE_PER_CPU(struct kprobe *, current_kprobe) = NULL; DEFINE_PER_CPU(struct kprobe_ctlblk, kprobe_ctlblk); -static void __kprobes +static void noinstrpost_kprobe_handler (struct kprobe *,struct kprobe_ctlblk *,struct pt_regs *);void *alloc_insn_page (void )@@ -170,7 +171,7 @@ void __kprobes arch_remove_kprobe(struct kprobe *p) } } -static void __kprobes save_previous_kprobe (struct kprobe_ctlblk *kcb) +static void noinstr save_previous_kprobe(struct kprobe_ctlblk *kcb) { kcb->prev_kprobe.kp =kprobe_running (); kcb->prev_kprobe.status = kcb->kprobe_status;@@ -184,7 +185,7 @@ static void __kprobes save_previous_kprobe(struct kprobe_ctlblk *kcb) kcb->prev_kprobe.saved_irqflag = kcb->saved_irqflag; } -static void __kprobes restore_previous_kprobe (struct kprobe_ctlblk *kcb) +static void noinstr restore_previous_kprobe(struct kprobe_ctlblk *kcb) {__this_cpu_write (current_kprobe, kcb->prev_kprobe.kp); kcb->kprobe_status = kcb->prev_kprobe.status;@@ -197,7 +198,7 @@ static void __kprobes restore_previous_kprobe(struct kprobe_ctlblk *kcb) kcb->saved_irqflag = kcb->prev_kprobe.saved_irqflag; } -static void __kprobes set_current_kprobe (struct kprobe *p) +static void noinstr set_current_kprobe(struct kprobe *p) {__this_cpu_write (current_kprobe, p); }@@ -207,23 +208,23 @@ static void __kprobes set_current_kprobe(struct kprobe *p)* simple and avoid nesting exceptions. Interrupts do have to be disabled since * the kprobe state is per-CPU and doesn't get migrated. */ -static void __kprobes kprobes_save_local_irqflag (struct kprobe_ctlblk *kcb, -struct pt_regs *regs) +static void noinstr kprobes_save_local_irqflag(struct kprobe_ctlblk *kcb, +struct pt_regs *regs) { kcb->saved_irqflag = regs->pstate &DAIF_MASK ; regs->pstate |=DAIF_MASK ; } -static void __kprobes kprobes_restore_local_irqflag (struct kprobe_ctlblk *kcb, -struct pt_regs *regs) +static void noinstr kprobes_restore_local_irqflag(struct kprobe_ctlblk *kcb, +struct pt_regs *regs) { regs->pstate &= ~DAIF_MASK ; regs->pstate |= kcb->saved_irqflag; } -static void __kprobes setup_singlestep (struct kprobe *p, -struct pt_regs *regs, -struct kprobe_ctlblk *kcb,int reenter) +static void noinstr setup_singlestep(struct kprobe *p, +struct pt_regs *regs, +struct kprobe_ctlblk *kcb,int reenter) {unsigned long slot;@@ -244,13 +245,15 @@ static void __kprobes setup_singlestep(struct kprobe *p,instruction_pointer_set (regs, slot); }else {/* insn simulation */ +instrumentation_begin ();arch_simulate_insn (p, regs); +instrumentation_end (); } } -static int __kprobes reenter_kprobe (struct kprobe *p, -struct pt_regs *regs, -struct kprobe_ctlblk *kcb) +static int noinstr reenter_kprobe(struct kprobe *p, +struct pt_regs *regs, +struct kprobe_ctlblk *kcb) {switch (kcb->kprobe_status) {case KPROBE_HIT_SSDONE :@@ -262,23 +265,29 @@ static int __kprobes reenter_kprobe(struct kprobe *p,* recoverable one-level reentry, so handle it in the same way as * reentry from KPROBE_HIT_ACTIVE or KPROBE_HIT_SSDONE. */ +instrumentation_begin ();kprobes_inc_nmissed_count (p); +instrumentation_end ();setup_singlestep (p, regs, kcb,1 );break ;case KPROBE_REENTER : +instrumentation_begin ();pr_warn ("Failed to recover from reentered kprobes.\n" );dump_kprobe (p);BUG (); +instrumentation_end ();break ;default : +instrumentation_begin ();WARN_ON (1 ); +instrumentation_end ();return 0 ; }return 1 ; } -static void __kprobes +static void noinstrpost_kprobe_handler (struct kprobe *cur,struct kprobe_ctlblk *kcb,struct pt_regs *regs) {/* return addr restore if non-branching insn */ --2.47.3
Re: [RFC PATCH 4/9] arm64/kprobes: Make the single-step machinery noinstr
Hi Mark, Ada,
Quoted 9 lines · Hongyan Xia
Quoted 1 line · Mark Rutland
On Mon, Jul 27, 2026 at 12:25:39PM +0000, Hongyan Xia wrote:
[...]
Quoted 6 lines
I don't think that's sufficient. If we can instrument that code, then we can have unbounded recursion, unless I'm missing something?
I think we need to fix that code to be noinstr safe.
Quoted 8 lines
Quoted 3 lines
Ada Cc'd was looking into making the insn code generally noinstr-safe, but that's a big job. Maybe it's possible to clean up the subset that's necessary for
arch_simulate_insn()?That would be fantastic. Yes, I believe this is the correct way to do noinstr here.
I'll investigate what that subset could be.
The biggest violation is the GCS handling in update_lr() and simulate_ret(). You might want to see if they can be noinstr'ed or they need instrumentation_{begin/end}() treatment. Others are trivial.
Actually I wonder if it's worth turning our noinstr check script into an RFC arm64 objtool --noinstr series. We've been using it to check noinstr changes. We don't need to merge it now, but it might be useful to just have the series somewhere for you to apply and verify noinstr changes.
Quoted 1 line
[...]
RFC PATCH 5/9 arm64/kprobes: Invoke pre/post handlers inside instrumentation
In reply to Hongyan XiaFrom: Hongyan Xia <[email protected]>
The kprobe pre_handler() and post_handler() are arbitrary instrumentable code and can themselves trace, fault, or hit other kprobes. They are the only parts of the kprobe debug exception flow that legitimately need instrumentation.
--- arch/arm64/include/asm/kprobes.h | 9 +++------ arch/arm64/kernel/probes/kprobes.c | 22 ++++++++++++++++++---- 2 files changed, 21 insertions(+), 10 deletions(-)
arch/arm64/include/asm/kprobes.h +3 -6
@@ -48,11 +48,8 @@ void __kprobes *trampoline_probe_handler(struct pt_regs *regs);#endif /* CONFIG_KPROBES */ -int __kprobes kprobe_brk_handler (struct pt_regs *regs, -unsigned long esr); -int __kprobes kprobe_ss_brk_handler (struct pt_regs *regs, -unsigned long esr); -int __kprobes kretprobe_brk_handler (struct pt_regs *regs, -unsigned long esr); +int noinstr kprobe_brk_handler(struct pt_regs *regs, unsigned long esr); +int noinstr kprobe_ss_brk_handler(struct pt_regs *regs, unsigned long esr); +int noinstr kretprobe_brk_handler(struct pt_regs *regs, unsigned long esr);#endif /* _ARM_KPROBES_H */
arch/arm64/kernel/probes/kprobes.c +18 -4
@@ -301,8 +301,11 @@ post_kprobe_handler(struct kprobe *cur, struct kprobe_ctlblk *kcb, struct pt_reg }/* call post handler */ kcb->kprobe_status =KPROBE_HIT_SSDONE ; + +instrumentation_begin ();if (cur->post_handler) cur->post_handler (cur, regs,0 ); +instrumentation_end ();reset_current_kprobe (); }
@@ -359,24 +362,28 @@ int __kprobes kprobe_fault_handler(struct pt_regs *regs, unsigned int fsr)return 0 ; } -int __kprobes +int noinstrkprobe_brk_handler (struct pt_regs *regs,unsigned long esr) {struct kprobe *p, *cur_kprobe;struct kprobe_ctlblk *kcb;unsigned long addr =instruction_pointer (regs); +bool handled; kcb =get_kprobe_ctlblk (); cur_kprobe =kprobe_running (); +instrumentation_begin (); p =get_kprobe ((kprobe_opcode_t *) addr);if (WARN_ON_ONCE (!p)) { +instrumentation_end ();/* * Something went wrong. This BRK used an immediate reserved * for kprobes, but we couldn't find any corresponding probe. */ return DBG_HOOK_ERROR ; } +instrumentation_end ();if (cur_kprobe) {/* Hit a kprobe inside another kprobe */
@@ -387,6 +394,10 @@ kprobe_brk_handler(struct pt_regs *regs, unsigned long esr)set_current_kprobe (p); kcb->kprobe_status =KPROBE_HIT_ACTIVE ; +instrumentation_begin (); + handled = p->pre_handler && p->pre_handler (p, regs); +instrumentation_end (); +/* * If we have no pre-handler or it returned 0, we * continue with normal processing. If we have a
@@ -394,7 +405,7 @@ kprobe_brk_handler(struct pt_regs *regs, unsigned long esr)* modify the execution path and not need to single-step * Let's just reset current kprobe and exit. */ -if (!p->pre_handler || !p->pre_handler (p, regs)) +if (!handled)setup_singlestep (p, regs, kcb,0 );else reset_current_kprobe ();
@@ -403,7 +414,7 @@ kprobe_brk_handler(struct pt_regs *regs, unsigned long esr)return DBG_HOOK_HANDLED ; } -int __kprobes +int noinstrkprobe_ss_brk_handler (struct pt_regs *regs,unsigned long esr) {struct kprobe_ctlblk *kcb =get_kprobe_ctlblk ();
@@ -422,13 +433,16 @@ kprobe_ss_brk_handler(struct pt_regs *regs, unsigned long esr)return DBG_HOOK_ERROR ; } -int __kprobes +int noinstrkretprobe_brk_handler (struct pt_regs *regs,unsigned long esr) {if (regs->pc != (unsigned long )__kretprobe_trampoline)return DBG_HOOK_ERROR ; +instrumentation_begin (); regs->pc =kretprobe_trampoline_handler (regs, (void *)regs->regs[29 ]); +instrumentation_end (); +return DBG_HOOK_HANDLED ; }
Signature · 1 line
2.47.3
Mark Rutland
reviewerRe: [RFC PATCH 5/9] arm64/kprobes: Invoke pre/post handlers inside instrumentation
Quoted 6 lines · Hongyan Xia
From: Hongyan Xia <[email protected]>
The kprobe
pre_handler()andpost_handler()are arbitrary instrumentable code and can themselves trace, fault, or hit other kprobes. They are the only parts of the kprobe debug exception flow that legitimately need instrumentation.
That doesn't explain what this patch changes, or why.
I'm fine with marking all the kprobe functions noinstr, but we don't need the intrumentation_{begin,end}() gunk.
In general, it's not sound for pre_handler and post_handler to invoke arbitrary code, given that they could lead to recursion. We just hope people don't do stupid things with them.
Mark.
Quoted 121 lines
Signed-off-by Hongyan Xia <[email protected]>--- arch/arm64/include/asm/kprobes.h | 9 +++------ arch/arm64/kernel/probes/kprobes.c | 22 ++++++++++++++++++---- 2 files changed, 21 insertions(+), 10 deletions(-)arch/arm64/include/asm/kprobes.h +3
-6@@ -48,11 +48,8 @@ void __kprobes *trampoline_probe_handler(struct pt_regs *regs);#endif /* CONFIG_KPROBES */ -int __kprobes kprobe_brk_handler (struct pt_regs *regs, -unsigned long esr); -int __kprobes kprobe_ss_brk_handler (struct pt_regs *regs, -unsigned long esr); -int __kprobes kretprobe_brk_handler (struct pt_regs *regs, -unsigned long esr); +int noinstr kprobe_brk_handler(struct pt_regs *regs, unsigned long esr); +int noinstr kprobe_ss_brk_handler(struct pt_regs *regs, unsigned long esr); +int noinstr kretprobe_brk_handler(struct pt_regs *regs, unsigned long esr);#endif /* _ARM_KPROBES_H */ arch/arm64/kernel/probes/kprobes.c +18
-4@@ -301,8 +301,11 @@ post_kprobe_handler(struct kprobe *cur, struct kprobe_ctlblk *kcb, struct pt_reg }/* call post handler */ kcb->kprobe_status =KPROBE_HIT_SSDONE ; + +instrumentation_begin ();if (cur->post_handler) cur->post_handler (cur, regs,0 ); +instrumentation_end ();reset_current_kprobe (); }@@ -359,24 +362,28 @@ int __kprobes kprobe_fault_handler(struct pt_regs *regs, unsigned int fsr)return 0 ; } -int __kprobes +int noinstrkprobe_brk_handler (struct pt_regs *regs,unsigned long esr) {struct kprobe *p, *cur_kprobe;struct kprobe_ctlblk *kcb;unsigned long addr =instruction_pointer (regs); +bool handled; kcb =get_kprobe_ctlblk (); cur_kprobe =kprobe_running (); +instrumentation_begin (); p =get_kprobe ((kprobe_opcode_t *) addr);if (WARN_ON_ONCE (!p)) { +instrumentation_end ();/* * Something went wrong. This BRK used an immediate reserved * for kprobes, but we couldn't find any corresponding probe. */ return DBG_HOOK_ERROR ; } +instrumentation_end ();if (cur_kprobe) {/* Hit a kprobe inside another kprobe */ @@ -387,6 +394,10 @@ kprobe_brk_handler(struct pt_regs *regs, unsigned long esr)set_current_kprobe (p); kcb->kprobe_status =KPROBE_HIT_ACTIVE ; +instrumentation_begin (); + handled = p->pre_handler && p->pre_handler (p, regs); +instrumentation_end (); +/* * If we have no pre-handler or it returned 0, we * continue with normal processing. If we have a @@ -394,7 +405,7 @@ kprobe_brk_handler(struct pt_regs *regs, unsigned long esr)* modify the execution path and not need to single-step * Let's just reset current kprobe and exit. */ -if (!p->pre_handler || !p->pre_handler (p, regs)) +if (!handled)setup_singlestep (p, regs, kcb,0 );else reset_current_kprobe ();@@ -403,7 +414,7 @@ kprobe_brk_handler(struct pt_regs *regs, unsigned long esr)return DBG_HOOK_HANDLED ; } -int __kprobes +int noinstrkprobe_ss_brk_handler (struct pt_regs *regs,unsigned long esr) {struct kprobe_ctlblk *kcb =get_kprobe_ctlblk ();@@ -422,13 +433,16 @@ kprobe_ss_brk_handler(struct pt_regs *regs, unsigned long esr)return DBG_HOOK_ERROR ; } -int __kprobes +int noinstrkretprobe_brk_handler (struct pt_regs *regs,unsigned long esr) {if (regs->pc != (unsigned long )__kretprobe_trampoline)return DBG_HOOK_ERROR ; +instrumentation_begin (); regs->pc =kretprobe_trampoline_handler (regs, (void *)regs->regs[29 ]); +instrumentation_end (); +return DBG_HOOK_HANDLED ; }Signature · 1 line
2.47.3
Re: [RFC PATCH 5/9] arm64/kprobes: Invoke pre/post handlers inside instrumentation
Quoted 8 lines · Mark Rutland
Quoted 6 lines · Hongyan Xia
From: Hongyan Xia <[email protected]>
The kprobe
pre_handler()andpost_handler()are arbitrary instrumentable code and can themselves trace, fault, or hit other kprobes. They are the only parts of the kprobe debug exception flow that legitimately need instrumentation.
Quoted 4 lines
That doesn't explain what this patch changes, or why.
I'm fine with marking all the kprobe functions noinstr, but we don't need the intrumentation_{begin,end}() gunk.
Some of the helpers are quite difficult to deal with. The get_kprobe() has RCU debugging inside, just like kprobe_inc_nmissed_count(). We could further narrow down the instrumentation window, but making it completely noinstr might not be feasible.
Quoted 3 lines
In general, it's not sound for
pre_handlerandpost_handlerto invoke arbitrary code, given that they could lead to recursion. We just hope people don't do stupid things with them.
I think this is still a big step forward. Before, we spent several weeks tracking down the Kprobe bug which eventually resulted in 879a6754 and 23f851ac. It was super time-consuming to debug because 1) well, it's Kprobe and 2) the entire path could be instrumented and hard to narrow down. Now that we know such things could only happen in a couple of tiny helpers and the pre- and post-handlers, the surface is massively reduced and much easier to reason about, even if it's not 100% noinstr.
But of course, this is RFC. Would greatly appreciate better ideas. I'm also not keen on leaving instrumentation_{begin/end}() around.
Quoted 124 lines
Mark.
Quoted 121 lines
Signed-off-by Hongyan Xia <[email protected]>--- arch/arm64/include/asm/kprobes.h | 9 +++------ arch/arm64/kernel/probes/kprobes.c | 22 ++++++++++++++++++---- 2 files changed, 21 insertions(+), 10 deletions(-)arch/arm64/include/asm/kprobes.h +3
-6@@ -48,11 +48,8 @@ void __kprobes *trampoline_probe_handler(struct pt_regs *regs);#endif /* CONFIG_KPROBES */ -int __kprobes kprobe_brk_handler (struct pt_regs *regs, -unsigned long esr); -int __kprobes kprobe_ss_brk_handler (struct pt_regs *regs, -unsigned long esr); -int __kprobes kretprobe_brk_handler (struct pt_regs *regs, -unsigned long esr); +int noinstr kprobe_brk_handler(struct pt_regs *regs, unsigned long esr); +int noinstr kprobe_ss_brk_handler(struct pt_regs *regs, unsigned long esr); +int noinstr kretprobe_brk_handler(struct pt_regs *regs, unsigned long esr);#endif /* _ARM_KPROBES_H */ arch/arm64/kernel/probes/kprobes.c +18
-5@@ -301,8 +301,11 @@ post_kprobe_handler(struct kprobe *cur, struct kprobe_ctlblk *kcb, struct pt_reg }/* call post handler */ kcb->kprobe_status =KPROBE_HIT_SSDONE ; + +instrumentation_begin ();if (cur->post_handler) cur->post_handler (cur, regs,0 ); +instrumentation_end ();reset_current_kprobe (); }@@ -359,24 +362,28 @@ int __kprobes kprobe_fault_handler(struct pt_regs *regs, unsigned int fsr)return 0 ; } -int __kprobes +int noinstrkprobe_brk_handler (struct pt_regs *regs,unsigned long esr) {struct kprobe *p, *cur_kprobe;struct kprobe_ctlblk *kcb;unsigned long addr =instruction_pointer (regs); +bool handled; kcb =get_kprobe_ctlblk (); cur_kprobe =kprobe_running (); +instrumentation_begin (); p =get_kprobe ((kprobe_opcode_t *) addr);if (WARN_ON_ONCE (!p)) { +instrumentation_end ();/* * Something went wrong. This BRK used an immediate reserved * for kprobes, but we couldn't find any corresponding probe. */ return DBG_HOOK_ERROR ; } +instrumentation_end ();if (cur_kprobe) {/* Hit a kprobe inside another kprobe */ @@ -387,6 +394,10 @@ kprobe_brk_handler(struct pt_regs *regs, unsigned long esr)set_current_kprobe (p); kcb->kprobe_status =KPROBE_HIT_ACTIVE ; +instrumentation_begin (); + handled = p->pre_handler && p->pre_handler (p, regs); +instrumentation_end (); +/* * If we have no pre-handler or it returned 0, we * continue with normal processing. If we have a @@ -394,7 +405,7 @@ kprobe_brk_handler(struct pt_regs *regs, unsigned long esr)* modify the execution path and not need to single-step * Let's just reset current kprobe and exit. */ -if (!p->pre_handler || !p->pre_handler (p, regs)) +if (!handled)setup_singlestep (p, regs, kcb,0 );else reset_current_kprobe ();@@ -403,7 +414,7 @@ kprobe_brk_handler(struct pt_regs *regs, unsigned long esr)return DBG_HOOK_HANDLED ; } -int __kprobes +int noinstrkprobe_ss_brk_handler (struct pt_regs *regs,unsigned long esr) {struct kprobe_ctlblk *kcb =get_kprobe_ctlblk ();@@ -422,13 +433,16 @@ kprobe_ss_brk_handler(struct pt_regs *regs, unsigned long esr)return DBG_HOOK_ERROR ; } -int __kprobes +int noinstrkretprobe_brk_handler (struct pt_regs *regs,unsigned long esr) {if (regs->pc != (unsigned long )__kretprobe_trampoline)return DBG_HOOK_ERROR ; +instrumentation_begin (); regs->pc =kretprobe_trampoline_handler (regs, (void *)regs->regs[29 ]); +instrumentation_end (); +return DBG_HOOK_HANDLED ; } --2.47.3
RFC PATCH 6/9 arm64/kprobes: Make kprobe_fault_handler() noinstr
In reply to Hongyan XiaFrom: Hongyan Xia <[email protected]>
kprobe_fault_handler() is reached from do_page_fault() when a fault happens while a kprobe single-step is armed. Everything it calls (the save/restore helpers, kprobes_restore_local_irqflag(), reset_current_kprobe(), register access) is noinstr after the previous patch, and being called from an instrumentable caller into a noinstr callee is fine.
--- arch/arm64/kernel/probes/kprobes.c | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-)
arch/arm64/kernel/probes/kprobes.c +1 -1
@@ -310,7 +310,7 @@ post_kprobe_handler(struct kprobe *cur, struct kprobe_ctlblk *kcb, struct pt_regreset_current_kprobe (); } -int __kprobes kprobe_fault_handler (struct pt_regs *regs,unsigned int fsr) +int noinstr kprobe_fault_handler(struct pt_regs *regs, unsigned int fsr) {struct kprobe *cur =kprobe_running ();struct kprobe_ctlblk *kcb =get_kprobe_ctlblk ();
Signature · 1 line
2.47.3
Mark Rutland
reviewerRe: [RFC PATCH 6/9] arm64/kprobes: Make kprobe_fault_handler() noinstr
Quoted 8 lines · Hongyan Xia
From: Hongyan Xia <[email protected]>
kprobe_fault_handler()is reached fromdo_page_fault()when a fault happens while a kprobe single-step is armed. Everything it calls (the save/restore helpers,kprobes_restore_local_irqflag(),reset_current_kprobe(), register access) is noinstr after the previous patch, and being called from an instrumentable caller into a noinstr callee is fine.
I think it's fine to mark this as noinstr.
I don't think we need kprobe_fault_handler() in the first place given we don't permit kprobes to be placed on instructions with an extable entry. Any instruction which can be probled cannot legitimately cause a fault.
That and kprobes_restore_local_irqflag() can't do the right thing w.r.t DAIF, since it only fiddles with the value in the pt_regs, and doesn't touch the live inherited DAIF value. If we need to fiddle with the live DAIF value, we must do that much earlier in the entry-common code.
Mark.
Quoted 21 lines
Signed-off-by Hongyan Xia <[email protected]>--- arch/arm64/kernel/probes/kprobes.c | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-)arch/arm64/kernel/probes/kprobes.c +1
-1@@ -310,7 +310,7 @@ post_kprobe_handler(struct kprobe *cur, struct kprobe_ctlblk *kcb, struct pt_regreset_current_kprobe (); } -int __kprobes kprobe_fault_handler (struct pt_regs *regs,unsigned int fsr) +int noinstr kprobe_fault_handler(struct pt_regs *regs, unsigned int fsr) {struct kprobe *cur =kprobe_running ();struct kprobe_ctlblk *kcb =get_kprobe_ctlblk ();Signature · 1 line
2.47.3
Re: [RFC PATCH 6/9] arm64/kprobes: Make kprobe_fault_handler() noinstr
Quoted 10 lines · Mark Rutland
Quoted 8 lines · Hongyan Xia
From: Hongyan Xia <[email protected]>
kprobe_fault_handler()is reached fromdo_page_fault()when a fault happens while a kprobe single-step is armed. Everything it calls (the save/restore helpers,kprobes_restore_local_irqflag(),reset_current_kprobe(), register access) is noinstr after the previous patch, and being called from an instrumentable caller into a noinstr callee is fine.
Quoted 5 lines
I think it's fine to mark this as noinstr.
I don't think we need
kprobe_fault_handler()in the first place given we don't permit kprobes to be placed on instructions with an extable entry. Any instruction which can be probled cannot legitimately cause a fault.
I think so, too. If we cannot attach to ex_table entry instructions, then there is no point in handling a page fault. The only thing worth being in kprobe_fault_handler() is probably just a BUG(). Or, we could remove it entirely.
Quoted 4 lines
That and
kprobes_restore_local_irqflag()can't do the right thing w.r.t DAIF, since it only fiddles with the value in thept_regs, and doesn't touch the live inherited DAIF value. If we need to fiddle with the live DAIF value, we must do that much earlier in the entry-common code.
This should be fine. It manipulates DAIF in the previous fault, not the live one, and the value will be picked up correctly on ERET. kprobes_{save/restore}_local_irqflag() doesn't change anything in the current exception, unlike local_irq_save().
Unless you mean something else?
Quoted 23 lines
Mark.
Quoted 21 lines
Signed-off-by Hongyan Xia <[email protected]>--- arch/arm64/kernel/probes/kprobes.c | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-)arch/arm64/kernel/probes/kprobes.c +1
-2@@ -310,7 +310,7 @@ post_kprobe_handler(struct kprobe *cur, struct kprobe_ctlblk *kcb, struct pt_regreset_current_kprobe (); } -int __kprobes kprobe_fault_handler (struct pt_regs *regs,unsigned int fsr) +int noinstr kprobe_fault_handler(struct pt_regs *regs, unsigned int fsr) {struct kprobe *cur =kprobe_running ();struct kprobe_ctlblk *kcb =get_kprobe_ctlblk (); --2.47.3
RFC PATCH 7/9 arm64/kprobes: Drop the KPROBE_HIT_SS reentry special case
In reply to Hongyan XiaFrom: Hongyan Xia <[email protected]>
With the debug exception path noinstr from the vectors down to the kprobe handlers and the only instrumentable code in the flow confined to instrumentation_begin()/end() windows, a kprobe hit while another probe is in KPROBE_HIT_SS can no longer happen.
--- arch/arm64/include/asm/kprobes.h | 6 ------ arch/arm64/kernel/probes/kprobes.c | 21 +-------------------- 2 files changed, 1 insertion(+), 26 deletions(-)
arch/arm64/include/asm/kprobes.h +0 -6
@@ -26,12 +26,6 @@struct prev_kprobe {struct kprobe *kp;unsigned int status; - -/* -* The original DAIF state of the outer kprobe, saved here before -* a nested kprobe overwrites kcb->saved_irqflag during reentry. -*/ -unsigned long saved_irqflag; };/* per-cpu kprobe control block */
arch/arm64/kernel/probes/kprobes.c +1 -20
@@ -176,13 +176,6 @@ static void noinstr save_previous_kprobe(struct kprobe_ctlblk *kcb) kcb->prev_kprobe.kp =kprobe_running (); kcb->prev_kprobe.status = kcb->kprobe_status; -/* -* Save the outer kprobe's original DAIF flags before the nested -* kprobe calls kprobes_save_local_irqflag() and overwrites -* kcb->saved_irqflag. Without this, the outer kprobe will restore -* the wrong DAIF state and leave interrupts permanently masked. -*/ - kcb->prev_kprobe.saved_irqflag = kcb->saved_irqflag; } static void noinstr restore_previous_kprobe(struct kprobe_ctlblk *kcb)
@@ -190,12 +183,6 @@ static void noinstr restore_previous_kprobe(struct kprobe_ctlblk *kcb)__this_cpu_write (current_kprobe, kcb->prev_kprobe.kp); kcb->kprobe_status = kcb->prev_kprobe.status; -/* -* Restore the outer kprobe's saved_irqflag so that when its -* single-step completes, kprobes_restore_local_irqflag() uses -* the correct original DAIF value. -*/ - kcb->saved_irqflag = kcb->prev_kprobe.saved_irqflag; } static void noinstr set_current_kprobe(struct kprobe *p)
@@ -258,18 +245,12 @@ static int noinstr reenter_kprobe(struct kprobe *p,switch (kcb->kprobe_status) {case KPROBE_HIT_SSDONE :case KPROBE_HIT_ACTIVE : -case KPROBE_HIT_SS : -/* -* A probe can be hit while another kprobe is preparing or -* executing its XOL single-step instruction. This is still a -* recoverable one-level reentry, so handle it in the same way as -* reentry from KPROBE_HIT_ACTIVE or KPROBE_HIT_SSDONE. -*/ instrumentation_begin ();kprobes_inc_nmissed_count (p);instrumentation_end ();setup_singlestep (p, regs, kcb,1 );break ; +case KPROBE_HIT_SS :case KPROBE_REENTER :instrumentation_begin ();pr_warn ("Failed to recover from reentered kprobes.\n" );
Signature · 1 line
2.47.3
RFC PATCH 8/9 arm64/kprobes: Drop the XOL single-step fault PC check
In reply to Hongyan XiaFrom: Hongyan Xia <[email protected]>
With the armed single-step window closed (the whole path noinstr, and every piece of instrumentable code confined to instrumentation windows that run in KPROBE_HIT_ACTIVE/HIT_SSDONE state), no code other than the XOL instruction itself can execute while the state is KPROBE_HIT_SS or KPROBE_REENTER. A page fault taken in those states therefore necessarily originates from the XOL instruction, and the faulting-PC comparison is redundant; revert to handling all such faults as single-step faults (as before commit 879a6754d3d1).
The simulated-kprobe early bail is kept: simulated probes have no XOL slot and execute (inside an instrumentation window) in debug trap context, so a fault there can still come from the simulation handlers and must not be treated as a single-step fault.
--- arch/arm64/kernel/probes/kprobes.c | 12 ------------ 1 file changed, 12 deletions(-)
arch/arm64/kernel/probes/kprobes.c +0 -12
@@ -309,18 +309,6 @@ int noinstr kprobe_fault_handler(struct pt_regs *regs, unsigned int fsr)switch (kcb->kprobe_status) {case KPROBE_HIT_SS :case KPROBE_REENTER : -/* -* A page fault taken while in KPROBE_HIT_SS or -* KPROBE_REENTER state is only attributable to kprobe -* single-stepping if the faulting PC points to the -* current kprobe's XOL instruction. If the fault occurred -* elsewhere (e.g. in perf or tracing code invoked from the -* debug exception path), leave it for the normal page fault -* handler to process. -*/ -if (instruction_pointer (regs) != (unsigned long )cur->ainsn.xol_insn) -break ; -/* * We are here because the instruction being single * stepped caused a page fault. We reset the current
Signature · 1 line
2.47.3
Mark Rutland
reviewerRe: [RFC PATCH 8/9] arm64/kprobes: Drop the XOL single-step fault PC check
Quoted 11 lines · Hongyan Xia
From: Hongyan Xia <[email protected]>
With the armed single-step window closed (the whole path noinstr, and every piece of instrumentable code confined to instrumentation windows that run in KPROBE_HIT_ACTIVE/HIT_SSDONE state), no code other than the XOL instruction itself can execute while the state is
KPROBE_HIT_SSorKPROBE_REENTER. A page fault taken in those states therefore necessarily originates from the XOL instruction, and the faulting-PC comparison is redundant; revert to handling all such faults as single-step faults (as before commit879a6754d3d1).
Quoted 4 lines
The simulated-kprobe early bail is kept: simulated probes have no XOL slot and execute (inside an instrumentation window) in debug trap context, so a fault there can still come from the simulation handlers and must not be treated as a single-step fault.
As commented on an earlier patch, to fix this properly I think we have to make the simulation noinstr-safe.
Mark.
Quoted 32 lines
Signed-off-by Hongyan Xia <[email protected]>--- arch/arm64/kernel/probes/kprobes.c | 12 ------------ 1 file changed, 12 deletions(-)arch/arm64/kernel/probes/kprobes.c +0
-12@@ -309,18 +309,6 @@ int noinstr kprobe_fault_handler(struct pt_regs *regs, unsigned int fsr)switch (kcb->kprobe_status) {case KPROBE_HIT_SS :case KPROBE_REENTER : -/* -* A page fault taken while in KPROBE_HIT_SS or -* KPROBE_REENTER state is only attributable to kprobe -* single-stepping if the faulting PC points to the -* current kprobe's XOL instruction. If the fault occurred -* elsewhere (e.g. in perf or tracing code invoked from the -* debug exception path), leave it for the normal page fault -* handler to process. -*/ -if (instruction_pointer (regs) != (unsigned long )cur->ainsn.xol_insn) -break ; -/* * We are here because the instruction being single * stepped caused a page fault. We reset the current Signature · 1 line
2.47.3
RFC PATCH 9/9 arm64/debug: Mark debug exception helpers __always_inline
In reply to Hongyan XiaFrom: Hongyan Xia <[email protected]>
Static inline should be enough to actually inline functions for most compilers, but my Clang-19 somehow thinks it's better to outline them. These tiny helpers then live in normal .text sections instead of .noinstr sections, violating noinstr.
Mark them __always_inline so the compiler can never outline them.
--- arch/arm64/include/asm/esr.h | 2 +- arch/arm64/include/asm/percpu.h | 2 +- arch/arm64/include/asm/preempt.h | 4 ++-- arch/arm64/include/asm/ptrace.h | 4 ++-- arch/arm64/kernel/debug-monitors.c | 2 +- include/linux/kprobes.h | 8 ++++---- 6 files changed, 11 insertions(+), 11 deletions(-)
arch/arm64/include/asm/esr.h +1 -1
@@ -437,7 +437,7 @@#ifndef __ASSEMBLER__#include <asm/types.h> -static inline unsigned long esr_brk_comment (unsigned long esr) +static __always_inline unsigned long esr_brk_comment (unsigned long esr) {return esr & ESR_ELx_BRK64_ISS_COMMENT_MASK; }
arch/arm64/include/asm/percpu.h +1 -1
@@ -29,7 +29,7 @@ static inline unsigned long __hyp_my_cpu_offset(void)return read_sysreg (tpidr_el2); } -static inline unsigned long __kern_my_cpu_offset (void ) +static __always_inline unsigned long __kern_my_cpu_offset (void ) {unsigned long off;
arch/arm64/include/asm/preempt.h +2 -2
@@ -41,14 +41,14 @@ static inline bool test_preempt_need_resched(void)return !current_thread_info ()->preempt.need_resched; } -static inline void __preempt_count_add (int val) +static __always_inline void __preempt_count_add(int val) {u32 pc =READ_ONCE (current_thread_info ()->preempt.count); pc += val;WRITE_ONCE (current_thread_info ()->preempt.count, pc); } -static inline void __preempt_count_sub (int val) +static __always_inline void __preempt_count_sub(int val) {u32 pc =READ_ONCE (current_thread_info ()->preempt.count); pc -= val;
arch/arm64/include/asm/ptrace.h +2 -2
@@ -336,11 +336,11 @@ static inline unsigned long regs_get_kernel_argument(struct pt_regs *regs,struct task_struct ;int valid_user_regs (struct user_pt_regs *regs,struct task_struct *task); -static inline unsigned long instruction_pointer (struct pt_regs *regs) +static __always_inline unsigned long instruction_pointer (struct pt_regs *regs) {return regs->pc; } -static inline void instruction_pointer_set (struct pt_regs *regs, +static __always_inline void instruction_pointer_set(struct pt_regs *regs,unsigned long val) { regs->pc = val;
arch/arm64/kernel/debug-monitors.c +1 -1
@@ -146,7 +146,7 @@ postcore_initcall(debug_monitors_init);/* * Single step API and exception handling. */ -static void set_user_regs_spsr_ss (struct user_pt_regs *regs) +static __always_inline void set_user_regs_spsr_ss(struct user_pt_regs *regs) { regs->pstate |=DBG_SPSR_SS ; }
include/linux/kprobes.h +4 -4
@@ -384,17 +384,17 @@ static inline void kprobe_ftrace_kill(void) {}struct kprobe *get_kprobe (void *addr);/* kprobe_running() will just return the current_kprobe on this CPU */ -static inline struct kprobe *kprobe_running (void ) +static __always_inline struct kprobe *kprobe_running(void) {return __this_cpu_read (current_kprobe); } -static inline void reset_current_kprobe (void ) +static __always_inline void reset_current_kprobe(void) {__this_cpu_write (current_kprobe,NULL ); } -static inline struct kprobe_ctlblk *get_kprobe_ctlblk (void ) +static __always_inline struct kprobe_ctlblk *get_kprobe_ctlblk(void) {return this_cpu_ptr (&kprobe_ctlblk); }
@@ -449,7 +449,7 @@ static inline struct kprobe *get_kprobe(void *addr) {return NULL ; } -static inline struct kprobe *kprobe_running (void ) +static __always_inline struct kprobe *kprobe_running(void) {return NULL ; }
Signature · 1 line
2.47.3
Re: [RFC PATCH 9/9] arm64/debug: Mark debug exception helpers __always_inline
Quoted 7 lines · Hongyan Xia
From: Hongyan Xia <[email protected]>
Static inline should be enough to actually inline functions for most compilers, but my Clang-19 somehow thinks it's better to outline them. These tiny helpers then live in normal .text sections instead of .noinstr sections, violating noinstr.
Anything special about your build or config? For example, we've seen a recent influx of bugs around AutoFDO not inlining functions (despite constant propagating initdata constants into them) leading to modpost errors.
Can you perhaps share your config so that someone else could repro and take a look?
Happy to follow up in our issue tracker, too.
https://github.com/ClangBuiltLinux/linux/issues
Quoted 11 lines
Mark them
__always_inlineso the compiler can never outline them.Signed-off-by Hongyan Xia <[email protected]>--- arch/arm64/include/asm/esr.h | 2 +- arch/arm64/include/asm/percpu.h | 2 +- arch/arm64/include/asm/preempt.h | 4 ++-- arch/arm64/include/asm/ptrace.h | 4 ++-- arch/arm64/kernel/debug-monitors.c | 2 +- include/linux/kprobes.h | 8 ++++----
It might not be appropriate for the changes to kprobes.h to go through the arm64 tree, but maybe it is?
Signature · 2 lines
Thanks, ~Nick Desaulniers
Will Deacon
maintainerRe: [RFC PATCH 9/9] arm64/debug: Mark debug exception helpers __always_inline
Quoted 10 lines · Nick Desaulniers
Quoted 6 lines · Hongyan Xia
arch/arm64/include/asm/esr.h | 2 +- arch/arm64/include/asm/percpu.h | 2 +- arch/arm64/include/asm/preempt.h | 4 ++-- arch/arm64/include/asm/ptrace.h | 4 ++-- arch/arm64/kernel/debug-monitors.c | 2 +- include/linux/kprobes.h | 8 ++++----It might not be appropriate for the changes to
kprobes.hto go through the arm64 tree, but maybe it is?
They look pretty trivial, so we can take 'em with Masami's Ack once the rest of the series has been reviewed.
Will
Re: [RFC PATCH 9/9] arm64/debug: Mark debug exception helpers __always_inline
In reply to Nick DesaulniersQuoted 9 lines · Nick Desaulniers
Quoted 7 lines · Hongyan Xia
From: Hongyan Xia <[email protected]>
Static inline should be enough to actually inline functions for most compilers, but my Clang-19 somehow thinks it's better to outline them. These tiny helpers then live in normal .text sections instead of .noinstr sections, violating noinstr.
Quoted 4 lines
Anything special about your build or config? For example, we've seen a recent influx of bugs around AutoFDO not inlining functions (despite constant propagating initdata constants into them) leading to modpost errors.
I doubt there's anything special. The compiler is Debian clang version 19.1.7 (3+b1) installed through apt install in vanilla x86_64 Debian Trixie, compiled with 'make LLVM=1 ARCH=arm64 Image'.
Quoted 2 lines
Can you perhaps share your config so that someone else could repro and take a look?
Sure. Not sure how I can share the giant config without bombarding all the recipients. Let me just send to you privately first.
Quoted 15 lines
Happy to follow up in our issue tracker, too.
https://github.com/ClangBuiltLinux/linux/issuesQuoted 11 lines
Mark them
__always_inlineso the compiler can never outline them.Signed-off-by Hongyan Xia <[email protected]>--- arch/arm64/include/asm/esr.h | 2 +- arch/arm64/include/asm/percpu.h | 2 +- arch/arm64/include/asm/preempt.h | 4 ++-- arch/arm64/include/asm/ptrace.h | 4 ++-- arch/arm64/kernel/debug-monitors.c | 2 +- include/linux/kprobes.h | 8 ++++----
Quoted 2 lines
It might not be appropriate for the changes to
kprobes.hto go through the arm64 tree, but maybe it is?
I see Will has replied, so this might be a question for Kprobe maintainers. I do wonder that, if I see this in clang-19 compiling for arm64 without any special configs, eventually it might happen for other people after moving to clang-19?
Re: [RFC PATCH 9/9] arm64/debug: Mark debug exception helpers __always_inline
In reply to Hongyan XiaMasami,
Quoted 9 lines · On Mon, 27 Jul 2026 12:25:48 +0000 Hongyan Xia <[email protected]>…
From: Hongyan Xia <[email protected]>
Static inline should be enough to actually inline functions for most compilers, but my Clang-19 somehow thinks it's better to outline them. These tiny helpers then live in normal .text sections instead of .noinstr sections, violating noinstr.
Mark them
__always_inlineso the compiler can never outline them.
Quoted 8 lines
Signed-off-by Hongyan Xia <[email protected]>--- arch/arm64/include/asm/esr.h | 2 +- arch/arm64/include/asm/percpu.h | 2 +- arch/arm64/include/asm/preempt.h | 4 ++-- arch/arm64/include/asm/ptrace.h | 4 ++-- arch/arm64/kernel/debug-monitors.c | 2 +- include/linux/kprobes.h | 8 ++++----
Are you OK with this patch? If so, can you ack it?
-- Steve
Quoted 2 lines
6 files changed, 11 insertions(+), 11 deletions(-)
[..]
Quoted 34 lines
include/linux/kprobes.h +4
-4@@ -384,17 +384,17 @@ static inline void kprobe_ftrace_kill(void) {}struct kprobe *get_kprobe (void *addr);/* kprobe_running() will just return the current_kprobe on this CPU */ -static inline struct kprobe *kprobe_running (void ) +static __always_inline struct kprobe *kprobe_running(void) {return __this_cpu_read (current_kprobe); } -static inline void reset_current_kprobe (void ) +static __always_inline void reset_current_kprobe(void) {__this_cpu_write (current_kprobe,NULL ); } -static inline struct kprobe_ctlblk *get_kprobe_ctlblk (void ) +static __always_inline struct kprobe_ctlblk *get_kprobe_ctlblk(void) {return this_cpu_ptr (&kprobe_ctlblk); }@@ -449,7 +449,7 @@ static inline struct kprobe *get_kprobe(void *addr) {return NULL ; } -static inline struct kprobe *kprobe_running (void ) +static __always_inline struct kprobe *kprobe_running(void) {return NULL ; }
Re: [RFC PATCH 9/9] arm64/debug: Mark debug exception helpers __always_inline
Quoted 25 lines · On Wed, 29 Jul 2026 14:08:46 -0400 Steven Rostedt <[email protected]> wrote:
Masami,
Quoted 9 lines · On Mon, 27 Jul 2026 12:25:48 +0000 Hongyan Xia <[email protected]>…
From: Hongyan Xia <[email protected]>
Static inline should be enough to actually inline functions for most compilers, but my Clang-19 somehow thinks it's better to outline them. These tiny helpers then live in normal .text sections instead of .noinstr sections, violating noinstr.
Mark them
__always_inlineso the compiler can never outline them.Quoted 8 lines
Signed-off-by Hongyan Xia <[email protected]>--- arch/arm64/include/asm/esr.h | 2 +- arch/arm64/include/asm/percpu.h | 2 +- arch/arm64/include/asm/preempt.h | 4 ++-- arch/arm64/include/asm/ptrace.h | 4 ++-- arch/arm64/kernel/debug-monitors.c | 2 +- include/linux/kprobes.h | 8 ++++----
Quoted 1 line
Are you OK with this patch? If so, can you ack it?
Yeah, this patch looks good to me. Anyway these are expected to be inlined.
For other patches, Sashiko noted some comments, I need to check it.
Thank you,
Quoted 43 lines
-- Steve
Quoted 2 lines
6 files changed, 11 insertions(+), 11 deletions(-)[..]
Quoted 34 lines
include/linux/kprobes.h +4
-4@@ -384,17 +384,17 @@ static inline void kprobe_ftrace_kill(void) {}struct kprobe *get_kprobe (void *addr);/* kprobe_running() will just return the current_kprobe on this CPU */ -static inline struct kprobe *kprobe_running (void ) +static __always_inline struct kprobe *kprobe_running(void) {return __this_cpu_read (current_kprobe); } -static inline void reset_current_kprobe (void ) +static __always_inline void reset_current_kprobe(void) {__this_cpu_write (current_kprobe,NULL ); } -static inline struct kprobe_ctlblk *get_kprobe_ctlblk (void ) +static __always_inline struct kprobe_ctlblk *get_kprobe_ctlblk(void) {return this_cpu_ptr (&kprobe_ctlblk); }@@ -449,7 +449,7 @@ static inline struct kprobe *get_kprobe(void *addr) {return NULL ; } -static inline struct kprobe *kprobe_running (void ) +static __always_inline struct kprobe *kprobe_running(void) {return NULL ; }
Signature · 1 line
Masami Hiramatsu (Google) <[email protected]>
Re: [RFC PATCH 9/9] arm64/debug: Mark debug exception helpers __always_inline
Quoted 30 lines · Masami Hiramatsu
Quoted 25 lines · On Wed, 29 Jul 2026 14:08:46 -0400 Steven Rostedt <[email protected]> wrote:
Masami,
Quoted 9 lines · On Mon, 27 Jul 2026 12:25:48 +0000 Hongyan Xia <[email protected]>…
From: Hongyan Xia <[email protected]>
Static inline should be enough to actually inline functions for most compilers, but my Clang-19 somehow thinks it's better to outline them. These tiny helpers then live in normal .text sections instead of .noinstr sections, violating noinstr.
Mark them
__always_inlineso the compiler can never outline them.Quoted 8 lines
Signed-off-by Hongyan Xia <[email protected]>--- arch/arm64/include/asm/esr.h | 2 +- arch/arm64/include/asm/percpu.h | 2 +- arch/arm64/include/asm/preempt.h | 4 ++-- arch/arm64/include/asm/ptrace.h | 4 ++-- arch/arm64/kernel/debug-monitors.c | 2 +- include/linux/kprobes.h | 8 ++++----Quoted 1 line
Are you OK with this patch? If so, can you ack it?
Quoted 8 lines
Yeah, this patch looks good to me. Anyway these are expected to be inlined.
For other patches, Sashiko noted some comments, I need to check it.
Thank you,
Right, I just realized this is a badly-ordered series. Patch 9/9 should be merged first regardless, otherwise noinstr in the rest of the series simply won't work because of these outlined functions.
Quoted 1 line
[...]
Mark Rutland
reviewerRe: [RFC PATCH 9/9] arm64/debug: Mark debug exception helpers __always_inline
In reply to Hongyan XiaQuoted 10 lines · Hongyan Xia
From: Hongyan Xia <[email protected]>
Static inline should be enough to actually inline functions for most compilers, but my Clang-19 somehow thinks it's better to outline them. These tiny helpers then live in normal .text sections instead of .noinstr sections, violating noinstr.
Mark them
__always_inlineso the compiler can never outline them.Signed-off-by Hongyan Xia <[email protected]>
Quoted 8 lines
--- arch/arm64/include/asm/esr.h | 2 +- arch/arm64/include/asm/percpu.h | 2 +- arch/arm64/include/asm/preempt.h | 4 ++-- arch/arm64/include/asm/ptrace.h | 4 ++-- arch/arm64/kernel/debug-monitors.c | 2 +- include/linux/kprobes.h | 8 ++++---- 6 files changed, 11 insertions(+), 11 deletions(-)
I think this should be split into separate patches. In particular, the preempt count helpers are used in mane other places than debug exceptions, and if those are used in noinstr code today, we need to backport those as fixes.
Mark.
Quoted 115 lines
arch/arm64/include/asm/esr.h +1
-1@@ -437,7 +437,7 @@#ifndef __ASSEMBLER__#include <asm/types.h> -static inline unsigned long esr_brk_comment (unsigned long esr) +static __always_inline unsigned long esr_brk_comment (unsigned long esr) {return esr & ESR_ELx_BRK64_ISS_COMMENT_MASK; }arch/arm64/include/asm/percpu.h +1
-1@@ -29,7 +29,7 @@ static inline unsigned long __hyp_my_cpu_offset(void)return read_sysreg (tpidr_el2); } -static inline unsigned long __kern_my_cpu_offset (void ) +static __always_inline unsigned long __kern_my_cpu_offset (void ) {unsigned long off;arch/arm64/include/asm/preempt.h +2
-2@@ -41,14 +41,14 @@ static inline bool test_preempt_need_resched(void)return !current_thread_info ()->preempt.need_resched; } -static inline void __preempt_count_add (int val) +static __always_inline void __preempt_count_add(int val) {u32 pc =READ_ONCE (current_thread_info ()->preempt.count); pc += val;WRITE_ONCE (current_thread_info ()->preempt.count, pc); } -static inline void __preempt_count_sub (int val) +static __always_inline void __preempt_count_sub(int val) {u32 pc =READ_ONCE (current_thread_info ()->preempt.count); pc -= val;arch/arm64/include/asm/ptrace.h +2
-2@@ -336,11 +336,11 @@ static inline unsigned long regs_get_kernel_argument(struct pt_regs *regs,struct task_struct ;int valid_user_regs (struct user_pt_regs *regs,struct task_struct *task); -static inline unsigned long instruction_pointer (struct pt_regs *regs) +static __always_inline unsigned long instruction_pointer (struct pt_regs *regs) {return regs->pc; } -static inline void instruction_pointer_set (struct pt_regs *regs, +static __always_inline void instruction_pointer_set(struct pt_regs *regs,unsigned long val) { regs->pc = val;arch/arm64/kernel/debug-monitors.c +1
-1@@ -146,7 +146,7 @@ postcore_initcall(debug_monitors_init);/* * Single step API and exception handling. */ -static void set_user_regs_spsr_ss (struct user_pt_regs *regs) +static __always_inline void set_user_regs_spsr_ss(struct user_pt_regs *regs) { regs->pstate |=DBG_SPSR_SS ; }include/linux/kprobes.h +4
-4@@ -384,17 +384,17 @@ static inline void kprobe_ftrace_kill(void) {}struct kprobe *get_kprobe (void *addr);/* kprobe_running() will just return the current_kprobe on this CPU */ -static inline struct kprobe *kprobe_running (void ) +static __always_inline struct kprobe *kprobe_running(void) {return __this_cpu_read (current_kprobe); } -static inline void reset_current_kprobe (void ) +static __always_inline void reset_current_kprobe(void) {__this_cpu_write (current_kprobe,NULL ); } -static inline struct kprobe_ctlblk *get_kprobe_ctlblk (void ) +static __always_inline struct kprobe_ctlblk *get_kprobe_ctlblk(void) {return this_cpu_ptr (&kprobe_ctlblk); }@@ -449,7 +449,7 @@ static inline struct kprobe *get_kprobe(void *addr) {return NULL ; } -static inline struct kprobe *kprobe_running (void ) +static __always_inline struct kprobe *kprobe_running(void) {return NULL ; }Signature · 1 line
2.47.3