x86/cfi: Fix FineIBT
Subsystems (4)
Peter Zijlstra
maintainerv2 cover x86/cfi: Fix FineIBT
Hi!
Alyssa reported a FineIBT issue (patch 6) which led to the discovery of a kCFI issue (patch 5) and a bunch of cleanups and enhancements (the rest).
Backports can probably suffice with just the last two.
Much thanks to Brian for the better ret_from_fork() cleanup.
Tested using llvm-16 on an Alderlake with both FineIBT and kCFI.
Also available at:
git://git.kernel.org/pub/scm/linux/kernel/git/peterz/queue.git x86/urgent
(I'm aiming for the merge window, not this cycle)
v1: https://lkml.kernel.org/r/[email protected]
--- arch/um/kernel/um_arch.c | 2 +- arch/x86/entry/entry_32.S | 54 +++++++--------------------- arch/x86/entry/entry_64.S | 35 ++++++------------ arch/x86/include/asm/alternative.h | 2 +- arch/x86/include/asm/ibt.h | 2 +- arch/x86/include/asm/nospec-branch.h | 4 +++ arch/x86/include/asm/switch_to.h | 4 ++- arch/x86/kernel/alternative.c | 69 +++++++++++++++++++++++++++++++++--- arch/x86/kernel/module.c | 2 +- arch/x86/kernel/process.c | 22 +++++++++++- 10 files changed, 120 insertions(+), 76 deletions(-)
Peter Zijlstra
maintainerPATCH v2 1/6 x86/cfi: Extend {JMP,CAKK}_NOSPEC comment
With the introduction of kCFI these helpers are no longer equivalent to C indirect calls and should be used with care.
--- arch/x86/include/asm/nospec-branch.h | 4 ++++ 1 file changed, 4 insertions(+)
arch/x86/include/asm/nospec-branch.h +4 -0
@@ -234,6 +234,10 @@* JMP_NOSPEC and CALL_NOSPEC macros can be used instead of a simple * indirect jmp/call which may be susceptible to the Spectre variant 2 * attack. +* +* NOTE: these do not take kCFI into account and are thus not comparable to C +* indirect calls, take care when using. The target of these should be an ENDBR +* instruction irrespective of kCFI. */ .macro JMP_NOSPEC reg:req#ifdef CONFIG_RETPOLINE
Peter Zijlstra
maintainerPATCH v2 2/6 x86/alternative: Rename apply_ibt_endbr()
In reply to Peter ZijlstraThe current name doesn't reflect what it does very well.
--- arch/um/kernel/um_arch.c | 2 +- arch/x86/include/asm/alternative.h | 2 +- arch/x86/include/asm/ibt.h | 2 +- arch/x86/kernel/alternative.c | 9 ++++++--- arch/x86/kernel/module.c | 2 +- 5 files changed, 10 insertions(+), 7 deletions(-)
arch/um/kernel/um_arch.c +1 -1
@@ -437,7 +437,7 @@ void __init arch_cpu_finalize_init(void)os_check_bugs (); } -void apply_ibt_endbr (s32 *start,s32 *end) +void apply_seal_endbr (s32 *start,s32 *end) { }
arch/x86/include/asm/alternative.h +1 -1
@@ -96,7 +96,7 @@ extern void alternative_instructions(voiextern void apply_alternatives (struct alt_instr *start,struct alt_instr *end);extern void apply_retpolines (s32 *start,s32 *end);extern void apply_returns (s32 *start,s32 *end); -extern void apply_ibt_endbr (s32 *start,s32 *end); +extern void apply_seal_endbr (s32 *start,s32 *end);extern void apply_fineibt (s32 *start_retpoline,s32 *end_retpoine,s32 *start_cfi,s32 *end_cfi);
arch/x86/include/asm/ibt.h +1 -1
@@ -34,7 +34,7 @@/* * Create a dummy function pointer reference to prevent objtool from marking * the function as needing to be "sealed" (i.e. ENDBR converted to NOP by -* apply_ibt_endbr()). +* apply_seal_endbr()). */ #define IBT_NOSEAL (fname) \ ".pushsection .discard.ibt_endbr_noseal\n\t" \
arch/x86/kernel/alternative.c +6 -3
@@ -803,7 +803,7 @@ static void __init_or_module poison_endb/* * Generated by: objtool --ibt */ -void __init_or_module noinline apply_ibt_endbr(s32 *start, s32 *end) +void __init_or_module noinline apply_seal_endbr(s32 *start, s32 *end) {s32 *s;
@@ -818,7 +818,7 @@ void __init_or_module noinline apply_ibt#else -void __init_or_module apply_ibt_endbr(s32 *start, s32 *end) { } +void __init_or_module apply_seal_endbr(s32 *start, s32 *end) { }#endif /* CONFIG_X86_KERNEL_IBT */
@@ -1565,7 +1565,10 @@ void __init alternative_instructions(voi*/ callthunks_patch_builtin_calls (); -apply_ibt_endbr (__ibt_endbr_seal, __ibt_endbr_seal_end); +/* +* Seal all functions that do not have their address taken. +*/ +apply_seal_endbr (__ibt_endbr_seal, __ibt_endbr_seal_end);#ifdef CONFIG_SMP /* Patch to UP if other cpus not imminent. */
arch/x86/kernel/module.c +1 -1
@@ -358,7 +358,7 @@ int module_finalize(const Elf_Ehdr *hdr, }if (ibt_endbr) {void *iseg = (void *)ibt_endbr->sh_addr; -apply_ibt_endbr (iseg, iseg + ibt_endbr->sh_size); +apply_seal_endbr (iseg, iseg + ibt_endbr->sh_size); }if (locks) {void *lseg = (void *)locks->sh_addr;
Peter Zijlstra
maintainerPATCH v2 3/6 x86/cfi: Extend ENDBR sealing to kCFI
In reply to Peter ZijlstraKees noted that IBT sealing could be extended to kCFI.
Fundamentally it is the list of functions that do not have their address taken and are thus never called indirectly. It doesn't matter that objtool uses IBT infrastructure to determine this list, once we have it it can also be used to clobber kCFI hashes and seal kCFI indirect calls.
--- arch/x86/kernel/alternative.c | 44 +++++++++++++++++++++++++++++++++++++++++- 1 file changed, 43 insertions(+), 1 deletion(-)
arch/x86/kernel/alternative.c +43 -1
@@ -778,6 +778,8 @@ void __init_or_module noinline apply_ret#ifdef CONFIG_X86_KERNEL_IBT +static void poison_cfi (void *addr); + static void __init_or_module poison_endbr(void *addr, bool warn) {u32 endbr, poison =gen_endbr_poison ();
@@ -802,6 +804,9 @@ static void __init_or_module poison_endb/* * Generated by: objtool --ibt +* +* Seal the functions for indirect calls by clobbering the ENDBR instructions +* and the kCFI hash value. */ void __init_or_module noinline apply_seal_endbr(s32 *start, s32 *end) {
@@ -812,7 +817,7 @@ void __init_or_module noinline apply_seapoison_endbr (addr, true);if (IS_ENABLED (CONFIG_FINEIBT )) -poison_endbr (addr -16 , false); +poison_cfi (addr -16 ); } }
@@ -1177,6 +1182,41 @@ static void __apply_fineibt(s32 *start_rpr_err ("Something went horribly wrong trying to rewrite the CFI implementation.\n" ); } +static inline void poison_hash (void *addr) +{ + *(u32 *)addr =0 ; +} + +static void poison_cfi (void *addr) +{ +switch (cfi_mode) { +case CFI_FINEIBT : +/* +* __cfi_\func: +* osp nopl (%rax) +* subl $0, %r10d +* jz 1f +* ud2 +* 1: nop +*/ +poison_endbr (addr, false); +poison_hash (addr + fineibt_preamble_hash); +break ; + +case CFI_KCFI : +/* +* __cfi_\func: +* movl $0, %eax +* .skip 11, 0x90 +*/ +poison_hash (addr +1 ); +break ; + +default : +break ; + } +} +#else static void __apply_fineibt(s32 *start_retpoline, s32 *end_retpoline,
@@ -1184,6 +1224,8 @@ static void __apply_fineibt(s32 *start_r { } +static void poison_cfi (void *addr) { } +#endif void apply_fineibt(s32 *start_retpoline, s32 *end_retpoline,
Peter Zijlstra
maintainerPATCH v2 4/6 x86/32: Remove schedule_tail_wrapper()
In reply to Peter ZijlstraFrom: Brian Gerst <[email protected]>
The unwinder expects a return address at the very top of the kernel stack just below pt_regs and before any stack frame is created. Instead of calling a wrapper, set up a return address as if ret_from_fork() was called from the syscall entry code.
--- arch/x86/entry/entry_32.S | 32 ++++++++++---------------------- 1 file changed, 10 insertions(+), 22 deletions(-)
arch/x86/entry/entry_32.S +10 -22
@@ -720,26 +720,6 @@ SYM_CODE_END(__switch_to_asm) .popsection /* - * The unwinder expects the last frame on the stack to always be at the same - * offset from the end of the page, which allows it to validate the stack. - * Calling schedule_tail() directly would break that convention because its an - * asmlinkage function so its argument has to be pushed on the stack. This - * wrapper creates a proper "end of stack" frame header before the call. - */ -.pushsection .text, "ax" -SYM_FUNC_START(schedule_tail_wrapper) - FRAME_BEGIN - - pushl %eax - call schedule_tail - popl %eax - - FRAME_END - RET -SYM_FUNC_END(schedule_tail_wrapper) -.popsection - -/* * A newly forked process directly context switches into this address. * * eax: prev task we switched from
@@ -748,7 +728,13 @@ SYM_FUNC_END(schedule_tail_wrapper) */ .pushsection .text, "ax" SYM_CODE_START(ret_from_fork) - call schedule_tail_wrapper + /* return address for the stack unwinder */ + pushl $.Lsyscall_32_done + FRAME_BEGIN + + pushl %eax + call schedule_tail + addl $4, %esp testl %ebx, %ebx jnz 1f /* kernel threads are uncommon */
@@ -757,7 +743,9 @@ SYM_CODE_START(ret_from_fork) /* When we fork, we trace the syscall return in the child, too. */ movl %esp, %eax call syscall_exit_to_user_mode - jmp .Lsyscall_32_done + + FRAME_END + RET /* kernel thread */ 1: movl %edi, %eax
Peter Zijlstra
maintainerPATCH v2 5/6 x86: Rewrite ret_from_fork() in C
In reply to Peter ZijlstraFrom: Brian Gerst <[email protected]>
When kCFI is enabled, special handling is needed for the indirect call to the kernel thread function. Rewrite the ret_from_fork() function in C so that the compiler can properly handle the indirect call.
--- arch/x86/entry/entry_32.S | 30 +++++++----------------------- arch/x86/entry/entry_64.S | 35 ++++++++++------------------------- arch/x86/include/asm/switch_to.h | 4 +++- arch/x86/kernel/process.c | 22 +++++++++++++++++++++- 4 files changed, 41 insertions(+), 50 deletions(-)
arch/x86/entry/entry_32.S +7 -23
@@ -727,37 +727,21 @@ SYM_CODE_END(__switch_to_asm) * edi: kernel thread arg */ .pushsection .text, "ax" -SYM_CODE_START(ret_from_fork) +SYM_CODE_START(ret_from_fork_asm) /* return address for the stack unwinder */ pushl $.Lsyscall_32_done FRAME_BEGIN - pushl %eax - call schedule_tail + /* prev already in EAX */ + movl %esp, %edx /* regs */ + movl %ebx, %ecx /* fn */ + pushl %edi /* fn_arg */ + call ret_from_fork addl $4, %esp - testl %ebx, %ebx - jnz 1f /* kernel threads are uncommon */ - -2: - /* When we fork, we trace the syscall return in the child, too. */ - movl %esp, %eax - call syscall_exit_to_user_mode - FRAME_END RET - - /* kernel thread */ -1: movl %edi, %eax - CALL_NOSPEC ebx - /* - * A kernel thread is allowed to return here after successfully - * calling kernel_execve(). Exit to userspace to complete the execve() - * syscall. - */ - movl $0, PT_EAX(%esp) - jmp 2b -SYM_CODE_END(ret_from_fork) +SYM_CODE_END(ret_from_fork_asm) .popsection SYM_ENTRY(__begin_SYSENTER_singlestep_region, SYM_L_GLOBAL, SYM_A_NONE)
arch/x86/entry/entry_64.S +10 -25
@@ -284,36 +284,21 @@ SYM_FUNC_END(__switch_to_asm) * r12: kernel thread arg */ .pushsection .text, "ax" - __FUNC_ALIGN -SYM_CODE_START_NOALIGN(ret_from_fork) +SYM_CODE_START(ret_from_fork_asm) UNWIND_HINT_END_OF_STACK ANNOTATE_NOENDBR // copy_thread CALL_DEPTH_ACCOUNT - movq %rax, %rdi - call schedule_tail /* rdi: 'prev' task parameter */ - testq %rbx, %rbx /* from kernel_thread? */ - jnz 1f /* kernel threads are uncommon */ + /* return address for the stack unwinder */ + pushq $swapgs_restore_regs_and_return_to_usermode + UNWIND_HINT_FUNC -2: - UNWIND_HINT_REGS - movq %rsp, %rdi - call syscall_exit_to_user_mode /* returns with IRQs disabled */ - jmp swapgs_restore_regs_and_return_to_usermode - -1: - /* kernel thread */ - UNWIND_HINT_END_OF_STACK - movq %r12, %rdi - CALL_NOSPEC rbx - /* - * A kernel thread is allowed to return here after successfully - * calling kernel_execve(). Exit to userspace to complete the execve() - * syscall. - */ - movq $0, RAX(%rsp) - jmp 2b -SYM_CODE_END(ret_from_fork) + movq %rax, %rdi /* prev */ + movq %rsp, %rsi /* regs */ + movq %rbx, %rdx /* fn */ + movq %r12, %rcx /* fn_arg */ + jmp ret_from_fork +SYM_CODE_END(ret_from_fork_asm) .popsection .macro DEBUG_ENTRY_ASSERT_IRQS_OFF
arch/x86/include/asm/switch_to.h +3 -1
@@ -12,7 +12,9 @@ struct task_struct *__switch_to_asm(stru __visible struct task_struct *__switch_to(struct task_struct *prev,struct task_struct *next); -asmlinkage void ret_from_fork(void); +asmlinkage void ret_from_fork_asm(void); +__visible void ret_from_fork(struct task_struct *prev, struct pt_regs *regs, +int (*fn)(void *),void *fn_arg);/* * This is the structure pointed to by thread.sp for an inactive task. The
arch/x86/kernel/process.c +21 -1
@@ -28,6 +28,7 @@#include <linux/static_call.h> #include <trace/events/power.h> #include <linux/hw_breakpoint.h> +#include <linux/entry-common.h> #include <asm/cpu.h> #include <asm/apic.h> #include <linux/uaccess.h>
@@ -134,6 +135,25 @@ static int set_new_tls(struct task_strucreturn do_set_thread_area_64 (p,ARCH_SET_FS , tls); } +__visible void ret_from_fork(struct task_struct *prev, struct pt_regs *regs, +int (*fn)(void *),void *fn_arg) +{ +schedule_tail (prev); + +/* Is this a kernel thread? */ +if (unlikely (fn)) { +fn (fn_arg); +/* +* A kernel thread is allowed to return here after successfully +* calling kernel_execve(). Exit to userspace to complete the +* execve() syscall. +*/ + regs->ax =0 ; + } + +syscall_exit_to_user_mode (regs); +} +int copy_thread (struct task_struct *p,const struct kernel_clone_args *args) {unsigned long clone_flags = args->flags;
@@ -149,7 +169,7 @@ int copy_thread(struct task_struct *p, c frame = &fork_frame->frame; frame->bp =encode_frame_pointer (childregs); - frame->ret_addr = (unsigned long ) ret_from_fork; + frame->ret_addr = (unsigned long ) ret_from_fork_asm; p->thread.sp = (unsigned long ) fork_frame; p->thread.io_bitmap =NULL ; p->thread.iopl_warn =0 ;
Peter Zijlstra
maintainerPATCH v2 6/6 x86/fineibt: Poison ENDBR at +0
In reply to Peter ZijlstraAlyssa noticed that when building the kernel with CFI_CLANG+IBT and booting on IBT enabled hardware to obtain FineIBT, the indirect functions look like:
__cfi_foo:
endbr64 subl $hash, %r10d jz 1f ud2 nop 1: foo: endbr64
This is because the compiler generates code for kCFI+IBT. In that case the caller does the hash check and will jump to +0, so there must be an ENDBR there. The compiler doesn't know about FineIBT at all; also it is possible to actually use kCFI+IBT when booting with 'cfi=kcfi' on IBT enabled hardware.
Having this second ENDBR however makes it possible to elide the CFI check. Therefore, we should poison this second ENDBR when switching to FineIBT mode.
--- arch/x86/kernel/alternative.c | 16 ++++++++++++++++ 1 file changed, 16 insertions(+)
arch/x86/kernel/alternative.c +16 -0
@@ -1063,6 +1063,17 @@ static int cfi_rewrite_preamble(s32 *stareturn 0 ; } +static void cfi_rewrite_endbr (s32 *start,s32 *end) +{ +s32 *s; + +for (s = start; s < end; s++) { +void *addr = (void *)s + *s; + +poison_endbr (addr+16 , false); + } +} +/* .retpoline_sites */ static int cfi_rand_callers (s32 *start,s32 *end) {
@@ -1157,14 +1168,19 @@ static void __apply_fineibt(s32 *start_rreturn ;case CFI_FINEIBT : +/* place the FineIBT preamble at func()-16 */ ret =cfi_rewrite_preamble (start_cfi, end_cfi);if (ret) goto err; +/* rewrite the callers to target func()-16 */ ret =cfi_rewrite_callers (start_retpoline, end_retpoline);if (ret) goto err; +/* now that nobody targets func()+0, remove ENDBR there */ +cfi_rewrite_endbr (start_cfi, end_cfi); +if (builtin)pr_info ("Using FineIBT CFI\n" );return ;
Quoted 9 lines · Peter Zijlstra
Alyssa reported a FineIBT issue (patch 6) which led to the discovery of a kCFI issue (patch 5) and a bunch of cleanups and enhancements (the rest).
Backports can probably suffice with just the last two.
Much thanks to Brian for the better
ret_from_fork()cleanup.Tested using llvm-16 on an Alderlake with both FineIBT and kCFI.
Thanks! This looks really nice. For the series:
Signature · 1 line
Kees Cook
Quoted 10 lines · Peter Zijlstra
Hi!
Alyssa reported a FineIBT issue (patch 6) which led to the discovery of a kCFI issue (patch 5) and a bunch of cleanups and enhancements (the rest).
Backports can probably suffice with just the last two.
Much thanks to Brian for the better
ret_from_fork()cleanup.
This version looks even better, thanks!
Sami