x86/ibt: FineIBT-BHI
Peter Zijlstra
maintainercover x86/ibt: FineIBT-BHI
Hi!
Respin of the FineIBT-BHI patches.
Scott has managed to get LLVM bits merged:
https://github.com/llvm/llvm-project/commit/e223485c9b38a5579991b8cebb6a200153eee245
Which prompted me to update these patches once again.
They boot and build the next kernel on my ADL when booted with: cfi=fineibt+bhi
Aside from the last two patches -- which implement the FineIBT-BHI scheme proper -- I'm planning on getting these patches merged 'soon'.
Scott, what those last two patches need, aside from a lot more testing, is a coherent writeup of how the mitigation works and ideally also a few numbers proving the performance gains are worth it.
Last version at:
https://lore.kernel.org/all/[email protected]/T/#u
Current patches:
git://git.kernel.org/pub/scm/linux/kernel/git/peterz/queue.git x86/fineibt-bhi
Patches apply on top of tip/master.
Peter Zijlstra
maintainerPATCH 1/11 objtool: Move dodgy linker warn to verbose
The lld.ld borkage is fixed in the latest llvm release (?) but will not be backported, meaning we're stuck with broken linker for a fair while.
Lets not spam all clang build logs and move warning to verbose.
--- tools/objtool/check.c | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-)
tools/objtool/check.c +1 -1
@@ -2264,7 +2264,7 @@ static int read_annotate(struct objtool_if (sec->sh.sh_entsize !=8 ) {static bool warned = false; -if (!warned) { +if (!warned && opts.verbose) {WARN ("%s: dodgy linker, sh_entsize != 8" , sec->name); warned = true; }
Peter Zijlstra
maintainerPATCH 2/11 x86/ibt: Clean up is_endbr()
In reply to Peter ZijlstraPretty much every caller of is_endbr() actually wants to test something at an address and ends up doing get_kernel_nofault(). Fold the lot into a more convenient helper.
--- arch/x86/events/core.c | 2 +- arch/x86/include/asm/ftrace.h | 16 ++-------------- arch/x86/include/asm/ibt.h | 5 +++-- arch/x86/kernel/alternative.c | 20 ++++++++++++++------ arch/x86/kernel/kprobes/core.c | 11 +---------- arch/x86/net/bpf_jit_comp.c | 4 ++-- kernel/trace/bpf_trace.c | 21 ++++----------------- 7 files changed, 27 insertions(+), 52 deletions(-)
arch/x86/events/core.c +1 -1
@@ -2857,7 +2857,7 @@ static bool is_uprobe_at_func_entry(strureturn true;/* endbr64 (64-bit only) */ -if (user_64bit_mode (regs) &&is_endbr (*(u32 *)auprobe->insn)) +if (user_64bit_mode (regs) &&is_endbr ((u32 *)auprobe->insn))return true;return false;
arch/x86/include/asm/ftrace.h +2 -14
@@ -36,21 +36,9 @@ static inline unsigned long ftrace_call_static inline unsigned long arch_ftrace_get_symaddr (unsigned long fentry_ip) { -#ifdef CONFIG_X86_KERNEL_IBT -u32 instr; - -/* We want to be extra safe in case entry ip is on the page edge, -* but otherwise we need to avoid get_kernel_nofault()'s overhead. -*/ -if ((fentry_ip & ~PAGE_MASK ) <ENDBR_INSN_SIZE ) { -if (get_kernel_nofault (instr, (u32 *)(fentry_ip -ENDBR_INSN_SIZE ))) -return fentry_ip; - }else { - instr = *(u32 *)(fentry_ip -ENDBR_INSN_SIZE ); - } -if (is_endbr (instr)) +if (is_endbr ((void *)(fentry_ip -ENDBR_INSN_SIZE ))) fentry_ip -=ENDBR_INSN_SIZE ; -#endif +return fentry_ip; }#define ftrace_get_symaddr (fentry_ip) arch_ftrace_get_symaddr(fentry_ip)
arch/x86/include/asm/ibt.h +3 -2
@@ -65,7 +65,7 @@ static __always_inline __attribute_constreturn 0x001f0f66 ;/* osp nopl (%rax) */ } -static inline bool is_endbr (u32 val) +static inline bool __is_endbr (u32 val) {if (val ==gen_endbr_poison ())return true;
@@ -74,6 +74,7 @@ static inline bool is_endbr(u32 val)return val ==gen_endbr (); } +extern __noendbr bool is_endbr(u32 *val); extern __noendbr u64 ibt_save(bool disable); extern __noendbr void ibt_restore(u64 save);
@@ -98,7 +99,7 @@ extern __noendbr void ibt_restore(u64 sa#define __noendbr -static inline bool is_endbr (u32 val) {return false; } +static inline bool is_endbr (u32 *val) {return false; }static inline u64 ibt_save (bool disable) {return 0 ; }static inline void ibt_restore (u64 save) { }
arch/x86/kernel/alternative.c +14 -6
@@ -852,16 +852,24 @@ void __init_or_module noinline apply_ret#ifdef CONFIG_X86_KERNEL_IBT +__noendbr bool is_endbr(u32 *val) +{ +u32 endbr; + +__get_kernel_nofault (&endbr, val, u32, Efault); +return __is_endbr (endbr); + +Efault: +return false; +} +static void poison_cfi (void *addr); static void __init_or_module poison_endbr(void *addr, bool warn) { -u32 endbr, poison =gen_endbr_poison (); - -if (WARN_ON_ONCE (get_kernel_nofault (endbr, addr))) -return ; +u32 poison =gen_endbr_poison (); -if (!is_endbr (endbr)) { +if (!is_endbr (addr)) {WARN_ON_ONCE (warn);return ; }
@@ -984,7 +992,7 @@ static u32 cfi_seed __ro_after_init;static u32 cfi_rehash (u32 hash) { hash ^= cfi_seed; -while (unlikely (is_endbr (hash) ||is_endbr (-hash))) { +while (unlikely (__is_endbr (hash) ||__is_endbr (-hash))) {bool lsb = hash &1 ; hash >>=1 ;if (lsb)
arch/x86/kernel/kprobes/core.c +1 -10
@@ -373,16 +373,7 @@ static bool can_probe(unsigned long paddkprobe_opcode_t *arch_adjust_kprobe_addr (unsigned long addr,unsigned long offset,bool *on_func_entry) { -u32 insn; - -/* -* Since 'addr' is not guaranteed to be safe to access, use -* copy_from_kernel_nofault() to read the instruction: -*/ -if (copy_from_kernel_nofault (&insn, (void *)addr,sizeof (u32))) -return NULL ; - -if (is_endbr (insn)) { +if (is_endbr ((u32 *)addr)) { *on_func_entry = !offset || offset ==4 ;if (*on_func_entry) offset =4 ;
arch/x86/net/bpf_jit_comp.c +2 -2
@@ -641,7 +641,7 @@ int bpf_arch_text_poke(void *ip, enum bp* See emit_prologue(), for IBT builds the trampoline hook is preceded * with an ENDBR instruction. */ -if (is_endbr (*(u32 *)ip)) +if (is_endbr (ip)) ip +=ENDBR_INSN_SIZE ;return __bpf_arch_text_poke (ip, t, old_addr, new_addr);
@@ -3036,7 +3036,7 @@ static int __arch_prepare_bpf_trampoline/* skip patched call instruction and point orig_call to actual * body of the kernel function. */ -if (is_endbr (*(u32 *)orig_call)) +if (is_endbr (orig_call)) orig_call +=ENDBR_INSN_SIZE ; orig_call +=X86_PATCH_SIZE ; }
kernel/trace/bpf_trace.c +4 -17
@@ -1038,27 +1038,14 @@ static const struct bpf_func_proto bpf_g .arg1_type = ARG_PTR_TO_CTX, }; -#ifdef CONFIG_X86_KERNEL_IBT -static unsigned long get_entry_ip (unsigned long fentry_ip) +static inline unsigned long get_entry_ip (unsigned long fentry_ip) { -u32 instr; - -/* We want to be extra safe in case entry ip is on the page edge, -* but otherwise we need to avoid get_kernel_nofault()'s overhead. -*/ -if ((fentry_ip & ~PAGE_MASK ) <ENDBR_INSN_SIZE ) { -if (get_kernel_nofault (instr, (u32 *)(fentry_ip -ENDBR_INSN_SIZE ))) -return fentry_ip; - }else { - instr = *(u32 *)(fentry_ip -ENDBR_INSN_SIZE ); - } -if (is_endbr (instr)) +#ifdef CONFIG_X86_KERNEL_IBT +if (is_endbr ((void *)(fentry_ip -ENDBR_INSN_SIZE ))) fentry_ip -=ENDBR_INSN_SIZE ; +#endif return fentry_ip; } -#else -#define get_entry_ip (fentry_ip) fentry_ip -#endif BPF_CALL_1(bpf_get_func_ip_kprobe, struct pt_regs *, regs) {
[tip: x86/core] x86/ibt: Clean up is_endbr()
The following commit has been merged into the x86/core branch of tip:
The following commit has been merged into the x86/core branch of tip:
Commit-ID: 72e213a7ccf9dc78a85eecee8dc8170762ed876c Gitweb: https://git.kernel.org/tip/72e213a7ccf9dc78a85eecee8dc8170762ed876c Author: Peter Zijlstra <[email protected]> AuthorDate: Fri, 07 Feb 2025 13:15:31 +01:00 Committer: Peter Zijlstra <[email protected]> CommitterDate: Fri, 14 Feb 2025 10:32:04 +01:00
x86/ibt: Clean up is_endbr()
Pretty much every caller of is_endbr() actually wants to test something at an address and ends up doing get_kernel_nofault(). Fold the lot into a more convenient helper.
--- arch/x86/events/core.c | 2 +- arch/x86/include/asm/ftrace.h | 16 ++-------------- arch/x86/include/asm/ibt.h | 5 +++-- arch/x86/kernel/alternative.c | 20 ++++++++++++++------ arch/x86/kernel/kprobes/core.c | 11 +---------- arch/x86/net/bpf_jit_comp.c | 4 ++-- kernel/trace/bpf_trace.c | 21 ++++----------------- 7 files changed, 27 insertions(+), 52 deletions(-)
arch/x86/events/core.c +1 -1
@@ -2844,7 +2844,7 @@ static bool is_uprobe_at_func_entry(struct pt_regs *regs)return true;/* endbr64 (64-bit only) */ -if (user_64bit_mode (regs) &&is_endbr (*(u32 *)auprobe->insn)) +if (user_64bit_mode (regs) &&is_endbr ((u32 *)auprobe->insn))return true;return false;
arch/x86/include/asm/ftrace.h +2 -14
@@ -36,21 +36,9 @@ static inline unsigned long ftrace_call_adjust(unsigned long addr)static inline unsigned long arch_ftrace_get_symaddr (unsigned long fentry_ip) { -#ifdef CONFIG_X86_KERNEL_IBT -u32 instr; - -/* We want to be extra safe in case entry ip is on the page edge, -* but otherwise we need to avoid get_kernel_nofault()'s overhead. -*/ -if ((fentry_ip & ~PAGE_MASK ) <ENDBR_INSN_SIZE ) { -if (get_kernel_nofault (instr, (u32 *)(fentry_ip -ENDBR_INSN_SIZE ))) -return fentry_ip; - }else { - instr = *(u32 *)(fentry_ip -ENDBR_INSN_SIZE ); - } -if (is_endbr (instr)) +if (is_endbr ((void *)(fentry_ip -ENDBR_INSN_SIZE ))) fentry_ip -=ENDBR_INSN_SIZE ; -#endif +return fentry_ip; }#define ftrace_get_symaddr (fentry_ip) arch_ftrace_get_symaddr(fentry_ip)
arch/x86/include/asm/ibt.h +3 -2
@@ -65,7 +65,7 @@ static inline __attribute_const__ u32 gen_endbr_poison(void)return 0x001f0f66 ;/* osp nopl (%rax) */ } -static inline bool is_endbr (u32 val) +static inline bool __is_endbr (u32 val) {if (val ==gen_endbr_poison ())return true;
@@ -74,6 +74,7 @@ static inline bool is_endbr(u32 val)return val ==gen_endbr (); } +extern __noendbr bool is_endbr(u32 *val); extern __noendbr u64 ibt_save(bool disable); extern __noendbr void ibt_restore(u64 save);
@@ -98,7 +99,7 @@ extern __noendbr void ibt_restore(u64 save);#define __noendbr -static inline bool is_endbr (u32 val) {return false; } +static inline bool is_endbr (u32 *val) {return false; }static inline u64 ibt_save (bool disable) {return 0 ; }static inline void ibt_restore (u64 save) { }
arch/x86/kernel/alternative.c +14 -6
@@ -852,16 +852,24 @@ void __init_or_module noinline apply_returns(s32 *start, s32 *end) { }#ifdef CONFIG_X86_KERNEL_IBT +__noendbr bool is_endbr(u32 *val) +{ +u32 endbr; + +__get_kernel_nofault (&endbr, val, u32, Efault); +return __is_endbr (endbr); + +Efault: +return false; +} +static void poison_cfi (void *addr); static void __init_or_module poison_endbr(void *addr, bool warn) { -u32 endbr, poison =gen_endbr_poison (); - -if (WARN_ON_ONCE (get_kernel_nofault (endbr, addr))) -return ; +u32 poison =gen_endbr_poison (); -if (!is_endbr (endbr)) { +if (!is_endbr (addr)) {WARN_ON_ONCE (warn);return ; }
@@ -988,7 +996,7 @@ static u32 cfi_seed __ro_after_init;static u32 cfi_rehash (u32 hash) { hash ^= cfi_seed; -while (unlikely (is_endbr (hash) ||is_endbr (-hash))) { +while (unlikely (__is_endbr (hash) ||__is_endbr (-hash))) {bool lsb = hash &1 ; hash >>=1 ;if (lsb)
arch/x86/kernel/kprobes/core.c +1 -10
@@ -373,16 +373,7 @@ out:kprobe_opcode_t *arch_adjust_kprobe_addr (unsigned long addr,unsigned long offset,bool *on_func_entry) { -u32 insn; - -/* -* Since 'addr' is not guaranteed to be safe to access, use -* copy_from_kernel_nofault() to read the instruction: -*/ -if (copy_from_kernel_nofault (&insn, (void *)addr,sizeof (u32))) -return NULL ; - -if (is_endbr (insn)) { +if (is_endbr ((u32 *)addr)) { *on_func_entry = !offset || offset ==4 ;if (*on_func_entry) offset =4 ;
arch/x86/net/bpf_jit_comp.c +2 -2
@@ -641,7 +641,7 @@ int bpf_arch_text_poke(void *ip, enum bpf_text_poke_type t,* See emit_prologue(), for IBT builds the trampoline hook is preceded * with an ENDBR instruction. */ -if (is_endbr (*(u32 *)ip)) +if (is_endbr (ip)) ip +=ENDBR_INSN_SIZE ;return __bpf_arch_text_poke (ip, t, old_addr, new_addr);
@@ -3036,7 +3036,7 @@ static int __arch_prepare_bpf_trampoline(struct bpf_tramp_image *im, void *rw_im/* skip patched call instruction and point orig_call to actual * body of the kernel function. */ -if (is_endbr (*(u32 *)orig_call)) +if (is_endbr (orig_call)) orig_call +=ENDBR_INSN_SIZE ; orig_call +=X86_PATCH_SIZE ; }
kernel/trace/bpf_trace.c +4 -17
@@ -1038,27 +1038,14 @@ static const struct bpf_func_proto bpf_get_func_ip_proto_tracing = { .arg1_type = ARG_PTR_TO_CTX, }; -#ifdef CONFIG_X86_KERNEL_IBT -static unsigned long get_entry_ip (unsigned long fentry_ip) +static inline unsigned long get_entry_ip (unsigned long fentry_ip) { -u32 instr; - -/* We want to be extra safe in case entry ip is on the page edge, -* but otherwise we need to avoid get_kernel_nofault()'s overhead. -*/ -if ((fentry_ip & ~PAGE_MASK ) <ENDBR_INSN_SIZE ) { -if (get_kernel_nofault (instr, (u32 *)(fentry_ip -ENDBR_INSN_SIZE ))) -return fentry_ip; - }else { - instr = *(u32 *)(fentry_ip -ENDBR_INSN_SIZE ); - } -if (is_endbr (instr)) +#ifdef CONFIG_X86_KERNEL_IBT +if (is_endbr ((void *)(fentry_ip -ENDBR_INSN_SIZE ))) fentry_ip -=ENDBR_INSN_SIZE ; +#endif return fentry_ip; } -#else -#define get_entry_ip (fentry_ip) fentry_ip -#endif BPF_CALL_1(bpf_get_func_ip_kprobe, struct pt_regs *, regs) {
Peter Zijlstra
maintainerPATCH 3/11 x86,kcfi: Fix EXPORT_SYMBOL vs kCFI
In reply to Peter ZijlstraThe expectation is that all EXPORT'ed symbols are free to have their address taken and called indirectly. The majority of the assembly defined functions currently violate this expectation.
Make then all use SYM_TYPED_FUNC_START() in order to emit the proper kCFI preamble.
--- arch/x86/crypto/camellia-aesni-avx-asm_64.S | 7 ++++--- arch/x86/crypto/camellia-aesni-avx2-asm_64.S | 1 + arch/x86/crypto/camellia-x86_64-asm_64.S | 9 +++++---- arch/x86/crypto/serpent-avx-x86_64-asm_64.S | 7 ++++--- arch/x86/crypto/twofish-x86_64-asm_64-3way.S | 5 +++-- arch/x86/crypto/twofish-x86_64-asm_64.S | 5 +++-- arch/x86/entry/entry.S | 3 ++- arch/x86/entry/entry_64.S | 5 +++-- arch/x86/lib/clear_page_64.S | 9 +++++---- arch/x86/lib/copy_page_64.S | 3 ++- arch/x86/lib/copy_user_64.S | 3 ++- arch/x86/lib/copy_user_uncached_64.S | 3 ++- arch/x86/lib/getuser.S | 17 +++++++++-------- arch/x86/lib/hweight.S | 5 +++-- arch/x86/lib/memmove_64.S | 3 ++- arch/x86/lib/memset_64.S | 3 ++- arch/x86/lib/msr-reg.S | 3 ++- arch/x86/lib/putuser.S | 17 +++++++++-------- 18 files changed, 63 insertions(+), 45 deletions(-)
arch/x86/crypto/camellia-aesni-avx-asm_64.S +4 -3
@@ -16,6 +16,7 @@ */ #include <linux/linkage.h> +#include <linux/cfi_types.h> #include <asm/frame.h> #define CAMELLIA_TABLE_BYTE_LEN 272
@@ -882,7 +883,7 @@ SYM_FUNC_START_LOCAL(__camellia_dec_blk1 jmp .Ldec_max24; SYM_FUNC_END(__camellia_dec_blk16) -SYM_FUNC_START(camellia_ecb_enc_16way) +SYM_TYPED_FUNC_START(camellia_ecb_enc_16way) /* input: * %rdi: ctx, CTX * %rsi: dst (16 blocks)
@@ -907,7 +908,7 @@ SYM_FUNC_START(camellia_ecb_enc_16way) RET; SYM_FUNC_END(camellia_ecb_enc_16way) -SYM_FUNC_START(camellia_ecb_dec_16way) +SYM_TYPED_FUNC_START(camellia_ecb_dec_16way) /* input: * %rdi: ctx, CTX * %rsi: dst (16 blocks)
@@ -937,7 +938,7 @@ SYM_FUNC_START(camellia_ecb_dec_16way) RET; SYM_FUNC_END(camellia_ecb_dec_16way) -SYM_FUNC_START(camellia_cbc_dec_16way) +SYM_TYPED_FUNC_START(camellia_cbc_dec_16way) /* input: * %rdi: ctx, CTX * %rsi: dst (16 blocks)
arch/x86/crypto/camellia-aesni-avx2-asm_64.S +1 -0
@@ -6,6 +6,7 @@ */ #include <linux/linkage.h> +#include <linux/cfi_types.h> #include <asm/frame.h> #define CAMELLIA_TABLE_BYTE_LEN 272
arch/x86/crypto/camellia-x86_64-asm_64.S +5 -4
@@ -6,6 +6,7 @@ */ #include <linux/linkage.h> +#include <linux/cfi_types.h> .file "camellia-x86_64-asm_64.S" .text
@@ -177,7 +178,7 @@ bswapq RAB0; \ movq RAB0, 4*2(RIO); -SYM_FUNC_START(__camellia_enc_blk) +SYM_TYPED_FUNC_START(__camellia_enc_blk) /* input: * %rdi: ctx, CTX * %rsi: dst
@@ -224,7 +225,7 @@ SYM_FUNC_START(__camellia_enc_blk) RET; SYM_FUNC_END(__camellia_enc_blk) -SYM_FUNC_START(camellia_dec_blk) +SYM_TYPED_FUNC_START(camellia_dec_blk) /* input: * %rdi: ctx, CTX * %rsi: dst
@@ -411,7 +412,7 @@ SYM_FUNC_END(camellia_dec_blk) bswapq RAB1; \ movq RAB1, 12*2(RIO); -SYM_FUNC_START(__camellia_enc_blk_2way) +SYM_TYPED_FUNC_START(__camellia_enc_blk_2way) /* input: * %rdi: ctx, CTX * %rsi: dst
@@ -460,7 +461,7 @@ SYM_FUNC_START(__camellia_enc_blk_2way) RET; SYM_FUNC_END(__camellia_enc_blk_2way) -SYM_FUNC_START(camellia_dec_blk_2way) +SYM_TYPED_FUNC_START(camellia_dec_blk_2way) /* input: * %rdi: ctx, CTX * %rsi: dst
arch/x86/crypto/serpent-avx-x86_64-asm_64.S +4 -3
@@ -9,6 +9,7 @@ */ #include <linux/linkage.h> +#include <linux/cfi_types.h> #include <asm/frame.h> #include "glue_helper-asm-avx.S"
@@ -656,7 +657,7 @@ SYM_FUNC_START_LOCAL(__serpent_dec_blk8_ RET; SYM_FUNC_END(__serpent_dec_blk8_avx) -SYM_FUNC_START(serpent_ecb_enc_8way_avx) +SYM_TYPED_FUNC_START(serpent_ecb_enc_8way_avx) /* input: * %rdi: ctx, CTX * %rsi: dst
@@ -674,7 +675,7 @@ SYM_FUNC_START(serpent_ecb_enc_8way_avx) RET; SYM_FUNC_END(serpent_ecb_enc_8way_avx) -SYM_FUNC_START(serpent_ecb_dec_8way_avx) +SYM_TYPED_FUNC_START(serpent_ecb_dec_8way_avx) /* input: * %rdi: ctx, CTX * %rsi: dst
@@ -692,7 +693,7 @@ SYM_FUNC_START(serpent_ecb_dec_8way_avx) RET; SYM_FUNC_END(serpent_ecb_dec_8way_avx) -SYM_FUNC_START(serpent_cbc_dec_8way_avx) +SYM_TYPED_FUNC_START(serpent_cbc_dec_8way_avx) /* input: * %rdi: ctx, CTX * %rsi: dst
arch/x86/crypto/twofish-x86_64-asm_64-3way.S +3 -2
@@ -6,6 +6,7 @@ */ #include <linux/linkage.h> +#include <linux/cfi_types.h> .file "twofish-x86_64-asm-3way.S" .text
@@ -220,7 +221,7 @@ rorq $32, RAB2; \ outunpack3(mov, RIO, 2, RAB, 2); -SYM_FUNC_START(__twofish_enc_blk_3way) +SYM_TYPED_FUNC_START(__twofish_enc_blk_3way) /* input: * %rdi: ctx, CTX * %rsi: dst
@@ -269,7 +270,7 @@ SYM_FUNC_START(__twofish_enc_blk_3way) RET; SYM_FUNC_END(__twofish_enc_blk_3way) -SYM_FUNC_START(twofish_dec_blk_3way) +SYM_TYPED_FUNC_START(twofish_dec_blk_3way) /* input: * %rdi: ctx, CTX * %rsi: dst
arch/x86/crypto/twofish-x86_64-asm_64.S +3 -2
@@ -8,6 +8,7 @@ .text #include <linux/linkage.h> +#include <linux/cfi_types.h> #include <asm/asm-offsets.h> #define a_offset 0
@@ -202,7 +203,7 @@ xor %r8d, d ## D;\ ror $1, d ## D; -SYM_FUNC_START(twofish_enc_blk) +SYM_TYPED_FUNC_START(twofish_enc_blk) pushq R1 /* %rdi contains the ctx address */
@@ -255,7 +256,7 @@ SYM_FUNC_START(twofish_enc_blk) RET SYM_FUNC_END(twofish_enc_blk) -SYM_FUNC_START(twofish_dec_blk) +SYM_TYPED_FUNC_START(twofish_dec_blk) pushq R1 /* %rdi contains the ctx address */
arch/x86/lib/clear_page_64.S +4 -3
@@ -1,6 +1,7 @@ /* SPDX-License-Identifier: GPL-2.0-only */ #include <linux/export.h> #include <linux/linkage.h> +#include <linux/cfi_types.h> #include <asm/asm.h> /*
@@ -14,7 +15,7 @@ * Zero a page. * %rdi - page */ -SYM_FUNC_START(clear_page_rep) +SYM_TYPED_FUNC_START(clear_page_rep) movl $4096/8,%ecx xorl %eax,%eax rep stosq
@@ -22,7 +23,7 @@ SYM_FUNC_START(clear_page_rep) SYM_FUNC_END(clear_page_rep) EXPORT_SYMBOL_GPL(clear_page_rep) -SYM_FUNC_START(clear_page_orig) +SYM_TYPED_FUNC_START(clear_page_orig) xorl %eax,%eax movl $4096/64,%ecx .p2align 4
@@ -44,7 +45,7 @@ SYM_FUNC_START(clear_page_orig) SYM_FUNC_END(clear_page_orig) EXPORT_SYMBOL_GPL(clear_page_orig) -SYM_FUNC_START(clear_page_erms) +SYM_TYPED_FUNC_START(clear_page_erms) movl $4096,%ecx xorl %eax,%eax rep stosb
arch/x86/lib/copy_page_64.S +2 -1
@@ -3,6 +3,7 @@ #include <linux/export.h> #include <linux/linkage.h> +#include <linux/cfi_types.h> #include <asm/cpufeatures.h> #include <asm/alternative.h>
@@ -13,7 +14,7 @@ * prefetch distance based on SMP/UP. */ ALIGN -SYM_FUNC_START(copy_page) +SYM_TYPED_FUNC_START(copy_page) ALTERNATIVE "jmp copy_page_regs", "", X86_FEATURE_REP_GOOD movl $4096/8, %ecx rep movsq
arch/x86/lib/memmove_64.S +2 -1
@@ -8,6 +8,7 @@ */ #include <linux/export.h> #include <linux/linkage.h> +#include <linux/cfi_types.h> #include <asm/cpufeatures.h> #include <asm/alternative.h>
@@ -26,7 +27,7 @@ * Output: * rax: dest */ -SYM_FUNC_START(__memmove) +SYM_TYPED_FUNC_START(__memmove) mov %rdi, %rax
arch/x86/lib/memset_64.S +2 -1
@@ -3,6 +3,7 @@ #include <linux/export.h> #include <linux/linkage.h> +#include <linux/cfi_types.h> #include <asm/cpufeatures.h> #include <asm/alternative.h>
@@ -28,7 +29,7 @@ * only for the return value that is the same as the source input, * which the compiler could/should do much better anyway. */ -SYM_FUNC_START(__memset) +SYM_TYPED_FUNC_START(__memset) ALTERNATIVE "jmp memset_orig", "", X86_FEATURE_FSRS movq %rdi,%r9
arch/x86/lib/msr-reg.S +2 -1
@@ -1,6 +1,7 @@ /* SPDX-License-Identifier: GPL-2.0 */ #include <linux/linkage.h> #include <linux/errno.h> +#include <linux/cfi_types.h> #include <asm/asm.h> #include <asm/msr.h>
@@ -12,7 +13,7 @@ * */ .macro op_safe_regs op -SYM_FUNC_START(\op\()_safe_regs) +SYM_TYPED_FUNC_START(\op\()_safe_regs) pushq %rbx pushq %r12 movq %rdi, %r10 /* Save pointer */
[tip: x86/core] x86,kcfi: Fix EXPORT_SYMBOL vs kCFI
The following commit has been merged into the x86/core branch of tip:
The following commit has been merged into the x86/core branch of tip:
Commit-ID: 2981557cb0408e142480bc1eea30558cf5a2382f Gitweb: https://git.kernel.org/tip/2981557cb0408e142480bc1eea30558cf5a2382f Author: Peter Zijlstra <[email protected]> AuthorDate: Fri, 07 Feb 2025 13:15:32 +01:00 Committer: Peter Zijlstra <[email protected]> CommitterDate: Fri, 14 Feb 2025 10:32:05 +01:00
x86,kcfi: Fix EXPORT_SYMBOL vs kCFI
The expectation is that all EXPORT'ed symbols are free to have their address taken and called indirectly. The majority of the assembly defined functions currently violate this expectation.
Make then all use SYM_TYPED_FUNC_START() in order to emit the proper kCFI preamble.
--- arch/x86/crypto/camellia-aesni-avx-asm_64.S | 7 ++++--- arch/x86/crypto/camellia-aesni-avx2-asm_64.S | 1 + arch/x86/crypto/camellia-x86_64-asm_64.S | 9 +++++---- arch/x86/crypto/serpent-avx-x86_64-asm_64.S | 7 ++++--- arch/x86/crypto/twofish-x86_64-asm_64-3way.S | 5 +++-- arch/x86/crypto/twofish-x86_64-asm_64.S | 5 +++-- arch/x86/lib/clear_page_64.S | 7 ++++--- arch/x86/lib/copy_page_64.S | 3 ++- arch/x86/lib/memmove_64.S | 3 ++- arch/x86/lib/memset_64.S | 3 ++- arch/x86/lib/msr-reg.S | 3 ++- 11 files changed, 32 insertions(+), 21 deletions(-)
arch/x86/crypto/camellia-aesni-avx-asm_64.S +4 -3
@@ -16,6 +16,7 @@ */ #include <linux/linkage.h> +#include <linux/cfi_types.h> #include <asm/frame.h> #define CAMELLIA_TABLE_BYTE_LEN 272
@@ -882,7 +883,7 @@ SYM_FUNC_START_LOCAL(__camellia_dec_blk16) jmp .Ldec_max24; SYM_FUNC_END(__camellia_dec_blk16) -SYM_FUNC_START(camellia_ecb_enc_16way) +SYM_TYPED_FUNC_START(camellia_ecb_enc_16way) /* input: * %rdi: ctx, CTX * %rsi: dst (16 blocks)
@@ -907,7 +908,7 @@ SYM_FUNC_START(camellia_ecb_enc_16way) RET; SYM_FUNC_END(camellia_ecb_enc_16way) -SYM_FUNC_START(camellia_ecb_dec_16way) +SYM_TYPED_FUNC_START(camellia_ecb_dec_16way) /* input: * %rdi: ctx, CTX * %rsi: dst (16 blocks)
@@ -937,7 +938,7 @@ SYM_FUNC_START(camellia_ecb_dec_16way) RET; SYM_FUNC_END(camellia_ecb_dec_16way) -SYM_FUNC_START(camellia_cbc_dec_16way) +SYM_TYPED_FUNC_START(camellia_cbc_dec_16way) /* input: * %rdi: ctx, CTX * %rsi: dst (16 blocks)
arch/x86/crypto/camellia-aesni-avx2-asm_64.S +1 -0
@@ -6,6 +6,7 @@ */ #include <linux/linkage.h> +#include <linux/cfi_types.h> #include <asm/frame.h> #define CAMELLIA_TABLE_BYTE_LEN 272
arch/x86/crypto/camellia-x86_64-asm_64.S +5 -4
@@ -6,6 +6,7 @@ */ #include <linux/linkage.h> +#include <linux/cfi_types.h> .file "camellia-x86_64-asm_64.S" .text
@@ -177,7 +178,7 @@ bswapq RAB0; \ movq RAB0, 4*2(RIO); -SYM_FUNC_START(__camellia_enc_blk) +SYM_TYPED_FUNC_START(__camellia_enc_blk) /* input: * %rdi: ctx, CTX * %rsi: dst
@@ -224,7 +225,7 @@ SYM_FUNC_START(__camellia_enc_blk) RET; SYM_FUNC_END(__camellia_enc_blk) -SYM_FUNC_START(camellia_dec_blk) +SYM_TYPED_FUNC_START(camellia_dec_blk) /* input: * %rdi: ctx, CTX * %rsi: dst
@@ -411,7 +412,7 @@ SYM_FUNC_END(camellia_dec_blk) bswapq RAB1; \ movq RAB1, 12*2(RIO); -SYM_FUNC_START(__camellia_enc_blk_2way) +SYM_TYPED_FUNC_START(__camellia_enc_blk_2way) /* input: * %rdi: ctx, CTX * %rsi: dst
@@ -460,7 +461,7 @@ SYM_FUNC_START(__camellia_enc_blk_2way) RET; SYM_FUNC_END(__camellia_enc_blk_2way) -SYM_FUNC_START(camellia_dec_blk_2way) +SYM_TYPED_FUNC_START(camellia_dec_blk_2way) /* input: * %rdi: ctx, CTX * %rsi: dst
arch/x86/crypto/serpent-avx-x86_64-asm_64.S +4 -3
@@ -9,6 +9,7 @@ */ #include <linux/linkage.h> +#include <linux/cfi_types.h> #include <asm/frame.h> #include "glue_helper-asm-avx.S"
@@ -656,7 +657,7 @@ SYM_FUNC_START_LOCAL(__serpent_dec_blk8_avx) RET; SYM_FUNC_END(__serpent_dec_blk8_avx) -SYM_FUNC_START(serpent_ecb_enc_8way_avx) +SYM_TYPED_FUNC_START(serpent_ecb_enc_8way_avx) /* input: * %rdi: ctx, CTX * %rsi: dst
@@ -674,7 +675,7 @@ SYM_FUNC_START(serpent_ecb_enc_8way_avx) RET; SYM_FUNC_END(serpent_ecb_enc_8way_avx) -SYM_FUNC_START(serpent_ecb_dec_8way_avx) +SYM_TYPED_FUNC_START(serpent_ecb_dec_8way_avx) /* input: * %rdi: ctx, CTX * %rsi: dst
@@ -692,7 +693,7 @@ SYM_FUNC_START(serpent_ecb_dec_8way_avx) RET; SYM_FUNC_END(serpent_ecb_dec_8way_avx) -SYM_FUNC_START(serpent_cbc_dec_8way_avx) +SYM_TYPED_FUNC_START(serpent_cbc_dec_8way_avx) /* input: * %rdi: ctx, CTX * %rsi: dst
arch/x86/crypto/twofish-x86_64-asm_64-3way.S +3 -2
@@ -6,6 +6,7 @@ */ #include <linux/linkage.h> +#include <linux/cfi_types.h> .file "twofish-x86_64-asm-3way.S" .text
@@ -220,7 +221,7 @@ rorq $32, RAB2; \ outunpack3(mov, RIO, 2, RAB, 2); -SYM_FUNC_START(__twofish_enc_blk_3way) +SYM_TYPED_FUNC_START(__twofish_enc_blk_3way) /* input: * %rdi: ctx, CTX * %rsi: dst
@@ -269,7 +270,7 @@ SYM_FUNC_START(__twofish_enc_blk_3way) RET; SYM_FUNC_END(__twofish_enc_blk_3way) -SYM_FUNC_START(twofish_dec_blk_3way) +SYM_TYPED_FUNC_START(twofish_dec_blk_3way) /* input: * %rdi: ctx, CTX * %rsi: dst
arch/x86/crypto/twofish-x86_64-asm_64.S +3 -2
@@ -8,6 +8,7 @@ .text #include <linux/linkage.h> +#include <linux/cfi_types.h> #include <asm/asm-offsets.h> #define a_offset 0
@@ -202,7 +203,7 @@ xor %r8d, d ## D;\ ror $1, d ## D; -SYM_FUNC_START(twofish_enc_blk) +SYM_TYPED_FUNC_START(twofish_enc_blk) pushq R1 /* %rdi contains the ctx address */
@@ -255,7 +256,7 @@ SYM_FUNC_START(twofish_enc_blk) RET SYM_FUNC_END(twofish_enc_blk) -SYM_FUNC_START(twofish_dec_blk) +SYM_TYPED_FUNC_START(twofish_dec_blk) pushq R1 /* %rdi contains the ctx address */
arch/x86/lib/clear_page_64.S +4 -3
@@ -1,6 +1,7 @@ /* SPDX-License-Identifier: GPL-2.0-only */ #include <linux/export.h> #include <linux/linkage.h> +#include <linux/cfi_types.h> #include <asm/asm.h> /*
@@ -14,7 +15,7 @@ * Zero a page. * %rdi - page */ -SYM_FUNC_START(clear_page_rep) +SYM_TYPED_FUNC_START(clear_page_rep) movl $4096/8,%ecx xorl %eax,%eax rep stosq
@@ -22,7 +23,7 @@ SYM_FUNC_START(clear_page_rep) SYM_FUNC_END(clear_page_rep) EXPORT_SYMBOL_GPL(clear_page_rep) -SYM_FUNC_START(clear_page_orig) +SYM_TYPED_FUNC_START(clear_page_orig) xorl %eax,%eax movl $4096/64,%ecx .p2align 4
@@ -44,7 +45,7 @@ SYM_FUNC_START(clear_page_orig) SYM_FUNC_END(clear_page_orig) EXPORT_SYMBOL_GPL(clear_page_orig) -SYM_FUNC_START(clear_page_erms) +SYM_TYPED_FUNC_START(clear_page_erms) movl $4096,%ecx xorl %eax,%eax rep stosb
arch/x86/lib/copy_page_64.S +2 -1
@@ -3,6 +3,7 @@ #include <linux/export.h> #include <linux/linkage.h> +#include <linux/cfi_types.h> #include <asm/cpufeatures.h> #include <asm/alternative.h>
@@ -13,7 +14,7 @@ * prefetch distance based on SMP/UP. */ ALIGN -SYM_FUNC_START(copy_page) +SYM_TYPED_FUNC_START(copy_page) ALTERNATIVE "jmp copy_page_regs", "", X86_FEATURE_REP_GOOD movl $4096/8, %ecx rep movsq
arch/x86/lib/memmove_64.S +2 -1
@@ -8,6 +8,7 @@ */ #include <linux/export.h> #include <linux/linkage.h> +#include <linux/cfi_types.h> #include <asm/cpufeatures.h> #include <asm/alternative.h>
@@ -26,7 +27,7 @@ * Output: * rax: dest */ -SYM_FUNC_START(__memmove) +SYM_TYPED_FUNC_START(__memmove) mov %rdi, %rax
arch/x86/lib/memset_64.S +2 -1
@@ -3,6 +3,7 @@ #include <linux/export.h> #include <linux/linkage.h> +#include <linux/cfi_types.h> #include <asm/cpufeatures.h> #include <asm/alternative.h>
@@ -28,7 +29,7 @@ * only for the return value that is the same as the source input, * which the compiler could/should do much better anyway. */ -SYM_FUNC_START(__memset) +SYM_TYPED_FUNC_START(__memset) ALTERNATIVE "jmp memset_orig", "", X86_FEATURE_FSRS movq %rdi,%r9
arch/x86/lib/msr-reg.S +2 -1
@@ -1,6 +1,7 @@ /* SPDX-License-Identifier: GPL-2.0 */ #include <linux/linkage.h> #include <linux/errno.h> +#include <linux/cfi_types.h> #include <asm/asm.h> #include <asm/msr.h>
@@ -12,7 +13,7 @@ * */ .macro op_safe_regs op -SYM_FUNC_START(\op\()_safe_regs) +SYM_TYPED_FUNC_START(\op\()_safe_regs) pushq %rbx pushq %r12 movq %rdi, %r10 /* Save pointer */
Peter Zijlstra
maintainerPATCH 4/11 x86/cfi: Clean up linkage
In reply to Peter ZijlstraWith the introduction of kCFI the addition of ENDBR to SYM_FUNC_START* no longer suffices to make the function indirectly callable. This now requires the use of SYM_TYPED_FUNC_START.
As such, remove the implicit ENDBR from SYM_FUNC_START* and add some explicit annotations to fix things up again.
--- arch/x86/crypto/aesni-intel_asm.S | 2 ++ arch/x86/entry/calling.h | 1 + arch/x86/entry/entry.S | 2 ++ arch/x86/entry/entry_64.S | 3 +++ arch/x86/entry/entry_64_fred.S | 1 + arch/x86/entry/vdso/Makefile | 1 + arch/x86/include/asm/linkage.h | 18 ++++++------------ arch/x86/include/asm/page_64.h | 1 + arch/x86/include/asm/paravirt_types.h | 12 +++++++++++- arch/x86/include/asm/special_insns.h | 4 ++-- arch/x86/include/asm/string_64.h | 2 ++ arch/x86/kernel/acpi/madt_playdead.S | 1 + arch/x86/kernel/acpi/wakeup_64.S | 1 + arch/x86/kernel/alternative.c | 8 ++------ arch/x86/kernel/ftrace_64.S | 5 +++++ arch/x86/kernel/irqflags.S | 1 + arch/x86/kernel/paravirt.c | 14 ++++++++++++-- arch/x86/lib/clear_page_64.S | 2 ++ arch/x86/lib/copy_user_64.S | 3 +++ arch/x86/lib/copy_user_uncached_64.S | 2 ++ arch/x86/lib/getuser.S | 9 +++++++++ arch/x86/lib/hweight.S | 3 +++ arch/x86/lib/putuser.S | 9 +++++++++ arch/x86/lib/retpoline.S | 1 + arch/x86/mm/mem_encrypt_boot.S | 1 + arch/x86/power/hibernate_asm_64.S | 2 ++ arch/x86/xen/xen-asm.S | 5 +++++ arch/x86/xen/xen-head.S | 2 ++ include/linux/compiler.h | 10 ++++++++++ 29 files changed, 103 insertions(+), 23 deletions(-)
arch/x86/crypto/aesni-intel_asm.S +2 -0
@@ -17,6 +17,7 @@ */ #include <linux/linkage.h> +#include <linux/objtool.h> #include <asm/frame.h> #define STATE1 %xmm0
@@ -1071,6 +1072,7 @@ SYM_FUNC_END(_aesni_inc) * size_t len, u8 *iv) */ SYM_FUNC_START(aesni_ctr_enc) + ANNOTATE_NOENDBR FRAME_BEGIN cmp $16, LEN jb .Lctr_enc_just_ret
arch/x86/entry/calling.h +1 -0
@@ -431,6 +431,7 @@ For 32-bit we have the following convent/* rdi: arg1 ... normal C conventions. rax is saved/restored. */ .macro THUNK name, func SYM_FUNC_START(\name) + ANNOTATE_NOENDBR pushq %rbp movq %rsp, %rbp
arch/x86/entry/entry.S +2 -0
@@ -5,6 +5,7 @@ #include <linux/export.h> #include <linux/linkage.h> +#include <linux/objtool.h> #include <asm/msr-index.h> #include <asm/unwind_hints.h> #include <asm/segment.h>
@@ -17,6 +18,7 @@ .pushsection .noinstr.text, "ax" SYM_FUNC_START(entry_ibpb) + ANNOTATE_NOENDBR movl $MSR_IA32_PRED_CMD, %ecx movl $PRED_CMD_IBPB, %eax xorl %edx, %edx
arch/x86/entry/entry_64.S +3 -0
@@ -175,6 +175,7 @@ SYM_CODE_END(entry_SYSCALL_64) */ .pushsection .text, "ax" SYM_FUNC_START(__switch_to_asm) + ANNOTATE_NOENDBR /* * Save callee-saved registers * This must match the order in inactive_task_frame
@@ -742,6 +743,7 @@ _ASM_NOKPROBE(common_interrupt_return) * Is in entry.text as it shouldn't be instrumented. */ SYM_FUNC_START(asm_load_gs_index) + ANNOTATE_NOENDBR FRAME_BEGIN swapgs .Lgs_change:
@@ -1526,6 +1528,7 @@ SYM_CODE_END(rewind_stack_and_make_dead) * refactored in the future if needed. */ SYM_FUNC_START(clear_bhb_loop) + ANNOTATE_NOENDBR push %rbp mov %rsp, %rbp movl $5, %ecx
arch/x86/entry/entry_64_fred.S +1 -0
@@ -58,6 +58,7 @@ SYM_CODE_END(asm_fred_entrypoint_kernel) #if IS_ENABLED(CONFIG_KVM_INTEL) SYM_FUNC_START(asm_fred_entry_from_kvm) + ANNOTATE_NOENDBR push %rbp mov %rsp, %rbp
arch/x86/entry/vdso/Makefile +1 -0
@@ -133,6 +133,7 @@ KBUILD_CFLAGS_32 += -fno-stack-protector KBUILD_CFLAGS_32 += $(call cc-option, -foptimize-sibling-calls) KBUILD_CFLAGS_32 += -fno-omit-frame-pointer KBUILD_CFLAGS_32 += -DDISABLE_BRANCH_PROFILING +KBUILD_CFLAGS_32 += -DBUILD_VDSO ifdef CONFIG_MITIGATION_RETPOLINE ifneq ($(RETPOLINE_VDSO_CFLAGS),)
arch/x86/include/asm/linkage.h +6 -12
@@ -119,33 +119,27 @@/* SYM_FUNC_START -- use for global functions */ #define SYM_FUNC_START (name) \ - SYM_START(name, SYM_L_GLOBAL, SYM_F_ALIGN) \ - ENDBR + SYM_START(name, SYM_L_GLOBAL, SYM_F_ALIGN)/* SYM_FUNC_START_NOALIGN -- use for global functions, w/o alignment */ #define SYM_FUNC_START_NOALIGN (name) \ - SYM_START(name, SYM_L_GLOBAL, SYM_A_NONE) \ - ENDBR + SYM_START(name, SYM_L_GLOBAL, SYM_A_NONE)/* SYM_FUNC_START_LOCAL -- use for local functions */ #define SYM_FUNC_START_LOCAL (name) \ - SYM_START(name, SYM_L_LOCAL, SYM_F_ALIGN) \ - ENDBR + SYM_START(name, SYM_L_LOCAL, SYM_F_ALIGN)/* SYM_FUNC_START_LOCAL_NOALIGN -- use for local functions, w/o alignment */ #define SYM_FUNC_START_LOCAL_NOALIGN (name) \ - SYM_START(name, SYM_L_LOCAL, SYM_A_NONE) \ - ENDBR + SYM_START(name, SYM_L_LOCAL, SYM_A_NONE)/* SYM_FUNC_START_WEAK -- use for weak functions */ #define SYM_FUNC_START_WEAK (name) \ - SYM_START(name, SYM_L_WEAK, SYM_F_ALIGN) \ - ENDBR + SYM_START(name, SYM_L_WEAK, SYM_F_ALIGN)/* SYM_FUNC_START_WEAK_NOALIGN -- use for weak functions, w/o alignment */ #define SYM_FUNC_START_WEAK_NOALIGN (name) \ - SYM_START(name, SYM_L_WEAK, SYM_A_NONE) \ - ENDBR + SYM_START(name, SYM_L_WEAK, SYM_A_NONE)#endif /* _ASM_X86_LINKAGE_H */
arch/x86/include/asm/page_64.h +1 -0
@@ -60,6 +60,7 @@ static inline void clear_page(void *page }void copy_page (void *to,void *from); +KCFI_REFERENCE (copy_page);#ifdef CONFIG_X86_5LEVEL /*
arch/x86/include/asm/paravirt_types.h +11 -1
@@ -244,7 +244,17 @@ extern struct paravirt_patch_template pvint paravirt_disable_iospace (void ); -/* This generates an indirect call based on the operation type number. */ +/* +* This generates an indirect call based on the operation type number. +* +* Since alternatives run after enabling CET/IBT -- the latter setting/clearing +* capabilities and the former requiring all capabilities being finalized -- +* these indirect calls are subject to IBT and the paravirt stubs should have +* ENDBR on. +* +* OTOH since this is effectively a __nocfi indirect call, the paravirt stubs +* don't need to bother with CFI prefixes. +*/ #define PARAVIRT_CALL \ ANNOTATE_RETPOLINE_SAFE \ "call *%[paravirt_opptr];"
arch/x86/include/asm/special_insns.h +2 -2
@@ -42,14 +42,14 @@ static __always_inline void native_write asmvolatile ("mov %0,%%cr2" : :"r" (val) :"memory" ); } -static inline unsigned long __native_read_cr3 (void ) +static __always_inline unsigned long __native_read_cr3 (void ) {unsigned long val; asm volatile("mov %%cr3,%0\n\t" : "=r" (val) : __FORCE_ORDER);return val; } -static inline void native_write_cr3 (unsigned long val) +static __always_inline void native_write_cr3(unsigned long val) { asmvolatile ("mov %0,%%cr3" : :"r" (val) :"memory" ); }
arch/x86/include/asm/string_64.h +2 -0
@@ -21,6 +21,7 @@ extern void *__memcpy(void *to, const vo#define __HAVE_ARCH_MEMSETvoid *memset (void *s,int c,size_t n);void *__memset (void *s,int c,size_t n); +KCFI_REFERENCE (__memset);/* * KMSAN needs to instrument as much code as possible. Use C versions of
@@ -70,6 +71,7 @@ static inline void *memset64(uint64_t *s#define __HAVE_ARCH_MEMMOVEvoid *memmove (void *dest,const void *src,size_t count);void *__memmove (void *dest,const void *src,size_t count); +KCFI_REFERENCE (__memmove);int memcmp (const void *cs,const void *ct,size_t count);size_t strlen (const char *s);
arch/x86/kernel/acpi/madt_playdead.S +1 -0
@@ -14,6 +14,7 @@ * rsi: PGD of the identity mapping */ SYM_FUNC_START(asm_acpi_mp_play_dead) + ANNOTATE_NOENDBR /* Turn off global entries. Following CR3 write will flush them. */ movq %cr4, %rdx andq $~(X86_CR4_PGE), %rdx
arch/x86/kernel/acpi/wakeup_64.S +1 -0
@@ -17,6 +17,7 @@ * Hooray, we are in Long 64-bit mode (but still running in low memory) */ SYM_FUNC_START(wakeup_long64) + ANNOTATE_NOENDBR movq saved_magic(%rip), %rax movq $0x123456789abcdef0, %rdx cmpq %rdx, %rax
arch/x86/kernel/alternative.c +2 -6
@@ -918,11 +918,7 @@ struct bpf_insn;extern unsigned int __bpf_prog_runX (const void *ctx,const struct bpf_insn *insn); -/* -* Force a reference to the external symbol so the compiler generates -* __kcfi_typid. -*/ -__ADDRESSABLE (__bpf_prog_runX); +KCFI_REFERENCE (__bpf_prog_runX);/* u32 __ro_after_init cfi_bpf_hash = __kcfi_typeid___bpf_prog_runX; */ asm (
@@ -939,7 +935,7 @@ asm (/* Must match bpf_callback_t */ extern u64 __bpf_callback_fn (u64 ,u64 ,u64 ,u64 ,u64 ); -__ADDRESSABLE (__bpf_callback_fn); +KCFI_REFERENCE (__bpf_callback_fn);/* u32 __ro_after_init cfi_bpf_subprog_hash = __kcfi_typeid___bpf_callback_fn; */ asm (
arch/x86/kernel/ftrace_64.S +5 -0
@@ -146,12 +146,14 @@ SYM_FUNC_END(ftrace_stub_graph) #ifdef CONFIG_DYNAMIC_FTRACE SYM_FUNC_START(__fentry__) + ANNOTATE_NOENDBR CALL_DEPTH_ACCOUNT RET SYM_FUNC_END(__fentry__) EXPORT_SYMBOL(__fentry__) SYM_FUNC_START(ftrace_caller) + ANNOTATE_NOENDBR /* save_mcount_regs fills in first two parameters */ save_mcount_regs
@@ -197,6 +199,7 @@ SYM_FUNC_END(ftrace_caller); STACK_FRAME_NON_STANDARD_FP(ftrace_caller) SYM_FUNC_START(ftrace_regs_caller) + ANNOTATE_NOENDBR /* Save the current flags before any operations that can change them */ pushfq
@@ -310,6 +313,7 @@ SYM_FUNC_END(ftrace_regs_caller) STACK_FRAME_NON_STANDARD_FP(ftrace_regs_caller) SYM_FUNC_START(ftrace_stub_direct_tramp) + ANNOTATE_NOENDBR CALL_DEPTH_ACCOUNT RET SYM_FUNC_END(ftrace_stub_direct_tramp)
@@ -317,6 +321,7 @@ SYM_FUNC_END(ftrace_stub_direct_tramp) #else /* ! CONFIG_DYNAMIC_FTRACE */ SYM_FUNC_START(__fentry__) + ANNOTATE_NOENDBR CALL_DEPTH_ACCOUNT cmpq $ftrace_stub, ftrace_trace_function
arch/x86/kernel/irqflags.S +1 -0
@@ -9,6 +9,7 @@ */ .pushsection .noinstr.text, "ax" SYM_FUNC_START(native_save_fl) + ENDBR pushf pop %_ASM_AX RET
arch/x86/kernel/paravirt.c +12 -2
@@ -116,6 +116,16 @@ static noinstr void pv_native_write_cr2(native_write_cr2 (val); } +static noinstr unsigned long pv_native_read_cr3 (void ) +{ +return __native_read_cr3 (); +} + +static noinstr void pv_native_write_cr3(unsigned long cr3) +{ +native_write_cr3 (cr3); +} +static noinstr unsigned long pv_native_get_debugreg (int regno) {return native_get_debugreg (regno);
@@ -203,8 +213,8 @@ struct paravirt_patch_template pv_ops =#ifdef CONFIG_PARAVIRT_XXL .mmu.read_cr2 = __PV_IS_CALLEE_SAVE(pv_native_read_cr2), .mmu.write_cr2 = pv_native_write_cr2, - .mmu.read_cr3 = __native_read_cr3, - .mmu.write_cr3 = native_write_cr3, + .mmu.read_cr3 = pv_native_read_cr3, + .mmu.write_cr3 = pv_native_write_cr3, .mmu.pgd_alloc = __paravirt_pgd_alloc, .mmu.pgd_free = paravirt_nop,
arch/x86/lib/clear_page_64.S +2 -0
@@ -2,6 +2,7 @@ #include <linux/export.h> #include <linux/linkage.h> #include <linux/cfi_types.h> +#include <linux/objtool.h> #include <asm/asm.h> /*
@@ -64,6 +65,7 @@ EXPORT_SYMBOL_GPL(clear_page_erms) * rcx: uncleared bytes or 0 if successful. */ SYM_FUNC_START(rep_stos_alternative) + ANNOTATE_NOENDBR cmpq $64,%rcx jae .Lunrolled
arch/x86/lib/copy_user_64.S +3 -0
@@ -8,6 +8,8 @@ #include <linux/export.h> #include <linux/linkage.h> +#include <linux/cfi_types.h> +#include <linux/objtool.h> #include <asm/cpufeatures.h> #include <asm/alternative.h> #include <asm/asm.h>
@@ -30,6 +32,7 @@ * it simpler for us, we can clobber rsi/rdi and rax freely. */ SYM_FUNC_START(rep_movs_alternative) + ANNOTATE_NOENDBR cmpq $64,%rcx jae .Llarge
arch/x86/lib/copy_user_uncached_64.S +2 -0
@@ -5,6 +5,7 @@ #include <linux/export.h> #include <linux/linkage.h> +#include <linux/objtool.h> #include <asm/asm.h> /*
@@ -27,6 +28,7 @@ * rax uncopied bytes or 0 if successful. */ SYM_FUNC_START(__copy_user_nocache) + ANNOTATE_NOENDBR /* If destination is not 7-byte aligned, we'll have to align it */ testb $7,%dil jne .Lalign
arch/x86/lib/getuser.S +9 -0
@@ -28,6 +28,7 @@ #include <linux/export.h> #include <linux/linkage.h> +#include <linux/objtool.h> #include <asm/page_types.h> #include <asm/errno.h> #include <asm/asm-offsets.h>
@@ -62,6 +63,7 @@ .text SYM_FUNC_START(__get_user_1) + ANNOTATE_NOENDBR check_range size=1 ASM_STAC UACCESS movzbl (%_ASM_AX),%edx
@@ -72,6 +74,7 @@ SYM_FUNC_END(__get_user_1) EXPORT_SYMBOL(__get_user_1) SYM_FUNC_START(__get_user_2) + ANNOTATE_NOENDBR check_range size=2 ASM_STAC UACCESS movzwl (%_ASM_AX),%edx
@@ -82,6 +85,7 @@ SYM_FUNC_END(__get_user_2) EXPORT_SYMBOL(__get_user_2) SYM_FUNC_START(__get_user_4) + ANNOTATE_NOENDBR check_range size=4 ASM_STAC UACCESS movl (%_ASM_AX),%edx
@@ -92,6 +96,7 @@ SYM_FUNC_END(__get_user_4) EXPORT_SYMBOL(__get_user_4) SYM_FUNC_START(__get_user_8) + ANNOTATE_NOENDBR #ifndef CONFIG_X86_64 xor %ecx,%ecx #endif
@@ -111,6 +116,7 @@ EXPORT_SYMBOL(__get_user_8) /* .. and the same for __get_user, just without the range checks */ SYM_FUNC_START(__get_user_nocheck_1) + ANNOTATE_NOENDBR ASM_STAC ASM_BARRIER_NOSPEC UACCESS movzbl (%_ASM_AX),%edx
@@ -121,6 +127,7 @@ SYM_FUNC_END(__get_user_nocheck_1) EXPORT_SYMBOL(__get_user_nocheck_1) SYM_FUNC_START(__get_user_nocheck_2) + ANNOTATE_NOENDBR ASM_STAC ASM_BARRIER_NOSPEC UACCESS movzwl (%_ASM_AX),%edx
@@ -131,6 +138,7 @@ SYM_FUNC_END(__get_user_nocheck_2) EXPORT_SYMBOL(__get_user_nocheck_2) SYM_FUNC_START(__get_user_nocheck_4) + ANNOTATE_NOENDBR ASM_STAC ASM_BARRIER_NOSPEC UACCESS movl (%_ASM_AX),%edx
@@ -141,6 +149,7 @@ SYM_FUNC_END(__get_user_nocheck_4) EXPORT_SYMBOL(__get_user_nocheck_4) SYM_FUNC_START(__get_user_nocheck_8) + ANNOTATE_NOENDBR ASM_STAC ASM_BARRIER_NOSPEC #ifdef CONFIG_X86_64
arch/x86/lib/hweight.S +3 -0
@@ -1,6 +1,7 @@ /* SPDX-License-Identifier: GPL-2.0 */ #include <linux/export.h> #include <linux/linkage.h> +#include <linux/objtool.h> #include <asm/asm.h>
@@ -9,6 +10,7 @@ * %rdi: w */ SYM_FUNC_START(__sw_hweight32) + ANNOTATE_NOENDBR #ifdef CONFIG_X86_64 movl %edi, %eax # w
@@ -42,6 +44,7 @@ EXPORT_SYMBOL(__sw_hweight32) */ #ifdef CONFIG_X86_64 SYM_FUNC_START(__sw_hweight64) + ANNOTATE_NOENDBR pushq %rdi pushq %rdx
arch/x86/lib/putuser.S +9 -0
@@ -13,6 +13,7 @@ */ #include <linux/export.h> #include <linux/linkage.h> +#include <linux/objtool.h> #include <asm/thread_info.h> #include <asm/errno.h> #include <asm/asm.h>
@@ -45,6 +46,7 @@ .text SYM_FUNC_START(__put_user_1) + ANNOTATE_NOENDBR check_range size=1 ASM_STAC 1: movb %al,(%_ASM_CX)
@@ -55,6 +57,7 @@ SYM_FUNC_END(__put_user_1) EXPORT_SYMBOL(__put_user_1) SYM_FUNC_START(__put_user_nocheck_1) + ANNOTATE_NOENDBR ASM_STAC 2: movb %al,(%_ASM_CX) xor %ecx,%ecx
@@ -64,6 +67,7 @@ SYM_FUNC_END(__put_user_nocheck_1) EXPORT_SYMBOL(__put_user_nocheck_1) SYM_FUNC_START(__put_user_2) + ANNOTATE_NOENDBR check_range size=2 ASM_STAC 3: movw %ax,(%_ASM_CX)
@@ -74,6 +78,7 @@ SYM_FUNC_END(__put_user_2) EXPORT_SYMBOL(__put_user_2) SYM_FUNC_START(__put_user_nocheck_2) + ANNOTATE_NOENDBR ASM_STAC 4: movw %ax,(%_ASM_CX) xor %ecx,%ecx
@@ -83,6 +88,7 @@ SYM_FUNC_END(__put_user_nocheck_2) EXPORT_SYMBOL(__put_user_nocheck_2) SYM_FUNC_START(__put_user_4) + ANNOTATE_NOENDBR check_range size=4 ASM_STAC 5: movl %eax,(%_ASM_CX)
@@ -93,6 +99,7 @@ SYM_FUNC_END(__put_user_4) EXPORT_SYMBOL(__put_user_4) SYM_FUNC_START(__put_user_nocheck_4) + ANNOTATE_NOENDBR ASM_STAC 6: movl %eax,(%_ASM_CX) xor %ecx,%ecx
@@ -102,6 +109,7 @@ SYM_FUNC_END(__put_user_nocheck_4) EXPORT_SYMBOL(__put_user_nocheck_4) SYM_FUNC_START(__put_user_8) + ANNOTATE_NOENDBR check_range size=8 ASM_STAC 7: mov %_ASM_AX,(%_ASM_CX)
@@ -115,6 +123,7 @@ SYM_FUNC_END(__put_user_8) EXPORT_SYMBOL(__put_user_8) SYM_FUNC_START(__put_user_nocheck_8) + ANNOTATE_NOENDBR ASM_STAC 9: mov %_ASM_AX,(%_ASM_CX) #ifdef CONFIG_X86_32
arch/x86/lib/retpoline.S +1 -0
@@ -326,6 +326,7 @@ SYM_FUNC_END(retbleed_untrain_ret) #if defined(CONFIG_MITIGATION_UNRET_ENTRY) || defined(CONFIG_MITIGATION_SRSO) SYM_FUNC_START(entry_untrain_ret) + ANNOTATE_NOENDBR ALTERNATIVE JMP_RETBLEED_UNTRAIN_RET, JMP_SRSO_UNTRAIN_RET, X86_FEATURE_SRSO SYM_FUNC_END(entry_untrain_ret) __EXPORT_THUNK(entry_untrain_ret)
arch/x86/mm/mem_encrypt_boot.S +1 -0
@@ -72,6 +72,7 @@ SYM_FUNC_START(sme_encrypt_execute) SYM_FUNC_END(sme_encrypt_execute) SYM_FUNC_START(__enc_copy) + ANNOTATE_NOENDBR /* * Routine used to encrypt memory in place. * This routine must be run outside of the kernel proper since
arch/x86/power/hibernate_asm_64.S +2 -0
@@ -26,6 +26,7 @@ /* code below belongs to the image kernel */ .align PAGE_SIZE SYM_FUNC_START(restore_registers) + ANNOTATE_NOENDBR /* go back to the original page tables */ movq %r9, %cr3
@@ -119,6 +120,7 @@ SYM_FUNC_END(restore_image) /* code below has been relocated to a safe page */ SYM_FUNC_START(core_restore_code) + ANNOTATE_NOENDBR /* switch to temporary page tables */ movq %rax, %cr3 /* flush TLB */
arch/x86/xen/xen-asm.S +5 -0
@@ -51,6 +51,7 @@ SYM_FUNC_END(xen_hypercall_pv) * non-zero. */ SYM_FUNC_START(xen_irq_disable_direct) + ENDBR movb $1, PER_CPU_VAR(xen_vcpu_info + XEN_vcpu_info_mask) RET SYM_FUNC_END(xen_irq_disable_direct)
@@ -90,6 +91,7 @@ SYM_FUNC_END(check_events) * then enter the hypervisor to get them handled. */ SYM_FUNC_START(xen_irq_enable_direct) + ENDBR FRAME_BEGIN /* Unmask events */ movb $0, PER_CPU_VAR(xen_vcpu_info + XEN_vcpu_info_mask)
@@ -120,6 +122,7 @@ SYM_FUNC_END(xen_irq_enable_direct) * x86 use opposite senses (mask vs enable). */ SYM_FUNC_START(xen_save_fl_direct) + ENDBR testb $0xff, PER_CPU_VAR(xen_vcpu_info + XEN_vcpu_info_mask) setz %ah addb %ah, %ah
@@ -127,6 +130,7 @@ SYM_FUNC_START(xen_save_fl_direct) SYM_FUNC_END(xen_save_fl_direct) SYM_FUNC_START(xen_read_cr2) + ENDBR FRAME_BEGIN _ASM_MOV PER_CPU_VAR(xen_vcpu), %_ASM_AX _ASM_MOV XEN_vcpu_info_arch_cr2(%_ASM_AX), %_ASM_AX
@@ -135,6 +139,7 @@ SYM_FUNC_START(xen_read_cr2) SYM_FUNC_END(xen_read_cr2); SYM_FUNC_START(xen_read_cr2_direct) + ENDBR FRAME_BEGIN _ASM_MOV PER_CPU_VAR(xen_vcpu_info + XEN_vcpu_info_arch_cr2), %_ASM_AX FRAME_END
arch/x86/xen/xen-head.S +2 -0
@@ -138,11 +138,13 @@ SYM_FUNC_START(xen_hypercall_hvm) SYM_FUNC_END(xen_hypercall_hvm) SYM_FUNC_START(xen_hypercall_amd) + ANNOTATE_NOENDBR vmmcall RET SYM_FUNC_END(xen_hypercall_amd) SYM_FUNC_START(xen_hypercall_intel) + ANNOTATE_NOENDBR vmcall RET SYM_FUNC_END(xen_hypercall_intel)
include/linux/compiler.h +10 -0
@@ -193,6 +193,16 @@ void ftrace_likely_update(struct ftrace_#endif /* __KERNEL__ */ +#if defined(CONFIG_CFI_CLANG ) && !defined(__DISABLE_EXPORTS) && !defined(BUILD_VDSO ) +/* +* Force a reference to the external symbol so the compiler generates +* __kcfi_typid. +*/ +#define KCFI_REFERENCE (sym) __ADDRESSABLE(sym) +#else +#define KCFI_REFERENCE (sym) +#endif +/** * offset_to_ptr - convert a relative memory offset to an absolute pointer * @off: the address of the 32-bit offset value
[tip: x86/core] x86/cfi: Clean up linkage
The following commit has been merged into the x86/core branch of tip:
The following commit has been merged into the x86/core branch of tip:
Commit-ID: 582077c94052bd69a544b3f9d7619c9c6a67c34b Gitweb: https://git.kernel.org/tip/582077c94052bd69a544b3f9d7619c9c6a67c34b Author: Peter Zijlstra <[email protected]> AuthorDate: Fri, 07 Feb 2025 13:15:33 +01:00 Committer: Peter Zijlstra <[email protected]> CommitterDate: Fri, 14 Feb 2025 10:32:05 +01:00
x86/cfi: Clean up linkage
With the introduction of kCFI the addition of ENDBR to SYM_FUNC_START* no longer suffices to make the function indirectly callable. This now requires the use of SYM_TYPED_FUNC_START.
As such, remove the implicit ENDBR from SYM_FUNC_START* and add some explicit annotations to fix things up again.
--- arch/x86/crypto/aesni-intel_asm.S | 2 ++ arch/x86/entry/calling.h | 1 + arch/x86/entry/entry.S | 2 ++ arch/x86/entry/entry_64.S | 3 +++ arch/x86/entry/entry_64_fred.S | 1 + arch/x86/entry/vdso/Makefile | 1 + arch/x86/include/asm/linkage.h | 18 ++++++------------ arch/x86/include/asm/page_64.h | 1 + arch/x86/include/asm/paravirt_types.h | 12 +++++++++++- arch/x86/include/asm/special_insns.h | 4 ++-- arch/x86/include/asm/string_64.h | 2 ++ arch/x86/kernel/acpi/madt_playdead.S | 1 + arch/x86/kernel/acpi/wakeup_64.S | 1 + arch/x86/kernel/alternative.c | 8 ++------ arch/x86/kernel/ftrace_64.S | 5 +++++ arch/x86/kernel/irqflags.S | 1 + arch/x86/kernel/paravirt.c | 14 ++++++++++++-- arch/x86/lib/clear_page_64.S | 2 ++ arch/x86/lib/copy_user_64.S | 3 +++ arch/x86/lib/copy_user_uncached_64.S | 2 ++ arch/x86/lib/getuser.S | 9 +++++++++ arch/x86/lib/hweight.S | 3 +++ arch/x86/lib/putuser.S | 9 +++++++++ arch/x86/lib/retpoline.S | 1 + arch/x86/mm/mem_encrypt_boot.S | 1 + arch/x86/power/hibernate_asm_64.S | 2 ++ arch/x86/xen/xen-asm.S | 5 +++++ arch/x86/xen/xen-head.S | 2 ++ include/linux/compiler.h | 10 ++++++++++ 29 files changed, 103 insertions(+), 23 deletions(-)
arch/x86/crypto/aesni-intel_asm.S +2 -0
@@ -17,6 +17,7 @@ */ #include <linux/linkage.h> +#include <linux/objtool.h> #include <asm/frame.h> #define STATE1 %xmm0
@@ -1071,6 +1072,7 @@ SYM_FUNC_END(_aesni_inc) * size_t len, u8 *iv) */ SYM_FUNC_START(aesni_ctr_enc) + ANNOTATE_NOENDBR FRAME_BEGIN cmp $16, LEN jb .Lctr_enc_just_ret
arch/x86/entry/calling.h +1 -0
@@ -431,6 +431,7 @@ For 32-bit we have the following conventions - kernel is built with/* rdi: arg1 ... normal C conventions. rax is saved/restored. */ .macro THUNK name, func SYM_FUNC_START(\name) + ANNOTATE_NOENDBR pushq %rbp movq %rsp, %rbp
arch/x86/entry/entry.S +2 -0
@@ -5,6 +5,7 @@ #include <linux/export.h> #include <linux/linkage.h> +#include <linux/objtool.h> #include <asm/msr-index.h> #include <asm/unwind_hints.h> #include <asm/segment.h>
@@ -17,6 +18,7 @@ .pushsection .noinstr.text, "ax" SYM_FUNC_START(entry_ibpb) + ANNOTATE_NOENDBR movl $MSR_IA32_PRED_CMD, %ecx movl $PRED_CMD_IBPB, %eax xorl %edx, %edx
arch/x86/entry/entry_64.S +3 -0
@@ -175,6 +175,7 @@ SYM_CODE_END(entry_SYSCALL_64) */ .pushsection .text, "ax" SYM_FUNC_START(__switch_to_asm) + ANNOTATE_NOENDBR /* * Save callee-saved registers * This must match the order in inactive_task_frame
@@ -742,6 +743,7 @@ _ASM_NOKPROBE(common_interrupt_return) * Is in entry.text as it shouldn't be instrumented. */ SYM_FUNC_START(asm_load_gs_index) + ANNOTATE_NOENDBR FRAME_BEGIN swapgs .Lgs_change:
@@ -1526,6 +1528,7 @@ SYM_CODE_END(rewind_stack_and_make_dead) * refactored in the future if needed. */ SYM_FUNC_START(clear_bhb_loop) + ANNOTATE_NOENDBR push %rbp mov %rsp, %rbp movl $5, %ecx
arch/x86/entry/entry_64_fred.S +1 -0
@@ -58,6 +58,7 @@ SYM_CODE_END(asm_fred_entrypoint_kernel) #if IS_ENABLED(CONFIG_KVM_INTEL) SYM_FUNC_START(asm_fred_entry_from_kvm) + ANNOTATE_NOENDBR push %rbp mov %rsp, %rbp
arch/x86/entry/vdso/Makefile +1 -0
@@ -133,6 +133,7 @@ KBUILD_CFLAGS_32 += -fno-stack-protector KBUILD_CFLAGS_32 += $(call cc-option, -foptimize-sibling-calls) KBUILD_CFLAGS_32 += -fno-omit-frame-pointer KBUILD_CFLAGS_32 += -DDISABLE_BRANCH_PROFILING +KBUILD_CFLAGS_32 += -DBUILD_VDSO ifdef CONFIG_MITIGATION_RETPOLINE ifneq ($(RETPOLINE_VDSO_CFLAGS),)
arch/x86/include/asm/linkage.h +6 -12
@@ -119,33 +119,27 @@/* SYM_FUNC_START -- use for global functions */ #define SYM_FUNC_START (name) \ - SYM_START(name, SYM_L_GLOBAL, SYM_F_ALIGN) \ - ENDBR + SYM_START(name, SYM_L_GLOBAL, SYM_F_ALIGN)/* SYM_FUNC_START_NOALIGN -- use for global functions, w/o alignment */ #define SYM_FUNC_START_NOALIGN (name) \ - SYM_START(name, SYM_L_GLOBAL, SYM_A_NONE) \ - ENDBR + SYM_START(name, SYM_L_GLOBAL, SYM_A_NONE)/* SYM_FUNC_START_LOCAL -- use for local functions */ #define SYM_FUNC_START_LOCAL (name) \ - SYM_START(name, SYM_L_LOCAL, SYM_F_ALIGN) \ - ENDBR + SYM_START(name, SYM_L_LOCAL, SYM_F_ALIGN)/* SYM_FUNC_START_LOCAL_NOALIGN -- use for local functions, w/o alignment */ #define SYM_FUNC_START_LOCAL_NOALIGN (name) \ - SYM_START(name, SYM_L_LOCAL, SYM_A_NONE) \ - ENDBR + SYM_START(name, SYM_L_LOCAL, SYM_A_NONE)/* SYM_FUNC_START_WEAK -- use for weak functions */ #define SYM_FUNC_START_WEAK (name) \ - SYM_START(name, SYM_L_WEAK, SYM_F_ALIGN) \ - ENDBR + SYM_START(name, SYM_L_WEAK, SYM_F_ALIGN)/* SYM_FUNC_START_WEAK_NOALIGN -- use for weak functions, w/o alignment */ #define SYM_FUNC_START_WEAK_NOALIGN (name) \ - SYM_START(name, SYM_L_WEAK, SYM_A_NONE) \ - ENDBR + SYM_START(name, SYM_L_WEAK, SYM_A_NONE)#endif /* _ASM_X86_LINKAGE_H */
arch/x86/include/asm/page_64.h +1 -0
@@ -60,6 +60,7 @@ static inline void clear_page(void *page) }void copy_page (void *to,void *from); +KCFI_REFERENCE (copy_page);#ifdef CONFIG_X86_5LEVEL /*
arch/x86/include/asm/paravirt_types.h +11 -1
@@ -244,7 +244,17 @@ extern struct paravirt_patch_template pv_ops;int paravirt_disable_iospace (void ); -/* This generates an indirect call based on the operation type number. */ +/* +* This generates an indirect call based on the operation type number. +* +* Since alternatives run after enabling CET/IBT -- the latter setting/clearing +* capabilities and the former requiring all capabilities being finalized -- +* these indirect calls are subject to IBT and the paravirt stubs should have +* ENDBR on. +* +* OTOH since this is effectively a __nocfi indirect call, the paravirt stubs +* don't need to bother with CFI prefixes. +*/ #define PARAVIRT_CALL \ ANNOTATE_RETPOLINE_SAFE \ "call *%[paravirt_opptr];"
arch/x86/include/asm/special_insns.h +2 -2
@@ -42,14 +42,14 @@ static __always_inline void native_write_cr2(unsigned long val) asmvolatile ("mov %0,%%cr2" : :"r" (val) :"memory" ); } -static inline unsigned long __native_read_cr3 (void ) +static __always_inline unsigned long __native_read_cr3 (void ) {unsigned long val; asm volatile("mov %%cr3,%0\n\t" : "=r" (val) : __FORCE_ORDER);return val; } -static inline void native_write_cr3 (unsigned long val) +static __always_inline void native_write_cr3(unsigned long val) { asmvolatile ("mov %0,%%cr3" : :"r" (val) :"memory" ); }
arch/x86/include/asm/string_64.h +2 -0
@@ -21,6 +21,7 @@ extern void *__memcpy(void *to, const void *from, size_t len);#define __HAVE_ARCH_MEMSETvoid *memset (void *s,int c,size_t n);void *__memset (void *s,int c,size_t n); +KCFI_REFERENCE (__memset);/* * KMSAN needs to instrument as much code as possible. Use C versions of
@@ -70,6 +71,7 @@ static inline void *memset64(uint64_t *s, uint64_t v, size_t n)#define __HAVE_ARCH_MEMMOVEvoid *memmove (void *dest,const void *src,size_t count);void *__memmove (void *dest,const void *src,size_t count); +KCFI_REFERENCE (__memmove);int memcmp (const void *cs,const void *ct,size_t count);size_t strlen (const char *s);
arch/x86/kernel/acpi/madt_playdead.S +1 -0
@@ -14,6 +14,7 @@ * rsi: PGD of the identity mapping */ SYM_FUNC_START(asm_acpi_mp_play_dead) + ANNOTATE_NOENDBR /* Turn off global entries. Following CR3 write will flush them. */ movq %cr4, %rdx andq $~(X86_CR4_PGE), %rdx
arch/x86/kernel/acpi/wakeup_64.S +1 -0
@@ -17,6 +17,7 @@ * Hooray, we are in Long 64-bit mode (but still running in low memory) */ SYM_FUNC_START(wakeup_long64) + ANNOTATE_NOENDBR movq saved_magic(%rip), %rax movq $0x123456789abcdef0, %rdx cmpq %rdx, %rax
arch/x86/kernel/alternative.c +2 -6
@@ -926,11 +926,7 @@ struct bpf_insn;extern unsigned int __bpf_prog_runX (const void *ctx,const struct bpf_insn *insn); -/* -* Force a reference to the external symbol so the compiler generates -* __kcfi_typid. -*/ -__ADDRESSABLE (__bpf_prog_runX); +KCFI_REFERENCE (__bpf_prog_runX);/* u32 __ro_after_init cfi_bpf_hash = __kcfi_typeid___bpf_prog_runX; */ asm (
@@ -947,7 +943,7 @@ asm (/* Must match bpf_callback_t */ extern u64 __bpf_callback_fn (u64 ,u64 ,u64 ,u64 ,u64 ); -__ADDRESSABLE (__bpf_callback_fn); +KCFI_REFERENCE (__bpf_callback_fn);/* u32 __ro_after_init cfi_bpf_subprog_hash = __kcfi_typeid___bpf_callback_fn; */ asm (
arch/x86/kernel/ftrace_64.S +5 -0
@@ -146,12 +146,14 @@ SYM_FUNC_END(ftrace_stub_graph) #ifdef CONFIG_DYNAMIC_FTRACE SYM_FUNC_START(__fentry__) + ANNOTATE_NOENDBR CALL_DEPTH_ACCOUNT RET SYM_FUNC_END(__fentry__) EXPORT_SYMBOL(__fentry__) SYM_FUNC_START(ftrace_caller) + ANNOTATE_NOENDBR /* save_mcount_regs fills in first two parameters */ save_mcount_regs
@@ -197,6 +199,7 @@ SYM_FUNC_END(ftrace_caller); STACK_FRAME_NON_STANDARD_FP(ftrace_caller) SYM_FUNC_START(ftrace_regs_caller) + ANNOTATE_NOENDBR /* Save the current flags before any operations that can change them */ pushfq
@@ -310,6 +313,7 @@ SYM_FUNC_END(ftrace_regs_caller) STACK_FRAME_NON_STANDARD_FP(ftrace_regs_caller) SYM_FUNC_START(ftrace_stub_direct_tramp) + ANNOTATE_NOENDBR CALL_DEPTH_ACCOUNT RET SYM_FUNC_END(ftrace_stub_direct_tramp)
@@ -317,6 +321,7 @@ SYM_FUNC_END(ftrace_stub_direct_tramp) #else /* ! CONFIG_DYNAMIC_FTRACE */ SYM_FUNC_START(__fentry__) + ANNOTATE_NOENDBR CALL_DEPTH_ACCOUNT cmpq $ftrace_stub, ftrace_trace_function
arch/x86/kernel/irqflags.S +1 -0
@@ -9,6 +9,7 @@ */ .pushsection .noinstr.text, "ax" SYM_FUNC_START(native_save_fl) + ENDBR pushf pop %_ASM_AX RET
arch/x86/kernel/paravirt.c +12 -2
@@ -116,6 +116,16 @@ static noinstr void pv_native_write_cr2(unsigned long val)native_write_cr2 (val); } +static noinstr unsigned long pv_native_read_cr3 (void ) +{ +return __native_read_cr3 (); +} + +static noinstr void pv_native_write_cr3(unsigned long cr3) +{ +native_write_cr3 (cr3); +} +static noinstr unsigned long pv_native_get_debugreg (int regno) {return native_get_debugreg (regno);
@@ -203,8 +213,8 @@ struct paravirt_patch_template pv_ops = {#ifdef CONFIG_PARAVIRT_XXL .mmu.read_cr2 = __PV_IS_CALLEE_SAVE(pv_native_read_cr2), .mmu.write_cr2 = pv_native_write_cr2, - .mmu.read_cr3 = __native_read_cr3, - .mmu.write_cr3 = native_write_cr3, + .mmu.read_cr3 = pv_native_read_cr3, + .mmu.write_cr3 = pv_native_write_cr3, .mmu.pgd_alloc = __paravirt_pgd_alloc, .mmu.pgd_free = paravirt_nop,
arch/x86/lib/clear_page_64.S +2 -0
@@ -2,6 +2,7 @@ #include <linux/export.h> #include <linux/linkage.h> #include <linux/cfi_types.h> +#include <linux/objtool.h> #include <asm/asm.h> /*
@@ -64,6 +65,7 @@ EXPORT_SYMBOL_GPL(clear_page_erms) * rcx: uncleared bytes or 0 if successful. */ SYM_FUNC_START(rep_stos_alternative) + ANNOTATE_NOENDBR cmpq $64,%rcx jae .Lunrolled
arch/x86/lib/copy_user_64.S +3 -0
@@ -8,6 +8,8 @@ #include <linux/export.h> #include <linux/linkage.h> +#include <linux/cfi_types.h> +#include <linux/objtool.h> #include <asm/cpufeatures.h> #include <asm/alternative.h> #include <asm/asm.h>
@@ -30,6 +32,7 @@ * it simpler for us, we can clobber rsi/rdi and rax freely. */ SYM_FUNC_START(rep_movs_alternative) + ANNOTATE_NOENDBR cmpq $64,%rcx jae .Llarge
arch/x86/lib/copy_user_uncached_64.S +2 -0
@@ -5,6 +5,7 @@ #include <linux/export.h> #include <linux/linkage.h> +#include <linux/objtool.h> #include <asm/asm.h> /*
@@ -27,6 +28,7 @@ * rax uncopied bytes or 0 if successful. */ SYM_FUNC_START(__copy_user_nocache) + ANNOTATE_NOENDBR /* If destination is not 7-byte aligned, we'll have to align it */ testb $7,%dil jne .Lalign
arch/x86/lib/getuser.S +9 -0
@@ -28,6 +28,7 @@ #include <linux/export.h> #include <linux/linkage.h> +#include <linux/objtool.h> #include <asm/page_types.h> #include <asm/errno.h> #include <asm/asm-offsets.h>
@@ -62,6 +63,7 @@ .text SYM_FUNC_START(__get_user_1) + ANNOTATE_NOENDBR check_range size=1 ASM_STAC UACCESS movzbl (%_ASM_AX),%edx
@@ -72,6 +74,7 @@ SYM_FUNC_END(__get_user_1) EXPORT_SYMBOL(__get_user_1) SYM_FUNC_START(__get_user_2) + ANNOTATE_NOENDBR check_range size=2 ASM_STAC UACCESS movzwl (%_ASM_AX),%edx
@@ -82,6 +85,7 @@ SYM_FUNC_END(__get_user_2) EXPORT_SYMBOL(__get_user_2) SYM_FUNC_START(__get_user_4) + ANNOTATE_NOENDBR check_range size=4 ASM_STAC UACCESS movl (%_ASM_AX),%edx
@@ -92,6 +96,7 @@ SYM_FUNC_END(__get_user_4) EXPORT_SYMBOL(__get_user_4) SYM_FUNC_START(__get_user_8) + ANNOTATE_NOENDBR #ifndef CONFIG_X86_64 xor %ecx,%ecx #endif
@@ -111,6 +116,7 @@ EXPORT_SYMBOL(__get_user_8) /* .. and the same for __get_user, just without the range checks */ SYM_FUNC_START(__get_user_nocheck_1) + ANNOTATE_NOENDBR ASM_STAC ASM_BARRIER_NOSPEC UACCESS movzbl (%_ASM_AX),%edx
@@ -121,6 +127,7 @@ SYM_FUNC_END(__get_user_nocheck_1) EXPORT_SYMBOL(__get_user_nocheck_1) SYM_FUNC_START(__get_user_nocheck_2) + ANNOTATE_NOENDBR ASM_STAC ASM_BARRIER_NOSPEC UACCESS movzwl (%_ASM_AX),%edx
@@ -131,6 +138,7 @@ SYM_FUNC_END(__get_user_nocheck_2) EXPORT_SYMBOL(__get_user_nocheck_2) SYM_FUNC_START(__get_user_nocheck_4) + ANNOTATE_NOENDBR ASM_STAC ASM_BARRIER_NOSPEC UACCESS movl (%_ASM_AX),%edx
@@ -141,6 +149,7 @@ SYM_FUNC_END(__get_user_nocheck_4) EXPORT_SYMBOL(__get_user_nocheck_4) SYM_FUNC_START(__get_user_nocheck_8) + ANNOTATE_NOENDBR ASM_STAC ASM_BARRIER_NOSPEC #ifdef CONFIG_X86_64
arch/x86/lib/hweight.S +3 -0
@@ -1,6 +1,7 @@ /* SPDX-License-Identifier: GPL-2.0 */ #include <linux/export.h> #include <linux/linkage.h> +#include <linux/objtool.h> #include <asm/asm.h>
@@ -9,6 +10,7 @@ * %rdi: w */ SYM_FUNC_START(__sw_hweight32) + ANNOTATE_NOENDBR #ifdef CONFIG_X86_64 movl %edi, %eax # w
@@ -42,6 +44,7 @@ EXPORT_SYMBOL(__sw_hweight32) */ #ifdef CONFIG_X86_64 SYM_FUNC_START(__sw_hweight64) + ANNOTATE_NOENDBR pushq %rdi pushq %rdx
arch/x86/lib/putuser.S +9 -0
@@ -13,6 +13,7 @@ */ #include <linux/export.h> #include <linux/linkage.h> +#include <linux/objtool.h> #include <asm/thread_info.h> #include <asm/errno.h> #include <asm/asm.h>
@@ -45,6 +46,7 @@ .text SYM_FUNC_START(__put_user_1) + ANNOTATE_NOENDBR check_range size=1 ASM_STAC 1: movb %al,(%_ASM_CX)
@@ -55,6 +57,7 @@ SYM_FUNC_END(__put_user_1) EXPORT_SYMBOL(__put_user_1) SYM_FUNC_START(__put_user_nocheck_1) + ANNOTATE_NOENDBR ASM_STAC 2: movb %al,(%_ASM_CX) xor %ecx,%ecx
@@ -64,6 +67,7 @@ SYM_FUNC_END(__put_user_nocheck_1) EXPORT_SYMBOL(__put_user_nocheck_1) SYM_FUNC_START(__put_user_2) + ANNOTATE_NOENDBR check_range size=2 ASM_STAC 3: movw %ax,(%_ASM_CX)
@@ -74,6 +78,7 @@ SYM_FUNC_END(__put_user_2) EXPORT_SYMBOL(__put_user_2) SYM_FUNC_START(__put_user_nocheck_2) + ANNOTATE_NOENDBR ASM_STAC 4: movw %ax,(%_ASM_CX) xor %ecx,%ecx
@@ -83,6 +88,7 @@ SYM_FUNC_END(__put_user_nocheck_2) EXPORT_SYMBOL(__put_user_nocheck_2) SYM_FUNC_START(__put_user_4) + ANNOTATE_NOENDBR check_range size=4 ASM_STAC 5: movl %eax,(%_ASM_CX)
@@ -93,6 +99,7 @@ SYM_FUNC_END(__put_user_4) EXPORT_SYMBOL(__put_user_4) SYM_FUNC_START(__put_user_nocheck_4) + ANNOTATE_NOENDBR ASM_STAC 6: movl %eax,(%_ASM_CX) xor %ecx,%ecx
@@ -102,6 +109,7 @@ SYM_FUNC_END(__put_user_nocheck_4) EXPORT_SYMBOL(__put_user_nocheck_4) SYM_FUNC_START(__put_user_8) + ANNOTATE_NOENDBR check_range size=8 ASM_STAC 7: mov %_ASM_AX,(%_ASM_CX)
@@ -115,6 +123,7 @@ SYM_FUNC_END(__put_user_8) EXPORT_SYMBOL(__put_user_8) SYM_FUNC_START(__put_user_nocheck_8) + ANNOTATE_NOENDBR ASM_STAC 9: mov %_ASM_AX,(%_ASM_CX) #ifdef CONFIG_X86_32
arch/x86/lib/retpoline.S +1 -0
@@ -326,6 +326,7 @@ SYM_FUNC_END(retbleed_untrain_ret) #if defined(CONFIG_MITIGATION_UNRET_ENTRY) || defined(CONFIG_MITIGATION_SRSO) SYM_FUNC_START(entry_untrain_ret) + ANNOTATE_NOENDBR ALTERNATIVE JMP_RETBLEED_UNTRAIN_RET, JMP_SRSO_UNTRAIN_RET, X86_FEATURE_SRSO SYM_FUNC_END(entry_untrain_ret) __EXPORT_THUNK(entry_untrain_ret)
arch/x86/mm/mem_encrypt_boot.S +1 -0
@@ -72,6 +72,7 @@ SYM_FUNC_START(sme_encrypt_execute) SYM_FUNC_END(sme_encrypt_execute) SYM_FUNC_START(__enc_copy) + ANNOTATE_NOENDBR /* * Routine used to encrypt memory in place. * This routine must be run outside of the kernel proper since
arch/x86/power/hibernate_asm_64.S +2 -0
@@ -26,6 +26,7 @@ /* code below belongs to the image kernel */ .align PAGE_SIZE SYM_FUNC_START(restore_registers) + ANNOTATE_NOENDBR /* go back to the original page tables */ movq %r9, %cr3
@@ -119,6 +120,7 @@ SYM_FUNC_END(restore_image) /* code below has been relocated to a safe page */ SYM_FUNC_START(core_restore_code) + ANNOTATE_NOENDBR /* switch to temporary page tables */ movq %rax, %cr3 /* flush TLB */
arch/x86/xen/xen-asm.S +5 -0
@@ -51,6 +51,7 @@ SYM_FUNC_END(xen_hypercall_pv) * non-zero. */ SYM_FUNC_START(xen_irq_disable_direct) + ENDBR movb $1, PER_CPU_VAR(xen_vcpu_info + XEN_vcpu_info_mask) RET SYM_FUNC_END(xen_irq_disable_direct)
@@ -90,6 +91,7 @@ SYM_FUNC_END(check_events) * then enter the hypervisor to get them handled. */ SYM_FUNC_START(xen_irq_enable_direct) + ENDBR FRAME_BEGIN /* Unmask events */ movb $0, PER_CPU_VAR(xen_vcpu_info + XEN_vcpu_info_mask)
@@ -120,6 +122,7 @@ SYM_FUNC_END(xen_irq_enable_direct) * x86 use opposite senses (mask vs enable). */ SYM_FUNC_START(xen_save_fl_direct) + ENDBR testb $0xff, PER_CPU_VAR(xen_vcpu_info + XEN_vcpu_info_mask) setz %ah addb %ah, %ah
@@ -127,6 +130,7 @@ SYM_FUNC_START(xen_save_fl_direct) SYM_FUNC_END(xen_save_fl_direct) SYM_FUNC_START(xen_read_cr2) + ENDBR FRAME_BEGIN _ASM_MOV PER_CPU_VAR(xen_vcpu), %_ASM_AX _ASM_MOV XEN_vcpu_info_arch_cr2(%_ASM_AX), %_ASM_AX
@@ -135,6 +139,7 @@ SYM_FUNC_START(xen_read_cr2) SYM_FUNC_END(xen_read_cr2); SYM_FUNC_START(xen_read_cr2_direct) + ENDBR FRAME_BEGIN _ASM_MOV PER_CPU_VAR(xen_vcpu_info + XEN_vcpu_info_arch_cr2), %_ASM_AX FRAME_END
arch/x86/xen/xen-head.S +2 -0
@@ -133,11 +133,13 @@ SYM_FUNC_START(xen_hypercall_hvm) SYM_FUNC_END(xen_hypercall_hvm) SYM_FUNC_START(xen_hypercall_amd) + ANNOTATE_NOENDBR vmmcall RET SYM_FUNC_END(xen_hypercall_amd) SYM_FUNC_START(xen_hypercall_intel) + ANNOTATE_NOENDBR vmcall RET SYM_FUNC_END(xen_hypercall_intel)
include/linux/compiler.h +10 -0
@@ -212,6 +212,16 @@ void ftrace_likely_update(struct ftrace_likely_data *f, int val,#endif /* __KERNEL__ */ +#if defined(CONFIG_CFI_CLANG ) && !defined(__DISABLE_EXPORTS) && !defined(BUILD_VDSO ) +/* +* Force a reference to the external symbol so the compiler generates +* __kcfi_typid. +*/ +#define KCFI_REFERENCE (sym) __ADDRESSABLE(sym) +#else +#define KCFI_REFERENCE (sym) +#endif +/** * offset_to_ptr - convert a relative memory offset to an absolute pointer * @off: the address of the 32-bit offset value
Peter Zijlstra
maintainerPATCH 5/11 x86/boot: Mark start_secondary() with __noendbr
In reply to Peter ZijlstraThe handoff between the boot stubs and start_secondary() are before IBT is enabled and is definitely not subject to kCFI. As such, suppress all that for this function.
Notably when the ENDBR poison would become fatal (ud1 instead of nop) this will trigger a tripple fault because we haven't set up the IDT to handle #UD yet.
--- arch/x86/kernel/smpboot.c | 3 ++- include/linux/objtool.h | 4 +++- 2 files changed, 5 insertions(+), 2 deletions(-)
arch/x86/kernel/smpboot.c +2 -1
@@ -229,7 +229,7 @@ static void ap_calibrate_delay(void)/* * Activate a secondary processor. */ -static void notrace start_secondary(void *unused) +static void notrace __noendbr start_secondary(void *unused) {/* * Don't put *anything* except direct CPU state initialization
@@ -314,6 +314,7 @@ static void notrace start_secondary(voidwmb ();cpu_startup_entry (CPUHP_AP_ONLINE_IDLE ); } +ANNOTATE_NOENDBR_SYM (start_secondary);/* * The bootstrap kernel entry code has set these up. Save them for
include/linux/objtool.h +3 -1
@@ -128,7 +128,7 @@#define UNWIND_HINT (type, sp_reg, sp_offset, signal) "\n\t"#define STACK_FRAME_NON_STANDARD (func)#define STACK_FRAME_NON_STANDARD_FP (func) -#define __ASM_ANNOTATE (label, type) +#define __ASM_ANNOTATE (label, type) ""#define ASM_ANNOTATE (type)#else .macro UNWIND_HINT type:req sp_reg=0 sp_offset=0 signal=0
@@ -147,6 +147,8 @@* these relocations will never be used for indirect calls. */ #define ANNOTATE_NOENDBR ASM_ANNOTATE(ANNOTYPE_NOENDBR) +#define ANNOTATE_NOENDBR_SYM (sym) asm(__ASM_ANNOTATE(sym, ANNOTYPE_NOENDBR)) +/* * This should be used immediately before an indirect jump/call. It tells * objtool the subsequent indirect jump/call is vouched safe for retpoline
[tip: x86/core] x86/boot: Mark start_secondary() with __noendbr
The following commit has been merged into the x86/core branch of tip:
The following commit has been merged into the x86/core branch of tip:
Commit-ID: 93f16a1ab78ca56e3cd997d1ea54c214774781ac Gitweb: https://git.kernel.org/tip/93f16a1ab78ca56e3cd997d1ea54c214774781ac Author: Peter Zijlstra <[email protected]> AuthorDate: Fri, 07 Feb 2025 13:15:34 +01:00 Committer: Peter Zijlstra <[email protected]> CommitterDate: Fri, 14 Feb 2025 10:32:05 +01:00
x86/boot: Mark start_secondary() with __noendbr
The handoff between the boot stubs and start_secondary() are before IBT is enabled and is definitely not subject to kCFI. As such, suppress all that for this function.
Notably when the ENDBR poison would become fatal (ud1 instead of nop) this will trigger a tripple fault because we haven't set up the IDT to handle #UD yet.
--- arch/x86/kernel/smpboot.c | 3 ++- include/linux/objtool.h | 4 +++- 2 files changed, 5 insertions(+), 2 deletions(-)
arch/x86/kernel/smpboot.c +2 -1
@@ -229,7 +229,7 @@ static void ap_calibrate_delay(void)/* * Activate a secondary processor. */ -static void notrace start_secondary(void *unused) +static void notrace __noendbr start_secondary(void *unused) {/* * Don't put *anything* except direct CPU state initialization
@@ -314,6 +314,7 @@ static void notrace start_secondary(void *unused)wmb ();cpu_startup_entry (CPUHP_AP_ONLINE_IDLE ); } +ANNOTATE_NOENDBR_SYM (start_secondary);/* * The bootstrap kernel entry code has set these up. Save them for
include/linux/objtool.h +3 -1
@@ -128,7 +128,7 @@#define UNWIND_HINT (type, sp_reg, sp_offset, signal) "\n\t"#define STACK_FRAME_NON_STANDARD (func)#define STACK_FRAME_NON_STANDARD_FP (func) -#define __ASM_ANNOTATE (label, type) +#define __ASM_ANNOTATE (label, type) ""#define ASM_ANNOTATE (type)#else .macro UNWIND_HINT type:req sp_reg=0 sp_offset=0 signal=0
@@ -147,6 +147,8 @@* these relocations will never be used for indirect calls. */ #define ANNOTATE_NOENDBR ASM_ANNOTATE(ANNOTYPE_NOENDBR) +#define ANNOTATE_NOENDBR_SYM (sym) asm(__ASM_ANNOTATE(sym, ANNOTYPE_NOENDBR)) +/* * This should be used immediately before an indirect jump/call. It tells * objtool the subsequent indirect jump/call is vouched safe for retpoline
Peter Zijlstra
maintainerPATCH 6/11 x86/alternative: Simplify callthunk patching
In reply to Peter ZijlstraNow that paravirt call patching is implemented using alternatives, it is possible to avoid having to patch the alternative sites by including the altinstr_replacement calls in the call_sites list.
This means we're now stacking relative adjustments like so:
callthunks_patch_builtin_calls():
patches all function calls to target: func() -> func()-10 since the CALL accounting lives in the CALL_PADDING.
This explicitly includes .altinstr_replacement
alt_replace_call():
patches: x86_BUG() -> target()
this patching is done in a relative manner, and will preserve the above adjustment, meaning that with calldepth patching it will do: x86_BUG()-10 -> target()-10
apply_relocation():
does code relocation, and adjusts all RIP-relative instructions to the new location, also in a relative manner.
--- arch/x86/include/asm/alternative.h | 1 - arch/x86/kernel/alternative.c | 8 ++++---- arch/x86/kernel/callthunks.c | 13 ------------- arch/x86/kernel/module.c | 17 ++++++----------- tools/objtool/arch/x86/decode.c | 1 + tools/objtool/check.c | 12 ++---------- 6 files changed, 13 insertions(+), 39 deletions(-)
arch/x86/include/asm/alternative.h +0 -1
@@ -100,7 +100,6 @@ struct module;struct callthunk_sites {s32 *call_start, *call_end; -struct alt_instr *alt_start, *alt_end; };#ifdef CONFIG_CALL_THUNKS
arch/x86/kernel/alternative.c +4 -4
@@ -1697,14 +1697,14 @@ void __init alternative_instructions(voiapply_retpolines (__retpoline_sites, __retpoline_sites_end);apply_returns (__return_sites, __return_sites_end); -apply_alternatives (__alt_instructions, __alt_instructions_end); -/* -* Now all calls are established. Apply the call thunks if -* required. +* Adjust all CALL instructions to point to func()-10, including +* those in .altinstr_replacement. */ callthunks_patch_builtin_calls (); +apply_alternatives (__alt_instructions, __alt_instructions_end); +/* * Seal all functions that do not have their address taken. */
arch/x86/kernel/callthunks.c +0 -13
@@ -240,21 +240,10 @@ patch_call_sites(s32 *start, s32 *end, c } static __init_or_module void -patch_alt_call_sites (struct alt_instr *start,struct alt_instr *end, -const struct core_text *ct) -{ -struct alt_instr *a; - -for (a = start; a < end; a++) -patch_call ((void *)&a->instr_offset + a->instr_offset, ct); -} - -static __init_or_module voidcallthunks_setup (struct callthunk_sites *cs,const struct core_text *ct) {prdbg ("Patching call sites %s\n" , ct->name);patch_call_sites (cs->call_start, cs->call_end, ct); -patch_alt_call_sites (cs->alt_start, cs->alt_end, ct);prdbg ("Patching call sites done%s\n" , ct->name); }
@@ -263,8 +252,6 @@ void __init callthunks_patch_builtin_calstruct callthunk_sites cs = { .call_start = __call_sites, .call_end = __call_sites_end, - .alt_start = __alt_instructions, - .alt_end = __alt_instructions_end };if (!cpu_feature_enabled (X86_FEATURE_CALL_DEPTH ))
arch/x86/kernel/module.c +6 -11
@@ -275,12 +275,7 @@ int module_finalize(const Elf_Ehdr *hdr,void *rseg = (void *)returns->sh_addr;apply_returns (rseg, rseg + returns->sh_size); } -if (alt) { -/* patch .altinstructions */ -void *aseg = (void *)alt->sh_addr; -apply_alternatives (aseg, aseg + alt->sh_size); - } -if (calls || alt) { +if (calls) {struct callthunk_sites cs = {};if (calls) {
@@ -288,13 +283,13 @@ int module_finalize(const Elf_Ehdr *hdr, cs.call_end = (void *)calls->sh_addr + calls->sh_size; } -if (alt) { - cs.alt_start = (void *)alt->sh_addr; - cs.alt_end = (void *)alt->sh_addr + alt->sh_size; - } -callthunks_patch_module_calls (&cs, me); } +if (alt) { +/* patch .altinstructions */ +void *aseg = (void *)alt->sh_addr; +apply_alternatives (aseg, aseg + alt->sh_size); + }if (ibt_endbr) {void *iseg = (void *)ibt_endbr->sh_addr;apply_seal_endbr (iseg, iseg + ibt_endbr->sh_size);
tools/objtool/arch/x86/decode.c +1 -0
@@ -850,5 +850,6 @@ bool arch_is_rethunk(struct symbol *sym)bool arch_is_embedded_insn (struct symbol *sym) {return !strcmp (sym->name,"retbleed_return_thunk" ) || + !strcmp (sym->name,"srso_alias_safe_ret" ) || !strcmp (sym->name,"srso_safe_ret" ); }
tools/objtool/check.c +2 -10
@@ -1283,15 +1283,6 @@ static void annotate_call_site(struct obif (!sym) sym = reloc->sym; -/* -* Alternative replacement code is just template code which is -* sometimes copied to the original instruction. For now, don't -* annotate it. (In the future we might consider annotating the -* original instruction if/when it ever makes sense to do so.) -*/ -if (!strcmp (insn->sec->name,".altinstr_replacement" )) -return ; -if (sym->static_call_tramp) {list_add_tail (&insn->call_node, &file->static_call_list);return ;
@@ -1349,7 +1340,8 @@ static void annotate_call_site(struct obreturn ; } -if (insn->type ==INSN_CALL && !insn->sec->init) +if (insn->type ==INSN_CALL && !insn->sec->init && + !insn->_call_dest->embedded_insn)list_add_tail (&insn->call_node, &file->call_list);if (!sibling &&dead_end_function (file, sym))
Josh Poimboeuf
maintainerRe: [PATCH 06/11] x86/alternative: Simplify callthunk patching
arch/x86/kernel/alternative.c void __init alternative_instructions(voi · line 1702
Quoted 12 lines · Peter Zijlstra
+++
b/arch/x86/kernel/alternative.carch/x86/kernel/alternative.c +2
-4@@ -1697,14 +1697,14 @@ void __init alternative_instructions(voiapply_retpolines (__retpoline_sites, __retpoline_sites_end);apply_returns (__return_sites, __return_sites_end); -apply_alternatives (__alt_instructions, __alt_instructions_end); -/* -* Now all calls are established. Apply the call thunks if -* required. +* Adjust all CALL instructions to point to func()-10, including +* those in .altinstr_replacement.
Clarify 10 is hex: "func()-0x10" ?
Signature · 1 line
Josh
[tip: x86/core] x86/alternative: Simplify callthunk patching
In reply to Peter ZijlstraThe following commit has been merged into the x86/core branch of tip:
The following commit has been merged into the x86/core branch of tip:
Commit-ID: ab9fea59487d8b5149a323e2092b7c0f53994dd5 Gitweb: https://git.kernel.org/tip/ab9fea59487d8b5149a323e2092b7c0f53994dd5 Author: Peter Zijlstra <[email protected]> AuthorDate: Fri, 07 Feb 2025 13:15:35 +01:00 Committer: Peter Zijlstra <[email protected]> CommitterDate: Fri, 14 Feb 2025 10:32:06 +01:00
x86/alternative: Simplify callthunk patching
Now that paravirt call patching is implemented using alternatives, it is possible to avoid having to patch the alternative sites by including the altinstr_replacement calls in the call_sites list.
This means we're now stacking relative adjustments like so:
callthunks_patch_builtin_calls():
patches all function calls to target: func() -> func()-10 since the CALL accounting lives in the CALL_PADDING.
This explicitly includes .altinstr_replacement
alt_replace_call():
patches: x86_BUG() -> target()
this patching is done in a relative manner, and will preserve the above adjustment, meaning that with calldepth patching it will do: x86_BUG()-10 -> target()-10
apply_relocation():
does code relocation, and adjusts all RIP-relative instructions to the new location, also in a relative manner.
--- arch/x86/include/asm/alternative.h | 1 - arch/x86/kernel/alternative.c | 8 ++++---- arch/x86/kernel/callthunks.c | 13 ------------- arch/x86/kernel/module.c | 17 ++++++----------- tools/objtool/arch/x86/decode.c | 1 + tools/objtool/check.c | 12 ++---------- 6 files changed, 13 insertions(+), 39 deletions(-)
arch/x86/include/asm/alternative.h +0 -1
@@ -100,7 +100,6 @@ struct module;struct callthunk_sites {s32 *call_start, *call_end; -struct alt_instr *alt_start, *alt_end; };#ifdef CONFIG_CALL_THUNKS
arch/x86/kernel/alternative.c +4 -4
@@ -1705,14 +1705,14 @@ void __init alternative_instructions(void)apply_retpolines (__retpoline_sites, __retpoline_sites_end);apply_returns (__return_sites, __return_sites_end); -apply_alternatives (__alt_instructions, __alt_instructions_end); -/* -* Now all calls are established. Apply the call thunks if -* required. +* Adjust all CALL instructions to point to func()-10, including +* those in .altinstr_replacement. */ callthunks_patch_builtin_calls (); +apply_alternatives (__alt_instructions, __alt_instructions_end); +/* * Seal all functions that do not have their address taken. */
arch/x86/kernel/callthunks.c +0 -13
@@ -240,21 +240,10 @@ patch_call_sites(s32 *start, s32 *end, const struct core_text *ct) } static __init_or_module void -patch_alt_call_sites (struct alt_instr *start,struct alt_instr *end, -const struct core_text *ct) -{ -struct alt_instr *a; - -for (a = start; a < end; a++) -patch_call ((void *)&a->instr_offset + a->instr_offset, ct); -} - -static __init_or_module voidcallthunks_setup (struct callthunk_sites *cs,const struct core_text *ct) {prdbg ("Patching call sites %s\n" , ct->name);patch_call_sites (cs->call_start, cs->call_end, ct); -patch_alt_call_sites (cs->alt_start, cs->alt_end, ct);prdbg ("Patching call sites done%s\n" , ct->name); }
@@ -263,8 +252,6 @@ void __init callthunks_patch_builtin_calls(void)struct callthunk_sites cs = { .call_start = __call_sites, .call_end = __call_sites_end, - .alt_start = __alt_instructions, - .alt_end = __alt_instructions_end };if (!cpu_feature_enabled (X86_FEATURE_CALL_DEPTH ))
arch/x86/kernel/module.c +6 -11
@@ -275,12 +275,7 @@ int module_finalize(const Elf_Ehdr *hdr,void *rseg = (void *)returns->sh_addr;apply_returns (rseg, rseg + returns->sh_size); } -if (alt) { -/* patch .altinstructions */ -void *aseg = (void *)alt->sh_addr; -apply_alternatives (aseg, aseg + alt->sh_size); - } -if (calls || alt) { +if (calls) {struct callthunk_sites cs = {};if (calls) {
@@ -288,13 +283,13 @@ int module_finalize(const Elf_Ehdr *hdr, cs.call_end = (void *)calls->sh_addr + calls->sh_size; } -if (alt) { - cs.alt_start = (void *)alt->sh_addr; - cs.alt_end = (void *)alt->sh_addr + alt->sh_size; - } -callthunks_patch_module_calls (&cs, me); } +if (alt) { +/* patch .altinstructions */ +void *aseg = (void *)alt->sh_addr; +apply_alternatives (aseg, aseg + alt->sh_size); + }if (ibt_endbr) {void *iseg = (void *)ibt_endbr->sh_addr;apply_seal_endbr (iseg, iseg + ibt_endbr->sh_size);
tools/objtool/arch/x86/decode.c +1 -0
@@ -850,5 +850,6 @@ bool arch_is_rethunk(struct symbol *sym)bool arch_is_embedded_insn (struct symbol *sym) {return !strcmp (sym->name,"retbleed_return_thunk" ) || + !strcmp (sym->name,"srso_alias_safe_ret" ) || !strcmp (sym->name,"srso_safe_ret" ); }
tools/objtool/check.c +2 -10
@@ -1283,15 +1283,6 @@ static void annotate_call_site(struct objtool_file *file,if (!sym) sym = reloc->sym; -/* -* Alternative replacement code is just template code which is -* sometimes copied to the original instruction. For now, don't -* annotate it. (In the future we might consider annotating the -* original instruction if/when it ever makes sense to do so.) -*/ -if (!strcmp (insn->sec->name,".altinstr_replacement" )) -return ; -if (sym->static_call_tramp) {list_add_tail (&insn->call_node, &file->static_call_list);return ;
@@ -1349,7 +1340,8 @@ static void annotate_call_site(struct objtool_file *file,return ; } -if (insn->type ==INSN_CALL && !insn->sec->init) +if (insn->type ==INSN_CALL && !insn->sec->init && + !insn->_call_dest->embedded_insn)list_add_tail (&insn->call_node, &file->call_list);if (!sibling &&dead_end_function (file, sym))
Peter Zijlstra
maintainerPATCH 7/11 x86/traps: Cleanup and robustify decode_bug()
In reply to Peter ZijlstraNotably, don't attempt to decode an immediate when MOD == 3.
Additionally have it return the instruction length, such that WARN like bugs can more reliably skip to the correct instruction.
--- arch/x86/include/asm/bug.h | 5 +- arch/x86/include/asm/ibt.h | 4 +- arch/x86/kernel/traps.c | 82 ++++++++++++++++++++++++++++++++------------- 3 files changed, 65 insertions(+), 26 deletions(-)
arch/x86/include/asm/bug.h +3 -2
@@ -22,8 +22,9 @@#define SECOND_BYTE_OPCODE_UD2 0x0b#define BUG_NONE 0xffff -#define BUG_UD1 0xfffe -#define BUG_UD2 0xfffd +#define BUG_UD2 0xfffe +#define BUG_UD1 0xfffd +#define BUG_UD1_UBSAN 0xfffc#ifdef CONFIG_GENERIC_BUG
arch/x86/include/asm/ibt.h +2 -2
@@ -41,7 +41,7 @@ _ASM_PTR fname "\n\t" \".popsection\n\t" -static inline __attribute_const__ u32 gen_endbr(void) +static __always_inline __attribute_const__ u32 gen_endbr(void) {u32 endbr;
@@ -56,7 +56,7 @@ static inline __attribute_const__ u32 gereturn endbr; } -static inline __attribute_const__ u32 gen_endbr_poison(void) +static __always_inline __attribute_const__ u32 gen_endbr_poison(void) {/* * 4 byte NOP that isn't NOP4 (in fact it is OSP NOP3), such that it
arch/x86/kernel/traps.c +60 -22
119 lines not on this page · raw message
[tip: x86/core] x86/traps: Cleanup and robustify decode_bug()
The following commit has been merged into the x86/core branch of tip:
The following commit has been merged into the x86/core branch of tip:
Commit-ID: c20ad96c9a8f0aeaf4e4057730a22de2657ad0c2 Gitweb: https://git.kernel.org/tip/c20ad96c9a8f0aeaf4e4057730a22de2657ad0c2 Author: Peter Zijlstra <[email protected]> AuthorDate: Fri, 07 Feb 2025 13:15:36 +01:00 Committer: Peter Zijlstra <[email protected]> CommitterDate: Fri, 14 Feb 2025 10:32:06 +01:00
x86/traps: Cleanup and robustify decode_bug()
Notably, don't attempt to decode an immediate when MOD == 3.
Additionally have it return the instruction length, such that WARN like bugs can more reliably skip to the correct instruction.
--- arch/x86/include/asm/bug.h | 5 +- arch/x86/include/asm/ibt.h | 4 +- arch/x86/kernel/traps.c | 82 +++++++++++++++++++++++++++---------- 3 files changed, 65 insertions(+), 26 deletions(-)
arch/x86/include/asm/bug.h +3 -2
@@ -22,8 +22,9 @@#define SECOND_BYTE_OPCODE_UD2 0x0b#define BUG_NONE 0xffff -#define BUG_UD1 0xfffe -#define BUG_UD2 0xfffd +#define BUG_UD2 0xfffe +#define BUG_UD1 0xfffd +#define BUG_UD1_UBSAN 0xfffc#ifdef CONFIG_GENERIC_BUG
arch/x86/include/asm/ibt.h +2 -2
@@ -41,7 +41,7 @@ _ASM_PTR fname "\n\t" \".popsection\n\t" -static inline __attribute_const__ u32 gen_endbr(void) +static __always_inline __attribute_const__ u32 gen_endbr(void) {u32 endbr;
@@ -56,7 +56,7 @@ static inline __attribute_const__ u32 gen_endbr(void)return endbr; } -static inline __attribute_const__ u32 gen_endbr_poison(void) +static __always_inline __attribute_const__ u32 gen_endbr_poison(void) {/* * 4 byte NOP that isn't NOP4 (in fact it is OSP NOP3), such that it
arch/x86/kernel/traps.c +60 -22
119 lines not on this page · raw message
Peter Zijlstra
maintainerPATCH 8/11 x86/ibt: Clean up poison_endbr()
In reply to Peter ZijlstraBasically, get rid of the .warn argument and explicitly don't call the function when we know there isn't an endbr. This makes the calling code clearer.
Note: perhaps don't add functions to .cfi_sites when the function doesn't have endbr -- OTOH why would the compiler emit the prefix if it has already determined there are no indirect callers and has omitted the ENDBR instruction.
--- arch/x86/kernel/alternative.c | 43 +++++++++++++++++++++++++++++++++++------- 1 file changed, 36 insertions(+), 7 deletions(-)
arch/x86/kernel/alternative.c +36 -7
@@ -865,14 +865,12 @@ __noendbr bool is_endbr(u32 *val)static void poison_cfi (void *addr); -static void __init_or_module poison_endbr(void *addr, bool warn) +static void __init_or_module poison_endbr(void *addr) {u32 poison =gen_endbr_poison (); -if (!is_endbr (addr)) { -WARN_ON_ONCE (warn); +if (WARN_ON_ONCE (!is_endbr (addr)))return ; - }DPRINTK (ENDBR ,"ENDBR at: %pS (%px)" , addr, addr);
@@ -897,7 +895,7 @@ void __init_or_module noinline apply_seafor (s = start; s < end; s++) {void *addr = (void *)s + *s; -poison_endbr (addr, true); +poison_endbr (addr);if (IS_ENABLED (CONFIG_FINEIBT ))poison_cfi (addr -16 ); }
@@ -1200,6 +1198,14 @@ static int cfi_rewrite_preamble(s32 *stavoid *addr = (void *)s + *s;u32 hash; +/* +* When the function doesn't start with ENDBR the compiler will +* have determined there are no indirect calls to it and we +* don't need no CFI either. +*/ +if (!is_endbr (addr +16 )) +continue ; + hash =decode_preamble_hash (addr);if (WARN (!hash,"no CFI hash found at: %pS %px %*ph\n" , addr, addr,5 , addr))
@@ -1220,7 +1226,10 @@ static void cfi_rewrite_endbr(s32 *startfor (s = start; s < end; s++) {void *addr = (void *)s + *s; -poison_endbr (addr+16 , false); +if (!is_endbr (addr +16 )) +continue ; + +poison_endbr (addr +16 ); } }
@@ -1353,9 +1362,23 @@ static inline void poison_hash(void *addstatic void poison_cfi (void *addr) { +/* +* Compilers manage to be inconsistent with ENDBR vs __cfi prefixes, +* some (static) functions for which they can determine the address +* is never taken do not get a __cfi prefix, but *DO* get an ENDBR. +* +* As such, these functions will get sealed, but we need to be careful +* to not unconditionally scribble the previous function. +*/ switch (cfi_mode) {case CFI_FINEIBT :/* +* FineIBT prefix should start with an ENDBR. +*/ +if (!is_endbr (addr)) +break ; + +/* * __cfi_\func: * osp nopl (%rax) * subl $0, %r10d
@@ -1363,12 +1386,18 @@ static void poison_cfi(void *addr)* ud2 * 1: nop */ -poison_endbr (addr, false); +poison_endbr (addr);poison_hash (addr + fineibt_preamble_hash);break ;case CFI_KCFI :/* +* kCFI prefix should start with a valid hash. +*/ +if (!decode_preamble_hash (addr)) +break ; + +/* * __cfi_\func: * movl $0, %eax * .skip 11, 0x90
[tip: x86/core] x86/ibt: Clean up poison_endbr()
The following commit has been merged into the x86/core branch of tip:
The following commit has been merged into the x86/core branch of tip:
Commit-ID: c4239a72a29d58a4377c2db8583c24f9e2b79d01 Gitweb: https://git.kernel.org/tip/c4239a72a29d58a4377c2db8583c24f9e2b79d01 Author: Peter Zijlstra <[email protected]> AuthorDate: Fri, 07 Feb 2025 13:15:37 +01:00 Committer: Peter Zijlstra <[email protected]> CommitterDate: Fri, 14 Feb 2025 10:32:06 +01:00
x86/ibt: Clean up poison_endbr()
Basically, get rid of the .warn argument and explicitly don't call the function when we know there isn't an endbr. This makes the calling code clearer.
Note: perhaps don't add functions to .cfi_sites when the function doesn't have endbr -- OTOH why would the compiler emit the prefix if it has already determined there are no indirect callers and has omitted the ENDBR instruction.
--- arch/x86/kernel/alternative.c | 43 ++++++++++++++++++++++++++++------ 1 file changed, 36 insertions(+), 7 deletions(-)
arch/x86/kernel/alternative.c +36 -7
@@ -865,14 +865,12 @@ Efault:static void poison_cfi (void *addr); -static void __init_or_module poison_endbr(void *addr, bool warn) +static void __init_or_module poison_endbr(void *addr) {u32 poison =gen_endbr_poison (); -if (!is_endbr (addr)) { -WARN_ON_ONCE (warn); +if (WARN_ON_ONCE (!is_endbr (addr)))return ; - }DPRINTK (ENDBR ,"ENDBR at: %pS (%px)" , addr, addr);
@@ -897,7 +895,7 @@ void __init_or_module noinline apply_seal_endbr(s32 *start, s32 *end)for (s = start; s < end; s++) {void *addr = (void *)s + *s; -poison_endbr (addr, true); +poison_endbr (addr);if (IS_ENABLED (CONFIG_FINEIBT ))poison_cfi (addr -16 ); }
@@ -1200,6 +1198,14 @@ static int cfi_rewrite_preamble(s32 *start, s32 *end)void *addr = (void *)s + *s;u32 hash; +/* +* When the function doesn't start with ENDBR the compiler will +* have determined there are no indirect calls to it and we +* don't need no CFI either. +*/ +if (!is_endbr (addr +16 )) +continue ; + hash =decode_preamble_hash (addr);if (WARN (!hash,"no CFI hash found at: %pS %px %*ph\n" , addr, addr,5 , addr))
@@ -1220,7 +1226,10 @@ static void cfi_rewrite_endbr(s32 *start, s32 *end)for (s = start; s < end; s++) {void *addr = (void *)s + *s; -poison_endbr (addr+16 , false); +if (!is_endbr (addr +16 )) +continue ; + +poison_endbr (addr +16 ); } }
@@ -1353,9 +1362,23 @@ static inline void poison_hash(void *addr)static void poison_cfi (void *addr) { +/* +* Compilers manage to be inconsistent with ENDBR vs __cfi prefixes, +* some (static) functions for which they can determine the address +* is never taken do not get a __cfi prefix, but *DO* get an ENDBR. +* +* As such, these functions will get sealed, but we need to be careful +* to not unconditionally scribble the previous function. +*/ switch (cfi_mode) {case CFI_FINEIBT :/* +* FineIBT prefix should start with an ENDBR. +*/ +if (!is_endbr (addr)) +break ; + +/* * __cfi_\func: * osp nopl (%rax) * subl $0, %r10d
@@ -1363,12 +1386,18 @@ static void poison_cfi(void *addr)* ud2 * 1: nop */ -poison_endbr (addr, false); +poison_endbr (addr);poison_hash (addr + fineibt_preamble_hash);break ;case CFI_KCFI :/* +* kCFI prefix should start with a valid hash. +*/ +if (!decode_preamble_hash (addr)) +break ; + +/* * __cfi_\func: * movl $0, %eax * .skip 11, 0x90
Peter Zijlstra
maintainerPATCH 9/11 x86/early_printk: Harden early_serial
In reply to Peter ZijlstraScott found that mem32_serial_in() is an ideal speculation gadget, an indirectly callable function that takes an adddress and offset and immediately does a load.
Use static_call() to take away the need for indirect calls and explicitly seal the functions to ensure they're not callable on IBT enabled parts.
--- arch/x86/kernel/early_printk.c | 51 ++++++++++++++++++++--------------------- 1 file changed, 25 insertions(+), 26 deletions(-)
arch/x86/kernel/early_printk.c +25 -26
@@ -19,6 +19,7 @@#include <linux/usb/ehci_def.h> #include <linux/usb/xhci-dbgp.h> #include <asm/pci_x86.h> +#include <linux/static_call.h> /* Simple VGA output */ #define VGABASE (__ISA_IO_base + 0xb8000)
@@ -94,26 +95,28 @@ static unsigned long early_serial_base =#define DLL 0/* Divisor Latch Low */ #define DLH 1/* Divisor latch High */ -static unsigned int io_serial_in (unsigned long addr,int offset) +static __noendbr unsigned int io_serial_in(unsigned long addr, int offset) {return inb (addr + offset); } +ANNOTATE_NOENDBR_SYM (io_serial_in); -static void io_serial_out (unsigned long addr,int offset,int value) +static __noendbr void io_serial_out(unsigned long addr, int offset, int value) {outb (value, addr + offset); } +ANNOTATE_NOENDBR_SYM (io_serial_out); -static unsigned int (*serial_in)(unsigned long addr,int offset) = io_serial_in; -static void (*serial_out)(unsigned long addr,int offset,int value) = io_serial_out; +DEFINE_STATIC_CALL (serial_in, io_serial_in); +DEFINE_STATIC_CALL (serial_out, io_serial_out);static int early_serial_putc (unsigned char ch) {unsigned timeout =0xffff ; -while ((serial_in (early_serial_base,LSR ) &XMTRDY ) ==0 && --timeout) +while ((static_call (serial_in)(early_serial_base,LSR ) &XMTRDY ) ==0 && --timeout)cpu_relax (); -serial_out (early_serial_base,TXR , ch); +static_call (serial_out)(early_serial_base,TXR , ch);return timeout ?0 :-1 ; }
@@ -131,16 +134,16 @@ static __init void early_serial_hw_init( {unsigned char c; -serial_out (early_serial_base,LCR ,0x3 );/* 8n1 */ -serial_out (early_serial_base,IER ,0 );/* no interrupt */ -serial_out (early_serial_base,FCR ,0 );/* no fifo */ -serial_out (early_serial_base,MCR ,0x3 );/* DTR + RTS */ - - c =serial_in (early_serial_base,LCR ); -serial_out (early_serial_base,LCR , c |DLAB ); -serial_out (early_serial_base,DLL , divisor &0xff ); -serial_out (early_serial_base,DLH , (divisor >>8 ) &0xff ); -serial_out (early_serial_base,LCR , c & ~DLAB ); +static_call (serial_out)(early_serial_base,LCR ,0x3 );/* 8n1 */ +static_call (serial_out)(early_serial_base,IER ,0 );/* no interrupt */ +static_call (serial_out)(early_serial_base,FCR ,0 );/* no fifo */ +static_call (serial_out)(early_serial_base,MCR ,0x3 );/* DTR + RTS */ + + c =static_call (serial_in)(early_serial_base,LCR ); +static_call (serial_out)(early_serial_base,LCR , c |DLAB ); +static_call (serial_out)(early_serial_base,DLL , divisor &0xff ); +static_call (serial_out)(early_serial_base,DLH , (divisor >>8 ) &0xff ); +static_call (serial_out)(early_serial_base,LCR , c & ~DLAB ); }#define DEFAULT_BAUD 9600
@@ -183,28 +186,26 @@ static __init void early_serial_init(cha/* Convert from baud to divisor value */ divisor =115200 / baud; -/* These will always be IO based ports */ - serial_in = io_serial_in; - serial_out = io_serial_out; -/* Set up the HW */ early_serial_hw_init (divisor); }#ifdef CONFIG_PCI -static void mem32_serial_out (unsigned long addr,int offset,int value) +static __noendbr void mem32_serial_out(unsigned long addr, int offset, int value) {u32 __iomem *vaddr = (u32 __iomem *)addr;/* shift implied by pointer type */ writel (value, vaddr + offset); } +ANNOTATE_NOENDBR_SYM (mem32_serial_out); -static unsigned int mem32_serial_in (unsigned long addr,int offset) +static __noendbr unsigned int mem32_serial_in(unsigned long addr, int offset) {u32 __iomem *vaddr = (u32 __iomem *)addr;/* shift implied by pointer type */ return readl (vaddr + offset); } +ANNOTATE_NOENDBR_SYM (mem32_serial_in);/* * early_pci_serial_init()
@@ -278,15 +279,13 @@ static __init void early_pci_serial_init*/ if ((bar0 &PCI_BASE_ADDRESS_SPACE ) ==PCI_BASE_ADDRESS_SPACE_IO ) {/* it is IO mapped */ - serial_in = io_serial_in; - serial_out = io_serial_out; early_serial_base = bar0 &PCI_BASE_ADDRESS_IO_MASK ;write_pci_config (bus, slot, func,PCI_COMMAND , cmdreg|PCI_COMMAND_IO ); }else {/* It is memory mapped - assume 32-bit alignment */ - serial_in = mem32_serial_in; - serial_out = mem32_serial_out; +static_call_update (serial_in, mem32_serial_in); +static_call_update (serial_out, mem32_serial_out);/* WARNING! assuming the address is always in the first 4G */ early_serial_base = (unsigned long )early_ioremap (bar0 &PCI_BASE_ADDRESS_MEM_MASK ,0x10 );
[tip: x86/core] x86/early_printk: Harden early_serial
The following commit has been merged into the x86/core branch of tip:
The following commit has been merged into the x86/core branch of tip:
Commit-ID: 306859de59e59f88662b6b56ff2ce3bbb1e375bb Gitweb: https://git.kernel.org/tip/306859de59e59f88662b6b56ff2ce3bbb1e375bb Author: Peter Zijlstra <[email protected]> AuthorDate: Fri, 07 Feb 2025 13:15:38 +01:00 Committer: Peter Zijlstra <[email protected]> CommitterDate: Fri, 14 Feb 2025 10:32:07 +01:00
x86/early_printk: Harden early_serial
Scott found that mem32_serial_in() is an ideal speculation gadget, an indirectly callable function that takes an adddress and offset and immediately does a load.
Use static_call() to take away the need for indirect calls and explicitly seal the functions to ensure they're not callable on IBT enabled parts.
--- arch/x86/kernel/early_printk.c | 49 ++++++++++++++++----------------- 1 file changed, 24 insertions(+), 25 deletions(-)
arch/x86/kernel/early_printk.c +24 -25
@@ -19,6 +19,7 @@#include <linux/usb/ehci_def.h> #include <linux/usb/xhci-dbgp.h> #include <asm/pci_x86.h> +#include <linux/static_call.h> /* Simple VGA output */ #define VGABASE (__ISA_IO_base + 0xb8000)
@@ -94,26 +95,28 @@ static unsigned long early_serial_base = 0x3f8; /* ttyS0 */#define DLL 0/* Divisor Latch Low */ #define DLH 1/* Divisor latch High */ -static unsigned int io_serial_in (unsigned long addr,int offset) +static __noendbr unsigned int io_serial_in(unsigned long addr, int offset) {return inb (addr + offset); } +ANNOTATE_NOENDBR_SYM (io_serial_in); -static void io_serial_out (unsigned long addr,int offset,int value) +static __noendbr void io_serial_out(unsigned long addr, int offset, int value) {outb (value, addr + offset); } +ANNOTATE_NOENDBR_SYM (io_serial_out); -static unsigned int (*serial_in)(unsigned long addr,int offset) = io_serial_in; -static void (*serial_out)(unsigned long addr,int offset,int value) = io_serial_out; +DEFINE_STATIC_CALL (serial_in, io_serial_in); +DEFINE_STATIC_CALL (serial_out, io_serial_out);static int early_serial_putc (unsigned char ch) {unsigned timeout =0xffff ; -while ((serial_in (early_serial_base,LSR ) &XMTRDY ) ==0 && --timeout) +while ((static_call (serial_in)(early_serial_base,LSR ) &XMTRDY ) ==0 && --timeout)cpu_relax (); -serial_out (early_serial_base,TXR , ch); +static_call (serial_out)(early_serial_base,TXR , ch);return timeout ?0 :-1 ; }
@@ -131,16 +134,16 @@ static __init void early_serial_hw_init(unsigned divisor) {unsigned char c; -serial_out (early_serial_base,LCR ,0x3 );/* 8n1 */ -serial_out (early_serial_base,IER ,0 );/* no interrupt */ -serial_out (early_serial_base,FCR ,0 );/* no fifo */ -serial_out (early_serial_base,MCR ,0x3 );/* DTR + RTS */ +static_call (serial_out)(early_serial_base,LCR ,0x3 );/* 8n1 */ +static_call (serial_out)(early_serial_base,IER ,0 );/* no interrupt */ +static_call (serial_out)(early_serial_base,FCR ,0 );/* no fifo */ +static_call (serial_out)(early_serial_base,MCR ,0x3 );/* DTR + RTS */ - c =serial_in (early_serial_base,LCR ); -serial_out (early_serial_base,LCR , c |DLAB ); -serial_out (early_serial_base,DLL , divisor &0xff ); -serial_out (early_serial_base,DLH , (divisor >>8 ) &0xff ); -serial_out (early_serial_base,LCR , c & ~DLAB ); + c =static_call (serial_in)(early_serial_base,LCR ); +static_call (serial_out)(early_serial_base,LCR , c |DLAB ); +static_call (serial_out)(early_serial_base,DLL , divisor &0xff ); +static_call (serial_out)(early_serial_base,DLH , (divisor >>8 ) &0xff ); +static_call (serial_out)(early_serial_base,LCR , c & ~DLAB ); }#define DEFAULT_BAUD 9600
@@ -183,28 +186,26 @@ static __init void early_serial_init(char *s)/* Convert from baud to divisor value */ divisor =115200 / baud; -/* These will always be IO based ports */ - serial_in = io_serial_in; - serial_out = io_serial_out; -/* Set up the HW */ early_serial_hw_init (divisor); }#ifdef CONFIG_PCI -static void mem32_serial_out (unsigned long addr,int offset,int value) +static __noendbr void mem32_serial_out(unsigned long addr, int offset, int value) {u32 __iomem *vaddr = (u32 __iomem *)addr;/* shift implied by pointer type */ writel (value, vaddr + offset); } +ANNOTATE_NOENDBR_SYM (mem32_serial_out); -static unsigned int mem32_serial_in (unsigned long addr,int offset) +static __noendbr unsigned int mem32_serial_in(unsigned long addr, int offset) {u32 __iomem *vaddr = (u32 __iomem *)addr;/* shift implied by pointer type */ return readl (vaddr + offset); } +ANNOTATE_NOENDBR_SYM (mem32_serial_in);/* * early_pci_serial_init()
@@ -278,15 +279,13 @@ static __init void early_pci_serial_init(char *s)*/ if ((bar0 &PCI_BASE_ADDRESS_SPACE ) ==PCI_BASE_ADDRESS_SPACE_IO ) {/* it is IO mapped */ - serial_in = io_serial_in; - serial_out = io_serial_out; early_serial_base = bar0 &PCI_BASE_ADDRESS_IO_MASK ;write_pci_config (bus, slot, func,PCI_COMMAND , cmdreg|PCI_COMMAND_IO ); }else {/* It is memory mapped - assume 32-bit alignment */ - serial_in = mem32_serial_in; - serial_out = mem32_serial_out; +static_call_update (serial_in, mem32_serial_in); +static_call_update (serial_out, mem32_serial_out);/* WARNING! assuming the address is always in the first 4G */ early_serial_base = (unsigned long )early_ioremap (bar0 &PCI_BASE_ADDRESS_MEM_MASK ,0x10 );
Peter Zijlstra
maintainerPATCH 10/11 x86: BHI stubs
In reply to Peter Zijlstra--- arch/x86/include/asm/cfi.h | 3 + arch/x86/lib/Makefile | 3 - arch/x86/lib/bhi.S | 130 +++++++++++++++++++++++++++++++++++++++++++++ 3 files changed, 135 insertions(+), 1 deletion(-)
arch/x86/include/asm/cfi.h +3 -0
@@ -101,6 +101,9 @@ enum cfi_mode {extern enum cfi_mode cfi_mode; +typedef u8 bhi_thunk [32 ]; +extern bhi_thunk __bhi_args[]; +struct pt_regs ;#ifdef CONFIG_CFI_CLANG
arch/x86/lib/Makefile +2 -1
@@ -66,5 +66,6 @@ endif lib-y += clear_page_64.o copy_page_64.o lib-y += memmove_64.o memset_64.o lib-y += copy_user_64.o copy_user_uncached_64.o - lib-y += cmpxchg16b_emu.o + lib-y += cmpxchg16b_emu.o + lib-y += bhi.o endif
arch/x86/lib/bhi.S new file +130 -0
130 lines not on this page · raw message
Peter Zijlstra
maintainerPATCH 11/11 x86/fineibt: Add FineIBT+BHI mitigation
In reply to Peter ZijlstraDue to FineIBT weakness, add an additional mitigation for BHI.
Relies on clang-cfi to emit an additional piece of magic in the kCFI pre-amble:
https://github.com/llvm/llvm-project/commit/e223485c9b38a5579991b8cebb6a200153eee245
Which encodes the number of argument registers in the target register:
movl 0x12345678, %reg // CFI hash
XXX get Scott to write a blurb about how this works to mitigate BHI
FineIBT FineIBT+BHI
__cfi_foo: __cfi_foo:
endbr64 endbr64
subl $0x12345678, %r10d subl $0x12345678, %r10d
jz 1f call __bhi_args[\reg]
ud2
1: nop
foo: foo:
osp nop3 osp nop3
... ...--- Makefile | 3 ++ arch/x86/Kconfig | 8 +++++ arch/x86/include/asm/cfi.h | 9 +++++- arch/x86/kernel/alternative.c | 60 +++++++++++++++++++++++++++++++++++------- arch/x86/net/bpf_jit_comp.c | 32 +++++++++++++++------- kernel/bpf/core.c | 4 +- 6 files changed, 94 insertions(+), 22 deletions(-)
Makefile +3 -0
@@ -1014,6 +1014,9 @@ CC_FLAGS_CFI := -fsanitize=kcfi ifdef CONFIG_CFI_ICALL_NORMALIZE_INTEGERS CC_FLAGS_CFI += -fsanitize-cfi-icall-experimental-normalize-integers endif +ifdef CONFIG_FINEIBT_BHI + CC_FLAGS_CFI += -fsanitize-kcfi-arity +endif ifdef CONFIG_RUST # Always pass -Zsanitizer-cfi-normalize-integers as CONFIG_RUST selects # CONFIG_CFI_ICALL_NORMALIZE_INTEGERS.
arch/x86/Kconfig +8 -0
@@ -2473,6 +2473,10 @@ config CC_HAS_RETURN_THUNK config CC_HAS_ENTRY_PADDING def_bool $(cc-option,-fpatchable-function-entry=16,16) +config CC_HAS_KCFI_ARITY + def_bool $(cc-option,-fsanitize=kcfi -fsanitize-kcfi-arity) + depends on CC_IS_CLANG && !RUST + config FUNCTION_PADDING_CFI int default 59 if FUNCTION_ALIGNMENT_64B
@@ -2498,6 +2502,10 @@ config FINEIBT depends on X86_KERNEL_IBT && CFI_CLANG && MITIGATION_RETPOLINE select CALL_PADDING +config FINEIBT_BHI + def_bool y + depends on FINEIBT && CC_HAS_KCFI_ARITY + config HAVE_CALL_THUNKS def_bool y depends on CC_HAS_ENTRY_PADDING && MITIGATION_RETHUNK && OBJTOOL
arch/x86/include/asm/cfi.h +8 -1
@@ -72,7 +72,7 @@* __cfi_foo: * endbr64 * subl 0x12345678, %r10d -* jz foo +* jz foo # call __bhi_args[\reg] when CONFIG_FINEIBT_BHI * ud2 * nop * foo:
@@ -97,6 +97,7 @@ enum cfi_mode {CFI_OFF ,/* Taditional / IBT depending on .config */ CFI_KCFI ,/* Optionally CALL_PADDING, IBT, RETPOLINE */ CFI_FINEIBT ,/* see arch/x86/kernel/alternative.c */ + CFI_FINEIBT_BHI, };extern enum cfi_mode cfi_mode;
@@ -116,6 +117,7 @@ static inline int cfi_get_offset(void) {switch (cfi_mode) {case CFI_FINEIBT : +case CFI_FINEIBT_BHI :return 16 ;case CFI_KCFI :if (IS_ENABLED (CONFIG_CALL_PADDING ))
@@ -128,6 +130,7 @@ static inline int cfi_get_offset(void)#define cfi_get_offset cfi_get_offsetextern u32 cfi_get_func_hash (void *func); +extern int cfi_get_func_arity (void *func);#else static inline enum bug_trap_type handle_cfi_failure (struct pt_regs *regs)
@@ -140,6 +143,10 @@ static inline u32 cfi_get_func_hash(void {return 0 ; } +static inline int cfi_get_func_arity (void *func) +{ +return 0 ; +}#endif /* CONFIG_CFI_CLANG */ #if HAS_KERNEL_IBT ==1
arch/x86/kernel/alternative.c +51 -9
158 lines not on this page · raw message
arch/x86/net/bpf_jit_comp.c +22 -10
@@ -410,16 +410,21 @@ static void emit_nops(u8 **pprog, int le* Emit the various CFI preambles, see asm/cfi.h and the comments about FineIBT * in arch/x86/kernel/alternative.c */ +static int emit_call (u8 **pprog,void *func,void *ip); -static void emit_fineibt (u8 **pprog,u32 hash) +static void emit_fineibt (u8 **pprog,u8 *ip,u32 hash,int reg) {u8 *prog = *pprog;EMIT_ENDBR ();EMIT3_off32 (0x41 ,0x81 ,0xea , hash);/* subl $hash, %r10d */ -EMIT2 (0x74 ,0x07 );/* jz.d8 +7 */ -EMIT2 (0x0f ,0x0b );/* ud2 */ -EMIT1 (0x90 );/* nop */ +if (cfi_mode ==CFI_FINEIBT_BHI && reg) { +emit_call (&prog, __bhi_args[reg], ip); + }else { +EMIT2 (0x74 ,0x02 );/* jz.d8 +2 */ +EMIT2 (0x0f ,0x0b );/* ud2 */ +EMIT1 (0x90 );/* nop */ + }EMIT_ENDBR_POISON (); *pprog = prog;
@@ -448,13 +453,14 @@ static void emit_kcfi(u8 **pprog, u32 ha *pprog = prog; } -static void emit_cfi (u8 **pprog,u32 hash) +static void emit_cfi (u8 **pprog,u8 *ip,u32 hash,int reg) {u8 *prog = *pprog;switch (cfi_mode) {case CFI_FINEIBT : -emit_fineibt (&prog, hash); +case CFI_FINEIBT_BHI : +emit_fineibt (&prog, ip, hash, reg);break ;case CFI_KCFI :
@@ -505,13 +511,17 @@ static void emit_prologue_tail_call(u8 ** bpf_tail_call helper will skip the first X86_TAIL_CALL_OFFSET bytes * while jumping to another program */ -static void emit_prologue (u8 **pprog,u32 stack_depth,bool ebpf_from_cbpf, +static void emit_prologue (u8 **pprog,u8 *ip,u32 stack_depth,bool ebpf_from_cbpf,bool tail_call_reachable,bool is_subprog,bool is_exception_cb) {u8 *prog = *pprog; -emit_cfi (&prog, is_subprog ? cfi_bpf_subprog_hash : cfi_bpf_hash); +if (is_subprog) { +emit_cfi (&prog, ip, cfi_bpf_subprog_hash,5 ); + }else { +emit_cfi (&prog, ip, cfi_bpf_hash,1 ); + }/* BPF trampoline can be made to work without these nops, * but let's waste 5 bytes for now and optimize later */
@@ -1480,7 +1490,7 @@ static int do_jit(struct bpf_prog *bpf_pdetect_reg_usage (insn, insn_cnt, callee_regs_used); -emit_prologue (&prog, stack_depth, +emit_prologue (&prog, image, stack_depth,bpf_prog_was_classic (bpf_prog), tail_call_reachable,bpf_is_subprog (bpf_prog), bpf_prog->aux->exception_cb);/* Exception callback will clobber callee regs for its own use, and
@@ -3047,7 +3057,9 @@ static int __arch_prepare_bpf_trampoline/* * Indirect call for bpf_struct_ops */ -emit_cfi (&prog,cfi_get_func_hash (func_addr)); +emit_cfi (&prog, image, +cfi_get_func_hash (func_addr), +cfi_get_func_arity (func_addr)); }else {/* * Direct-call fentry stub, as such it needs accounting for the
kernel/bpf/core.c +2 -2
@@ -707,7 +707,7 @@ void bpf_prog_kallsyms_add(struct bpf_pr* When FineIBT, code in the __cfi_foo() symbols can get executed * and hence unwinder needs help. */ -if (cfi_mode !=CFI_FINEIBT ) +if (cfi_mode !=CFI_FINEIBT && cfi_mode !=CFI_FINEIBT_BHI )return ; snprintf(fp->aux->ksym_prefix.name, KSYM_NAME_LEN,
@@ -727,7 +727,7 @@ void bpf_prog_kallsyms_del(struct bpf_prbpf_ksym_del (&fp->aux->ksym);#ifdef CONFIG_FINEIBT -if (cfi_mode !=CFI_FINEIBT ) +if (cfi_mode !=CFI_FINEIBT && cfi_mode !=CFI_FINEIBT_BHI )return ;bpf_ksym_del (&fp->aux->ksym_prefix);#endif
Re: [PATCH 11/11] x86/fineibt: Add FineIBT+BHI mitigation
arch/x86/kernel/alternative.c static __init int cfi_parse_cmdline(char · line 1043
Quoted 11 lines · Peter Zijlstra
Due to FineIBT weakness, add an additional mitigation for BHI. [...]
arch/x86/kernel/alternative.c +2
-0@@ -1020,6 +1036,8 @@ static __init int cfi_parse_cmdline(char cfi_mode =CFI_KCFI ; }else if (!strcmp (str,"fineibt" )) { cfi_mode =CFI_FINEIBT ; + }else if (IS_ENABLED (CONFIG_FINEIBT_BHI ) && !strcmp (str,"fineibt+bhi" )) { + cfi_mode =CFI_FINEIBT_BHI ; }else if (!strcmp (str,"norand" )) { cfi_rand = false; }else {
While looking at FineIBT vs entry, I noticed that FineIBT+BHI must be explicitly selected at boot. Did you want it to be enabled automatically when the compiler supports it and FineIBT is enabled? Does there need to be a check for BHI added?
Signature · 1 line
Kees Cook
Peter Zijlstra
maintainerRe: [PATCH 11/11] x86/fineibt: Add FineIBT+BHI mitigation
Quoted 17 lines · Kees Cook
arch/x86/kernel/alternative.c static __init int cfi_parse_cmdline(char · line 1043
Quoted 11 lines · Peter Zijlstra
Due to FineIBT weakness, add an additional mitigation for BHI. [...]
arch/x86/kernel/alternative.c +2
-0@@ -1020,6 +1036,8 @@ static __init int cfi_parse_cmdline(char cfi_mode =CFI_KCFI ; }else if (!strcmp (str,"fineibt" )) { cfi_mode =CFI_FINEIBT ; + }else if (IS_ENABLED (CONFIG_FINEIBT_BHI ) && !strcmp (str,"fineibt+bhi" )) { + cfi_mode =CFI_FINEIBT_BHI ; }else if (!strcmp (str,"norand" )) { cfi_rand = false; }else {While looking at FineIBT vs entry, I noticed that FineIBT+BHI must be explicitly selected at boot. Did you want it to be enabled automatically when the compiler supports it and FineIBT is enabled? Does there need to be a check for BHI added?
Yes, it needs to be tied in with the whole speculation mitigation crap. Didn't want to bother with all that just yet though.
Hi Peter,
Quoted 22 lines · Peter Zijlstra
Hi!
Respin of the FineIBT-BHI patches.
Scott has managed to get LLVM bits merged:
https://github.com/llvm/llvm-project/commit/e223485c9b38a5579991b8cebb6a200153eee245
Which prompted me to update these patches once again.
They boot and build the next kernel on my ADL when booted with: cfi=fineibt+bhi
Aside from the last two patches -- which implement the FineIBT-BHI scheme proper -- I'm planning on getting these patches merged 'soon'.
Scott, what those last two patches need, aside from a lot more testing, is a coherent writeup of how the mitigation works and ideally also a few numbers proving the performance gains are worth it.
Last version at:
Quoted 7 lines
https://lore.kernel.org/all/[email protected]/T/#u
Current patches:
git://git.kernel.org/pub/scm/linux/kernel/git/peterz/queue.gitx86/fineibt-bhiPatches apply on top of tip/master.
I gave this a spin with a ToT Clang; LKDTM 's CFI_FORWARD_PROTO test now traps in __bhi_args_1 as expected, and the changes look good to me. The is_endbr() clean-up also fixes the gendwarfksyms+ftrace build issue reported earlier. Feel free to add:
One thing I realized is that CONFIG_CFI_PERMISSIVE doesn't actually do anything when FineIBT is used since we lose track of CFI trap locations. I'm not sure if that's worth fixing, but perhaps we could disable FineIBT when permissive mode is enabled to avoid confusion?
Sami
Peter Zijlstra
maintainerQuoted 31 lines · Sami Tolvanen
Hi Peter,
Quoted 28 lines · Peter Zijlstra
Hi!
Respin of the FineIBT-BHI patches.
Scott has managed to get LLVM bits merged:
https://github.com/llvm/llvm-project/commit/e223485c9b38a5579991b8cebb6a200153eee245
Which prompted me to update these patches once again.
They boot and build the next kernel on my ADL when booted with: cfi=fineibt+bhi
Aside from the last two patches -- which implement the FineIBT-BHI scheme proper -- I'm planning on getting these patches merged 'soon'.
Scott, what those last two patches need, aside from a lot more testing, is a coherent writeup of how the mitigation works and ideally also a few numbers proving the performance gains are worth it.
Last version at:
https://lore.kernel.org/all/[email protected]/T/#u
Current patches:
git://git.kernel.org/pub/scm/linux/kernel/git/peterz/queue.gitx86/fineibt-bhi
Quoted 8 lines
Quoted 1 line
Patches apply on top of tip/master.
I gave this a spin with a ToT Clang; LKDTM 's
CFI_FORWARD_PROTOtest now traps in__bhi_args_1as expected, and the changes look good to me. Theis_endbr()clean-up also fixes the gendwarfksyms+ftrace build issue reported earlier. Feel free to add:Reviewed-by Sami Tolvanen <[email protected]>
Thanks!
Quoted 4 lines
One thing I realized is that
CONFIG_CFI_PERMISSIVEdoesn't actually do anything when FineIBT is used since we lose track of CFI trap locations. I'm not sure if that's worth fixing, but perhaps we could disable FineIBT when permissive mode is enabled to avoid confusion?
Hmm, yeah, that's one thing I keep forgetting about. Let me try and fix it and see how ugly it gets before offering an opinion :-)
Peter Zijlstra
maintainerQuoted 7 lines · Peter Zijlstra
Quoted 4 lines
One thing I realized is that
CONFIG_CFI_PERMISSIVEdoesn't actually do anything when FineIBT is used since we lose track of CFI trap locations. I'm not sure if that's worth fixing, but perhaps we could disable FineIBT when permissive mode is enabled to avoid confusion?Hmm, yeah, that's one thing I keep forgetting about. Let me try and fix it and see how ugly it gets before offering an opinion :-)
Completely untested and on top of this series minus the last two patches -- basically what I just pushed into:
git://git.kernel.org/pub/scm/linux/kernel/git/peterz/queue.git x86/core
WDYT?
---
arch/x86/kernel/alternative.c +35 -0
@@ -1410,6 +1410,36 @@ static void poison_cfi(void *addr) } } +bool decode_fineibt_insn (struct pt_regs *regs,unsigned long *target,u32 *type) +{ +/* +* FineIBT preamble: +* +* __cfi_foo: +* endbr64 +* subl $0x12345678, %r10d +* jz 1f +* ud2 +* 1: nop +* foo: +* +* regs->ip points to the UD2 instruction. +*/ +unsigned long addr = regs->ip - (4 +7 +2 ); +u32 hash; + +if (!is_endbr ((void *)addr)) { +Efault: +return false; + } + + *target = addr + fineibt_preamble_size; + +__get_kernel_nofault (&hash, *(u32 *)(addr + fineibt_preamble_hash), u32, Efault); + *type = (u32 )regs->r10 + hash; +return true; +} +#else static void __apply_fineibt(s32 *start_retpoline, s32 *end_retpoline,
@@ -1421,6 +1451,11 @@ static void __apply_fineibt(s32 *start_retpoline, s32 *end_retpoline,static void poison_cfi (void *addr) { }#endif +bool decode_fineibt_insn (struct pt_regs *regs,unsigned long *target,u32 *type) +{ +return false; +} +#endif void apply_fineibt(s32 *start_retpoline, s32 *end_retpoline,
arch/x86/kernel/cfi.c +20 -4
@@ -61,6 +61,8 @@ static bool decode_cfi_insn(struct pt_regs *regs, unsigned long *target,return true; } +extern bool decode_fineibt_insn (struct pt_regs *regs,unsigned long *target,u32 *type); +/* * Checks if a ud2 trap is because of a CFI failure, and handles the trap * if needed. Returns a bug_trap_type value similarly to report_bug.
@@ -70,11 +72,25 @@ enum bug_trap_type handle_cfi_failure(struct pt_regs *regs)unsigned long target;u32 type; -if (!is_cfi_trap (regs->ip)) -return BUG_TRAP_TYPE_NONE ; +switch (cfi_mode) { +case CFI_KCFI : +if (!is_cfi_trap (regs->ip)) +return BUG_TRAP_TYPE_NONE ; + +if (!decode_cfi_insn (regs, &target, &type)) +return report_cfi_failure_noaddr (regs, regs->ip); + +break ; -if (!decode_cfi_insn (regs, &target, &type)) -return report_cfi_failure_noaddr (regs, regs->ip); +case CFI_FINEIBT : +if (!decode_fineibt_insn (regs, &target, &type)) +return BUG_TRAP_TYPE_NONE ; + +break ; + +default : +return BUG_TRAP_TYPE_NONE ; + }return report_cfi_failure (regs, regs->ip, &target, type); }
Sami Tolvanen
Reviewed-byQuoted 43 lines · Peter Zijlstra
Quoted 7 lines · Peter Zijlstra
Quoted 4 lines
One thing I realized is that
CONFIG_CFI_PERMISSIVEdoesn't actually do anything when FineIBT is used since we lose track of CFI trap locations. I'm not sure if that's worth fixing, but perhaps we could disable FineIBT when permissive mode is enabled to avoid confusion?Hmm, yeah, that's one thing I keep forgetting about. Let me try and fix it and see how ugly it gets before offering an opinion :-)
Completely untested and on top of this series minus the last two patches -- basically what I just pushed into:
git://git.kernel.org/pub/scm/linux/kernel/git/peterz/queue.gitx86/coreWDYT?
---
arch/x86/kernel/alternative.c +16
-0@@ -1410,6 +1410,36 @@ static void poison_cfi(void *addr) } } +bool decode_fineibt_insn (struct pt_regs *regs,unsigned long *target,u32 *type) +{ +/* +* FineIBT preamble: +* +* __cfi_foo: +* endbr64 +* subl $0x12345678, %r10d +* jz 1f +* ud2 +* 1: nop +* foo: +* +* regs->ip points to the UD2 instruction. +*/ +unsigned long addr = regs->ip - (4 +7 +2 );
arch/x86/kernel/alternative.c static void poison_cfi(void *addr) · line 1438
Quoted 10 lines
arch/x86/kernel/alternative.c +10
-0+u32 hash; + +if (!is_endbr ((void *)addr)) { +Efault: +return false; + } + + *target = addr + fineibt_preamble_size; + +__get_kernel_nofault (&hash, *(u32 *)(addr + fineibt_preamble_hash), u32, Efault);
You have an extra * here, should be just (u32 *).
Quoted 52 lines
arch/x86/kernel/alternative.c +9
-0+ *type = (u32 )regs->r10 + hash; +return true; +} +#else static void __apply_fineibt(s32 *start_retpoline, s32 *end_retpoline,@@ -1421,6 +1451,11 @@ static void __apply_fineibt(s32 *start_retpoline, s32 *end_retpoline,static void poison_cfi (void *addr) { }#endif +bool decode_fineibt_insn (struct pt_regs *regs,unsigned long *target,u32 *type) +{ +return false; +} +#endif void apply_fineibt(s32 *start_retpoline, s32 *end_retpoline,arch/x86/kernel/cfi.c +13
-4@@ -61,6 +61,8 @@ static bool decode_cfi_insn(struct pt_regs *regs, unsigned long *target,return true; } +extern bool decode_fineibt_insn (struct pt_regs *regs,unsigned long *target,u32 *type); +/* * Checks if a ud2 trap is because of a CFI failure, and handles the trap * if needed. Returns a bug_trap_type value similarly to report_bug. @@ -70,11 +72,25 @@ enum bug_trap_type handle_cfi_failure(struct pt_regs *regs)unsigned long target;u32 type; -if (!is_cfi_trap (regs->ip)) -return BUG_TRAP_TYPE_NONE ; +switch (cfi_mode) { +case CFI_KCFI : +if (!is_cfi_trap (regs->ip)) +return BUG_TRAP_TYPE_NONE ; + +if (!decode_cfi_insn (regs, &target, &type)) +return report_cfi_failure_noaddr (regs, regs->ip); + +break ; -if (!decode_cfi_insn (regs, &target, &type)) -return report_cfi_failure_noaddr (regs, regs->ip); +case CFI_FINEIBT : +if (!decode_fineibt_insn (regs, &target, &type))
arch/x86/kernel/cfi.c enum bug_trap_type handle_cfi_failure(struct pt_regs *regs) · line 96
Quoted 10 lines
arch/x86/kernel/cfi.c +7
-0+return BUG_TRAP_TYPE_NONE ; + +break ; + +default : +return BUG_TRAP_TYPE_NONE ; + }return report_cfi_failure (regs, regs->ip, &target, type); }
Otherwise, LGTM.
One minor issue is that since the trap is in the preamble, we don't get caller information in the warning message:
[ 19.080184] CFI failure at __cfi_lkdtm_increment_int+0xd/0x10
(target: lkdtm_increment_int+0x0/0x20; expected type: 0x7e0c52a5)
But this is followed by a call trace, so it's not really necessary either. With the __get_kernel_nofault argument fixed:
Sami
Quoted 3 lines · Sami Tolvanen
Review
Quoted 1 line
(quoted diff) +1
-0+ __get_kernel_nofault(&hash, *(u32 *)(addr + fineibt_preamble_hash), u32, Efault);You have an extra * here, should be just (u32 *).
Doh :-), I started by doing a plain deref and then figured I should be careful and wrap it in the nofault thing.
Quoted 5 lines
Otherwise, LGTM.
One minor issue is that since the trap is in the preamble, we don't get caller information in the warning message:
Quoted 7 lines
[ 19.080184] CFI failure at __cfi_lkdtm_increment_int+0xd/0x10(target:
lkdtm_increment_int+0x0/0x20; expected type:0x7e0c52a5)But this is followed by a call trace, so it's not really necessary either. With the
__get_kernel_nofaultargument fixed:Reviewed-by Sami Tolvanen <[email protected]>
Thanks.
I did some more clean-ups. I'll stick it on top of those patches slated for x86/core.
---
Subject: x86/ibt: Handle FineIBT in handle_cfi_failure()
From: Peter Zijlstra <[email protected]>
Date: Thu, 13 Feb 2025 12:45:47 +0100
Sami reminded me that FineIBT failure does not hook into the regular CFI failure case, and as such CFI_PERMISSIVE does not work.
--- arch/x86/include/asm/cfi.h | 11 +++++++++++ arch/x86/kernel/alternative.c | 30 ++++++++++++++++++++++++++++++ arch/x86/kernel/cfi.c | 22 ++++++++++++++++++---- 3 files changed, 59 insertions(+), 4 deletions(-)
arch/x86/include/asm/cfi.h +11 -0
@@ -126,6 +126,17 @@ static inline int cfi_get_offset(void)extern u32 cfi_get_func_hash (void *func); +#ifdef CONFIG_FINEIBT +extern bool decode_fineibt_insn (struct pt_regs *regs,unsigned long *target,u32 *type); +#else +static inline bool +decode_fineibt_insn (struct pt_regs *regs,unsigned long *target,u32 *type) +{ +return false; +} + +#endif +#else static inline enum bug_trap_type handle_cfi_failure (struct pt_regs *regs) {
arch/x86/kernel/alternative.c +30 -0
@@ -1065,6 +1065,7 @@ asm( ".pushsection .rodata \n"" endbr64 \n" " subl $0x12345678, %r10d \n" " je fineibt_preamble_end \n" +"fineibt_preamble_ud2: \n" " ud2 \n" " nop \n" "fineibt_preamble_end: \n"
@@ -1072,9 +1073,11 @@ asm( ".pushsection .rodata \n" );extern u8 fineibt_preamble_start[]; +extern u8 fineibt_preamble_ud2[];extern u8 fineibt_preamble_end[];#define fineibt_preamble_size (fineibt_preamble_end - fineibt_preamble_start) +#define fineibt_preamble_ud2 (fineibt_preamble_ud2 - fineibt_preamble_start)#define fineibt_preamble_hash 7 asm( ".pushsection .rodata \n"
@@ -1410,6 +1413,33 @@ static void poison_cfi(void *addr) } } +/* +* regs->ip points to a UD2 instruction, return true and fill out target and +* type when this UD2 is from a FineIBT preamble. +* +* We check the preamble by checking for the ENDBR instruction relative to the +* UD2 instruction. +*/ +bool decode_fineibt_insn (struct pt_regs *regs,unsigned long *target,u32 *type) +{ +unsigned long addr = regs->ip - fineibt_preamble_ud2; +u32 endbr, hash; + +__get_kernel_nofault (&endbr, addr, u32, Efault); +if (endbr !=gen_endbr ()) +return false; + + *target = addr + fineibt_preamble_size; + +__get_kernel_nofault (&hash, addr + fineibt_preamble_hash, u32, Efault); + *type = (u32 )regs->r10 + hash; + +return true; + +Efault: +return false; +} +#else static void __apply_fineibt(s32 *start_retpoline, s32 *end_retpoline,
arch/x86/kernel/cfi.c +18 -4
@@ -70,11 +70,25 @@ enum bug_trap_type handle_cfi_failure(stunsigned long target;u32 type; -if (!is_cfi_trap (regs->ip)) -return BUG_TRAP_TYPE_NONE ; +switch (cfi_mode) { +case CFI_KCFI : +if (!is_cfi_trap (regs->ip)) +return BUG_TRAP_TYPE_NONE ; + +if (!decode_cfi_insn (regs, &target, &type)) +return report_cfi_failure_noaddr (regs, regs->ip); + +break ; -if (!decode_cfi_insn (regs, &target, &type)) -return report_cfi_failure_noaddr (regs, regs->ip); +case CFI_FINEIBT : +if (!decode_fineibt_insn (regs, &target, &type)) +return BUG_TRAP_TYPE_NONE ; + +break ; + +default : +return BUG_TRAP_TYPE_NONE ; + }return report_cfi_failure (regs, regs->ip, &target, type); }
[tip: x86/core] x86/ibt: Handle FineIBT in handle_cfi_failure()
The following commit has been merged into the x86/core branch of tip:
The following commit has been merged into the x86/core branch of tip:
Commit-ID: 882b86fd4e0d49bf91148dbadcdbece19ded40e6 Gitweb: https://git.kernel.org/tip/882b86fd4e0d49bf91148dbadcdbece19ded40e6 Author: Peter Zijlstra <[email protected]> AuthorDate: Thu, 13 Feb 2025 12:45:47 +01:00 Committer: Peter Zijlstra <[email protected]> CommitterDate: Fri, 14 Feb 2025 10:32:07 +01:00
x86/ibt: Handle FineIBT in handle_cfi_failure()
Sami reminded me that FineIBT failure does not hook into the regular CFI failure case, and as such CFI_PERMISSIVE does not work.
--- arch/x86/include/asm/cfi.h | 11 +++++++++++ arch/x86/kernel/alternative.c | 30 ++++++++++++++++++++++++++++++ arch/x86/kernel/cfi.c | 22 ++++++++++++++++++---- 3 files changed, 59 insertions(+), 4 deletions(-)
arch/x86/include/asm/cfi.h +11 -0
@@ -126,6 +126,17 @@ static inline int cfi_get_offset(void)extern u32 cfi_get_func_hash (void *func); +#ifdef CONFIG_FINEIBT +extern bool decode_fineibt_insn (struct pt_regs *regs,unsigned long *target,u32 *type); +#else +static inline bool +decode_fineibt_insn (struct pt_regs *regs,unsigned long *target,u32 *type) +{ +return false; +} + +#endif +#else static inline enum bug_trap_type handle_cfi_failure (struct pt_regs *regs) {
arch/x86/kernel/alternative.c +30 -0
@@ -1065,6 +1065,7 @@ asm( ".pushsection .rodata \n"" endbr64 \n" " subl $0x12345678, %r10d \n" " je fineibt_preamble_end \n" +"fineibt_preamble_ud2: \n" " ud2 \n" " nop \n" "fineibt_preamble_end: \n"
@@ -1072,9 +1073,11 @@ asm( ".pushsection .rodata \n" );extern u8 fineibt_preamble_start[]; +extern u8 fineibt_preamble_ud2[];extern u8 fineibt_preamble_end[];#define fineibt_preamble_size (fineibt_preamble_end - fineibt_preamble_start) +#define fineibt_preamble_ud2 (fineibt_preamble_ud2 - fineibt_preamble_start)#define fineibt_preamble_hash 7 asm( ".pushsection .rodata \n"
@@ -1410,6 +1413,33 @@ static void poison_cfi(void *addr) } } +/* +* regs->ip points to a UD2 instruction, return true and fill out target and +* type when this UD2 is from a FineIBT preamble. +* +* We check the preamble by checking for the ENDBR instruction relative to the +* UD2 instruction. +*/ +bool decode_fineibt_insn (struct pt_regs *regs,unsigned long *target,u32 *type) +{ +unsigned long addr = regs->ip - fineibt_preamble_ud2; +u32 endbr, hash; + +__get_kernel_nofault (&endbr, addr, u32, Efault); +if (endbr !=gen_endbr ()) +return false; + + *target = addr + fineibt_preamble_size; + +__get_kernel_nofault (&hash, addr + fineibt_preamble_hash, u32, Efault); + *type = (u32 )regs->r10 + hash; + +return true; + +Efault: +return false; +} +#else static void __apply_fineibt(s32 *start_retpoline, s32 *end_retpoline,
arch/x86/kernel/cfi.c +18 -4
@@ -70,11 +70,25 @@ enum bug_trap_type handle_cfi_failure(struct pt_regs *regs)unsigned long target;u32 type; -if (!is_cfi_trap (regs->ip)) -return BUG_TRAP_TYPE_NONE ; +switch (cfi_mode) { +case CFI_KCFI : +if (!is_cfi_trap (regs->ip)) +return BUG_TRAP_TYPE_NONE ; + +if (!decode_cfi_insn (regs, &target, &type)) +return report_cfi_failure_noaddr (regs, regs->ip); + +break ; -if (!decode_cfi_insn (regs, &target, &type)) -return report_cfi_failure_noaddr (regs, regs->ip); +case CFI_FINEIBT : +if (!decode_fineibt_insn (regs, &target, &type)) +return BUG_TRAP_TYPE_NONE ; + +break ; + +default : +return BUG_TRAP_TYPE_NONE ; + }return report_cfi_failure (regs, regs->ip, &target, type); }