x86,ibt: Use UDB instead of 0xEA
Subsystems (3)
PATCH v2 1/1 x86,ibt: Use UDB instead of 0xEA
A while ago [0] FineIBT started using the 0xEA instruction to raise #UD. All existing parts will generate #UD in 64bit mode on that instruction.
However; Intel/AMD have not blessed using this instruction, it is on their 'reserved' opcode list for future use.
Peter Anvin worked the committees and got use of 0xD6 blessed, it shall be called UDB (per the next SDM or so), and it being a single byte instruction is easy to slip into a single byte immediate -- as is done by this very patch.
Reworking the FineIBT code to use UDB wasn't entirely trivial. Notably the FineIBT-BHI1 case ran out of bytes. In order to condense the encoding some it was required to move the hash register from R10D to EAX (thanks hpa!).
Per the x86_64 ABI, RAX is used to pass the number of vector registers for vararg function calls -- something that should not happen in the kernel. More so, the kernel is built with -mskip-rax-setup, which should leave RAX completely unused, allowing its re-use.
Of note:
- The ENDBR poison value is changed to: 0f 1f 40 d6 - nopl -42(%rax) which is a (distinct) valid 4 byte NOP and provides the UDB for the preamble.
- The branches carry a not-taken hint where possible; additionally modern cores seem to prefer fall-through / not-taken - per Agner Fog.
- For BPF; while the bpf2bpf tail-call uses RAX in its calling convention, that does not use CFI and is unaffected. Only the 'regular' C->BPF transition is covered by CFI.
[0] 06926c6cdb95 ("x86/ibt: Optimize the FineIBT instruction sequence")
---
Changes since RFC:
- more comment updates
- move UDB into NOP4 immediate for ENDBR poison
- simplified reporting/recovery code
- replaced a nop by using jcc.32
- more actual testing
arch/x86/include/asm/bug.h | 9 + arch/x86/include/asm/cfi.h | 14 +- arch/x86/include/asm/ibt.h | 10 - arch/x86/kernel/alternative.c | 217 ++++++++++++++++++++++++------------------ arch/x86/kernel/traps.c | 8 - arch/x86/lib/bhi.S | 58 +++++------ arch/x86/lib/retpoline.S | 4 arch/x86/net/bpf_jit_comp.c | 6 - 8 files changed, 178 insertions(+), 148 deletions(-)
arch/x86/include/asm/bug.h +7 -2
@@ -5,14 +5,19 @@#include <linux/stringify.h> #include <linux/instrumentation.h> #include <linux/objtool.h> +#include <asm/asm.h> /* * Despite that some emulators terminate on UD2, we use it for WARN(). */ -#define ASM_UD2 ".byte 0x0f, 0x0b" +#define ASM_UD2 _ASM_BYTES(0x0f, 0x0b)#define INSN_UD2 0x0b0f#define LEN_UD2 2 +#define ASM_UDB _ASM_BYTES(0xd6) +#define INSN_UDB 0xd6 +#define LEN_UDB 1 +/* * In clang we have UD1s reporting UBSAN failures on X86, 64 and 32bit. */
@@ -26,7 +31,7 @@#define BUG_UD2 0xfffe#define BUG_UD1 0xfffd#define BUG_UD1_UBSAN 0xfffc -#define BUG_EA 0xffea +#define BUG_UDB 0xffd6#define BUG_LOCK 0xfff0#ifdef CONFIG_GENERIC_BUG
arch/x86/include/asm/cfi.h +6 -8
@@ -71,12 +71,10 @@* * __cfi_foo: * endbr64 -* subl 0x12345678, %r10d -* jz foo -* ud2 -* nop +* subl 0x12345678, %eax +* jne.32,pn foo+3 * foo: -* osp nop3 # was endbr64 +* nopl -42(%rax) # was endbr64 * ... code here ... * ret *
@@ -86,9 +84,9 @@* indirect caller: * lea foo(%rip), %r11 * ... -* movl $0x12345678, %r10d -* subl $16, %r11 -* nop4 +* movl $0x12345678, %eax +* lea -0x10(%r11), %r11 +* nop5 * call *%r11 * */
arch/x86/include/asm/ibt.h +3 -7
@@ -59,10 +59,10 @@ static __always_inline __attribute_const static __always_inline __attribute_const__ u32 gen_endbr_poison(void) {/* -* 4 byte NOP that isn't NOP4 (in fact it is OSP NOP3), such that it -* will be unique to (former) ENDBR sites. +* 4 byte NOP that isn't NOP4, such that it will be unique to (former) +* ENDBR sites. Additionally it carries UDB as immediate. */ -return 0x001f0f66 ;/* osp nopl (%rax) */ +return 0xd6401f0f ;/* nopl -42(%rax) */ }static inline bool __is_endbr (u32 val)
@@ -70,10 +70,6 @@ static inline bool __is_endbr(u32 val)if (val ==gen_endbr_poison ())return true; -/* See cfi_fineibt_bhi_preamble() */ -if (IS_ENABLED (CONFIG_FINEIBT_BHI ) && val ==0x001f0ff5 ) -return true; - val &= ~0x01000000U ;/* ENDBR32 -> ENDBR64 */ return val ==gen_endbr (); }
arch/x86/kernel/alternative.c +124 -93
398 lines not on this page · raw message
arch/x86/kernel/traps.c +4 -4
@@ -97,7 +97,7 @@ __always_inline int is_valid_bugaddr(uns* Check for UD1 or UD2, accounting for Address Size Override Prefixes. * If it's a UD1, further decode to determine its use: * -* FineIBT: ea (bad) +* FineIBT: d6 udb * FineIBT: f0 75 f9 lock jne . - 6 * UBSan{0}: 67 0f b9 00 ud1 (%eax),%eax * UBSan{10}: 67 0f b9 40 10 ud1 0x10(%eax),%eax
@@ -130,9 +130,9 @@ __always_inline int decode_bug(unsignedWARN_ON_ONCE (!lock);return BUG_LOCK ; -case 0xea : +case 0xd6 : *len = addr - start; -return BUG_EA ; +return BUG_UDB ;case OPCODE_ESCAPE :break ;
@@ -341,7 +341,7 @@ static noinstr bool handle_bug(struct pt } fallthrough; -case BUG_EA : +case BUG_UDB :case BUG_LOCK :if (handle_cfi_failure (regs) ==BUG_TRAP_TYPE_WARN ) { handled = true;
arch/x86/lib/bhi.S +29 -29
106 lines not on this page · raw message
arch/x86/lib/retpoline.S +2 -2
@@ -134,10 +134,10 @@ SYM_CODE_END(__x86_indirect_jump_thunk_a .macro ITS_THUNK reg /* - * If CFI paranoid is used then the ITS thunk starts with opcodes (0xea; jne 1b) + * If CFI paranoid is used then the ITS thunk starts with opcodes (1: udb; jne 1b) * that complete the fineibt_paranoid caller sequence. */ -1: .byte 0xea +1: ASM_UDB SYM_INNER_LABEL(__x86_indirect_paranoid_thunk_\reg, SYM_L_GLOBAL) UNWIND_HINT_UNDEFINED ANNOTATE_NOENDBR
arch/x86/net/bpf_jit_comp.c +3 -3
@@ -419,12 +419,12 @@ static void emit_fineibt(u8 **pprog, u8u8 *prog = *pprog;EMIT_ENDBR (); -EMIT3_off32 (0x41 ,0x81 ,0xea , hash);/* subl $hash, %r10d */ +EMIT1_off32 (0x2d , hash);/* subl $hash, %eax */ if (cfi_bhi) { +EMIT2 (0x2e ,0x2e );/* cs cs */ emit_call (&prog, __bhi_args[arity], ip +11 ); }else { -EMIT2 (0x75 ,0xf9 );/* jne.d8 .-7 */ -EMIT3 (0x0f ,0x1f ,0x00 );/* nop3 */ +EMIT3_off32 (0x2e ,0x0f ,0x85 ,3 );/* jne.d32,pn 3 */ }EMIT_ENDBR_POISON ();
Because this is all somewhat magical code, and this change is a little on the large side, it as been suggested I 'upgrade' the Changelog some.
Quoted 17 lines · Peter Zijlstra
A while ago [0] FineIBT started using the
0xEAinstruction to raise #UD. All existing parts will generate #UD in 64bit mode on that instruction.However; Intel/AMD have not blessed using this instruction, it is on their 'reserved' opcode list for future use.
Peter Anvin worked the committees and got use of
0xD6blessed, it shall be called UDB (per the next SDM or so), and it being a single byte instruction is easy to slip into a single byte immediate -- as is done by this very patch.Reworking the FineIBT code to use UDB wasn't entirely trivial. Notably the FineIBT-BHI1 case ran out of bytes. In order to condense the encoding some it was required to move the hash register from R10D to EAX (thanks hpa!).
Quoted 4 lines
Per the
x86_64ABI, RAX is used to pass the number of vector registers for vararg function calls -- something that should not happen in the kernel. More so, the kernel is built with -mskip-rax-setup, which should leave RAX completely unused, allowing its re-use.
[ For BPF; while the bpf2bpf tail-call uses RAX in its calling
convention, that does not use CFI and is unaffected. Only the 'regular' C->BPF transition is covered by CFI. ]
The ENDBR poison value is changed from 'OSP NOP3' to 'NOPL -42(%RAX)', this is basically NOP4 but with UDB as its immediate. As such it is still a non-standard NOP value unique to prior ENDBR sites, but now also provides UDB.
Per Agner Fog's optimization guide, Jcc is assumed not-taken. That is, the expected path should be the fallthrough case for improved throughput.
Since the preamble now relies on the ENDBR poison to provide UDB, the code is changed to write the poison right along with the initial preamble -- this is possible because the ITS mitigation already disabled IBT over rewriting the CFI scheme.
The scheme in detail:
Preamble:
FineIBT FineIBT-BHI1 FineIBT-BHI
__cfi_\func: __cfi_\func: __cfi_\func:
endbr endbr endbr
subl $0x12345678, %eax subl $0x12345678, %eax subl $0x12345678, %eax
jcc.d32,np \func+3 cmovne %rax, %rdi cs cs call __bhi_args_N
jcc.d8,np \func+3 \func: \func: \func:
nopl -42(%rax) nopl -42(%rax) nopl -42(%rax)
Notably there are 7 bytes available after the SUBL; this enables the BHI-1 case to fit without the nasty overlapping case it had previously. The !BHI case uses Jcc.d32 to consume all 7 bytes without the need for an additional NOP, while the BHI case uses CS padding to align the CALL with the end of the preamble such that it returns to \func+0.
Caller:
FineIBT Paranoid-FineIBT
fineibt_caller: fineibt_caller:
mov $0x12345678, %eax mov $0x12345678, %eax
lea -10(%r11), %r11 cmp -0x11(%r11), %eax
nop5 cs lea -0x10(%r11), %r11
retpoline: retpoline:
cs call __x86_indirect_thunk_r11 jne fineibt_caller+0xd
call *%r11
nopNotably this is before apply_retpolines() which will fix up the retpoline call -- since all parts with IBT also have eIBRS (lets ignore ITS). Typically the retpoline site is rewritten (when still intact) into:
call *r11
nop3Quoted 3 lines
[0]
06926c6cdb95("x86/ibt: Optimize the FineIBT instruction sequence")Signed-off-by Peter Zijlstra (Intel) <[email protected]>
And now I'm going to have to do a patch that makes apply_retpoline() do CS padding instead of NOP padding for CALL...
Quoted 18 lines · Peter Zijlstra
Caller:
FineIBT Paranoid-FineIBT
fineibt_caller:fineibt_caller:mov $0x12345678, %eax mov $0x12345678, %eax lea -10(%r11), %r11 cmp -0x11(%r11), %eax nop5 cs lea -0x10(%r11), %r11 retpoline: retpoline: cs call __x86_indirect_thunk_r11 jne fineibt_caller+0xd call *%r11 nopNotably this is before
apply_retpolines()which will fix up the retpoline call -- since all parts with IBT also have eIBRS (lets ignore ITS). Typically the retpoline site is rewritten (when still intact) into:
Quoted 6 lines
call *r11 nop3And now I'm going to have to do a patch that makes
apply_retpoline()do CS padding instead of NOP padding for CALL...
Finding the exact prefix decode penalties for uarchs that have eIBRS/BHI_NO is not a fun time. I've stuck to the general wisdom that 3 prefixes is mostly good (notably, the instruction at hand has no 0x0f escape which is sometimes counted towards the prefix budget -- it can have a REX prefix, but those are generally not counted towards the prefix budget).
In general Intel P-cores do not have prefix decode penalties, but the E-cores (or rather the Atom line) generally does. And since this all runs on hybrid cores, the code must accommodate them.
I hate all this.
---
Subject: x86,retpoline: Optimize patch_retpoline()
From: Peter Zijlstra <[email protected]>
Date: Tue Sep 2 11:20:35 CEST 2025
Currently the very common retpoline: "CS CALL __x86_indirect_thunk_r11" is transformed into "CALL *R11; NOP3" for eIBRS/BHI_NO parts.
Similarly, paranoid fineibt has: "CALL *R11; NOP".
Recognise that CS stuffing can avoid the extra NOP. However, due to prefix decode penalties, make sure to not emit too many CS prefixes.
Notably: "CS CALL __x86_indirect_thunk_rax" must not become "CS CS CS CS CALL *RAX". Prefix decode penalties are typically many more cycles than decoding an extra NOP.
Additionally, if the retpoline is a tail-call, the "JMP *%\reg" should be followed by INT3 for straight-line-speculation mitigation, since emit_indirect() now has a length argument, move this into emit_indirect() such that other users (paranoid-fineibt) also do this.
--- arch/x86/kernel/alternative.c | 40 +++++++++++++++++++++++++--------------- 1 file changed, 25 insertions(+), 15 deletions(-)
arch/x86/kernel/alternative.c +25 -15
@@ -715,18 +715,31 @@ static inline bool is_jcc32(struct insn/* * CALL/JMP *%\reg */ -static int emit_indirect (int op,int reg,u8 *bytes) +static int emit_indirect (int op,int reg,u8 *bytes,int len) { +int cs =0 , bp =0 ;int i =0 ;u8 modrm; +/* +* Set @len to the excess bytes after writing the instruction. +*/ + len -=2 + (reg >=8 ); +WARN_ON_ONCE (len <0 ); +switch (op) {case CALL_INSN_OPCODE : modrm =0x10 ;/* Reg = 2; CALL r/m */ +/* +* Additional NOP is better than prefix decode penalty. +*/ +if (len <=3 ) + cs = len;break ;case JMP32_INSN_OPCODE : modrm =0x20 ;/* Reg = 4; JMP r/m */ + bp = !!len;break ;default :
@@ -734,6 +747,9 @@ static int emit_indirect(int op, int regreturn -1 ; } +while (cs--) + bytes[i++] =0x2e ;/* CS-prefix */ +if (reg >=8 ) { bytes[i++] =0x41 ;/* REX.B prefix */ reg -=8 ;
@@ -745,6 +761,9 @@ static int emit_indirect(int op, int reg bytes[i++] =0xff ;/* opcode */ bytes[i++] = modrm; +if (bp) + bytes[i++] =0xcc ;/* INT3 */ +return i; }
@@ -918,20 +937,11 @@ static int patch_retpoline(void *addr, sreturn emit_its_trampoline (addr, insn, reg, bytes);#endif - ret =emit_indirect (op, reg, bytes + i); + ret =emit_indirect (op, reg, bytes + i, insn->length - i);if (ret <0 )return ret; i += ret; -/* -* The compiler is supposed to EMIT an INT3 after every unconditional -* JMP instruction due to AMD BTC. However, if the compiler is too old -* or MITIGATION_SLS isn't enabled, we still need an INT3 after -* indirect JMPs even on Intel. -*/ -if (op ==JMP32_INSN_OPCODE && i < insn->length) - bytes[i++] =INT3_INSN_OPCODE ; -for (; i < insn->length;) bytes[i++] =BYTES_NOP1 ;
@@ -1418,8 +1428,7 @@ asm( ".pushsection .rodata \n""#fineibt_caller_size: \n" " jne fineibt_paranoid_start+0xd \n" "fineibt_paranoid_ind: \n" -" call *%r11 \n" -" nop \n" +" cs call *%r11 \n" "fineibt_paranoid_end: \n" ".popsection \n" );
@@ -1721,8 +1730,9 @@ static int cfi_rewrite_callers(s32 *staremit_paranoid_trampoline (addr + fineibt_caller_size, &insn,11 , bytes + fineibt_caller_size); }else { - ret =emit_indirect (op,11 , bytes + fineibt_paranoid_ind); -if (WARN_ON_ONCE (ret !=3 )) +int len = fineibt_paranoid_size - fineibt_paranoid_ind; + ret =emit_indirect (op,11 , bytes + fineibt_paranoid_ind, len); +if (WARN_ON_ONCE (ret != len))continue ; }
Quoted 23 lines · Peter Zijlstra
Because this is all somewhat magical code, and this change is a little on the large side, it as been suggested I 'upgrade' the Changelog some.
Quoted 17 lines · Peter Zijlstra
A while ago [0] FineIBT started using the
0xEAinstruction to raise #UD. All existing parts will generate #UD in 64bit mode on that instruction.However; Intel/AMD have not blessed using this instruction, it is on their 'reserved' opcode list for future use.
Peter Anvin worked the committees and got use of
0xD6blessed, it shall be called UDB (per the next SDM or so), and it being a single byte instruction is easy to slip into a single byte immediate -- as is done by this very patch.Reworking the FineIBT code to use UDB wasn't entirely trivial. Notably the FineIBT-BHI1 case ran out of bytes. In order to condense the encoding some it was required to move the hash register from R10D to EAX (thanks hpa!).
Quoted 8 lines
Quoted 4 lines
Per the
x86_64ABI, RAX is used to pass the number of vector registers for vararg function calls -- something that should not happen in the kernel. More so, the kernel is built with -mskip-rax-setup, which should leave RAX completely unused, allowing its re-use.[ For BPF; while the bpf2bpf tail-call uses RAX in its calling
convention, that does not use CFI and is unaffected. Only the 'regular'C->BPFtransition is covered by CFI. ]
I cannot comprehend the new scheme, but thanks for heads up.
Quoted 29 lines · Alexei Starovoitov
Quoted 28 lines · Peter Zijlstra
Because this is all somewhat magical code, and this change is a little on the large side, it as been suggested I 'upgrade' the Changelog some.
Quoted 21 lines · Peter Zijlstra
A while ago [0] FineIBT started using the
0xEAinstruction to raise #UD. All existing parts will generate #UD in 64bit mode on that instruction.However; Intel/AMD have not blessed using this instruction, it is on their 'reserved' opcode list for future use.
Peter Anvin worked the committees and got use of
0xD6blessed, it shall be called UDB (per the next SDM or so), and it being a single byte instruction is easy to slip into a single byte immediate -- as is done by this very patch.Reworking the FineIBT code to use UDB wasn't entirely trivial. Notably the FineIBT-BHI1 case ran out of bytes. In order to condense the encoding some it was required to move the hash register from R10D to EAX (thanks hpa!).
Per the
x86_64ABI, RAX is used to pass the number of vector registers for vararg function calls -- something that should not happen in the kernel. More so, the kernel is built with -mskip-rax-setup, which should leave RAX completely unused, allowing its re-use.
Quoted 5 lines
Quoted 3 lines
[ For BPF; while the bpf2bpf tail-call uses RAX in its calling
convention, that does not use CFI and is unaffected. Only the 'regular'C->BPFtransition is covered by CFI. ]I cannot comprehend the new scheme, but thanks for heads up.
Anywhere in particular you're getting stuck? That is, where should I improve the comments / changelog?
Quoted 31 lines · Peter Zijlstra
Quoted 29 lines · Alexei Starovoitov
Quoted 28 lines · Peter Zijlstra
Because this is all somewhat magical code, and this change is a little on the large side, it as been suggested I 'upgrade' the Changelog some.
Quoted 21 lines · Peter Zijlstra
A while ago [0] FineIBT started using the
0xEAinstruction to raise #UD. All existing parts will generate #UD in 64bit mode on that instruction.However; Intel/AMD have not blessed using this instruction, it is on their 'reserved' opcode list for future use.
Peter Anvin worked the committees and got use of
0xD6blessed, it shall be called UDB (per the next SDM or so), and it being a single byte instruction is easy to slip into a single byte immediate -- as is done by this very patch.Reworking the FineIBT code to use UDB wasn't entirely trivial. Notably the FineIBT-BHI1 case ran out of bytes. In order to condense the encoding some it was required to move the hash register from R10D to EAX (thanks hpa!).
Per the
x86_64ABI, RAX is used to pass the number of vector registers for vararg function calls -- something that should not happen in the kernel. More so, the kernel is built with -mskip-rax-setup, which should leave RAX completely unused, allowing its re-use.
Quoted 8 lines
Quoted 5 lines
Quoted 3 lines
[ For BPF; while the bpf2bpf tail-call uses RAX in its calling
convention, that does not use CFI and is unaffected. Only the 'regular'C->BPFtransition is covered by CFI. ]I cannot comprehend the new scheme, but thanks for heads up.
Anywhere in particular you're getting stuck? That is, where should I improve the comments / changelog?
Well, I mean all these tricky changes are allegedly because "Intel/AMD have not blessed using this instruction, it is on their 'reserved' opcode list for future use"
I suspect that 'reserved' opcode will not be used any time soon. If 10 years from now the opcode is used in some future CPU that CPU is better to be not vulnerable and CFI, FineIBT things will be gone from the kernel by then.
So I would do absolutely nothing and just ignore the lack of blessing.
Quoted 9 lines · Alexei Starovoitov
Well, I mean all these tricky changes are allegedly because "Intel/AMD have not blessed using this instruction, it is on their 'reserved' opcode list for future use"
I suspect that 'reserved' opcode will not be used any time soon. If 10 years from now the opcode is used in some future CPU that CPU is better to be not vulnerable and CFI, FineIBT things will be gone from the kernel by then.
So I would do absolutely nothing and just ignore the lack of blessing.
Ah. CFI is not only about speculation, it is also very much a protection against pointer hijacking. Note how kCFI was not brought in because of speculation, but as a hardening against pointer hijacking.
Yes, my interest in Intel CET-IBT is mostly from the speculation avoidance angle, but CFI as a whole has wider applicability. Even a 'fixed' CPU would want to use CFI.
I would also love AMD to grow support for CET-IBT; they already implemented CET-SS.
Also, Intel pays me, if they say make it use 0xD6, this is what we do :-)
H. Peter Anvin
reviewerQuoted 7 lines · Alexei Starovoitov
I suspect that 'reserved' opcode will not be used any time soon. If 10 years from now the opcode is used in some future CPU that CPU is better to be not vulnerable and CFI, FineIBT things will be gone from the kernel by then.
So I would do absolutely nothing and just ignore the lack of blessing.
What makes you say that? *Please* don't make these kinds of assumptions -- it causes massive headaches for both us chip makers and for users.
-hpa