objtool: Detect and warn about indirect calls in __nocfi functions
Subsystems (11)
Peter Zijlstra
maintainerPATCH v3 4/16 x86/kvm/emulate: Introduce EM_ASM_2R
In reply to Peter ZijlstraReplace the FASTOP2R instruction.
--- arch/x86/kvm/emulate.c | 16 ++++++++++++---- 1 file changed, 12 insertions(+), 4 deletions(-)
arch/x86/kvm/emulate.c +12 -4
@@ -326,6 +326,15 @@ static int em_##op(struct x86_emulate_ct ON64(case 8: __EM_ASM_2(op##q, rax, rdx); break;) \ EM_ASM_END +/* 2-operand, reversed */ +#define EM_ASM_2R (op, name) \ + EM_ASM_START(name) \ + case 1: __EM_ASM_2(op##b, dl, al); break; \ + case 2: __EM_ASM_2(op##w, dx, ax); break; \ + case 4: __EM_ASM_2(op##l, edx, eax); break; \ + ON64(case 8: __EM_ASM_2(op##q, rdx, rax); break;) \ + EM_ASM_END +/* * fastop functions have a special calling convention: *
@@ -1077,8 +1086,7 @@ FASTOP2W(bts);FASTOP2W (btr);FASTOP2W (btc); - -FASTOP2R (cmp, cmp_r); +EM_ASM_2R (cmp, cmp_r);static int em_bsf_c (struct x86_emulate_ctxt *ctxt) {
@@ -4336,12 +4344,12 @@ static const struct opcode opcode_table[I2bv (DstAcc | SrcMem | Mov | MemAbs, em_mov),I2bv (DstMem | SrcAcc | Mov | MemAbs | PageTable, em_mov),I2bv (SrcSI | DstDI | Mov | String | TwoMemOp, em_mov), -F2bv (SrcSI | DstDI | String | NoWrite | TwoMemOp, em_cmp_r), +I2bv (SrcSI | DstDI | String | NoWrite | TwoMemOp, em_cmp_r),/* 0xA8 - 0xAF */ I2bv (DstAcc | SrcImm | NoWrite, em_test),I2bv (SrcAcc | DstDI | Mov | String, em_mov),I2bv (SrcSI | DstAcc | Mov | String, em_mov), -F2bv (SrcAcc | DstDI | String | NoWrite, em_cmp_r), +I2bv (SrcAcc | DstDI | String | NoWrite, em_cmp_r),/* 0xB0 - 0xB7 */ X8(I(ByteOp | DstReg | SrcImm | Mov, em_mov)),/* 0xB8 - 0xBF */
Peter Zijlstra
maintainerPATCH v3 5/16 x86/kvm/emulate: Introduce EM_ASM_2W
In reply to Peter ZijlstraReplace the FASTOP2W instructions.
--- arch/x86/kvm/emulate.c | 47 ++++++++++++++++++++++++++++------------------- 1 file changed, 28 insertions(+), 19 deletions(-)
arch/x86/kvm/emulate.c +28 -19
@@ -335,6 +335,15 @@ static int em_##op(struct x86_emulate_ct ON64(case 8: __EM_ASM_2(op##q, rdx, rax); break;) \ EM_ASM_END +/* 2-operand, word only (no byte op) */ +#define EM_ASM_2W (op) \ + EM_ASM_START(op) \ + case 1: break; \ + case 2: __EM_ASM_2(op##w, ax, dx); break; \ + case 4: __EM_ASM_2(op##l, eax, edx); break; \ + ON64(case 8: __EM_ASM_2(op##q, rax, rdx); break;) \ + EM_ASM_END +/* * fastop functions have a special calling convention: *
@@ -1064,7 +1073,7 @@ FASTOP1SRC2EX(idiv, idiv_ex);FASTOP3WCL (shld);FASTOP3WCL (shrd); -FASTOP2W (imul); +EM_ASM_2W (imul);EM_ASM_1 (not);EM_ASM_1 (neg);
@@ -1079,12 +1088,12 @@ FASTOP2CL(shl);FASTOP2CL (shr);FASTOP2CL (sar); -FASTOP2W (bsf); -FASTOP2W (bsr); -FASTOP2W (bt); -FASTOP2W (bts); -FASTOP2W (btr); -FASTOP2W (btc); +EM_ASM_2W (bsf); +EM_ASM_2W (bsr); +EM_ASM_2W (bt); +EM_ASM_2W (bts); +EM_ASM_2W (btr); +EM_ASM_2W (btc);EM_ASM_2R (cmp, cmp_r);
@@ -1093,7 +1102,7 @@ static int em_bsf_c(struct x86_emulate_c/* If src is zero, do not writeback, but update flags */ if (ctxt->src.val ==0 ) ctxt->dst.type =OP_NONE ; -return fastop (ctxt, em_bsf); +return em_bsf (ctxt); }static int em_bsr_c (struct x86_emulate_ctxt *ctxt)
@@ -1101,7 +1110,7 @@ static int em_bsr_c(struct x86_emulate_c/* If src is zero, do not writeback, but update flags */ if (ctxt->src.val ==0 ) ctxt->dst.type =OP_NONE ; -return fastop (ctxt, em_bsr); +return em_bsr (ctxt); } static __always_inline u8 test_cc(unsigned int condition, unsigned long flags)
@@ -3221,7 +3230,7 @@ static int em_xchg(struct x86_emulate_ctstatic int em_imul_3op (struct x86_emulate_ctxt *ctxt) { ctxt->dst.val = ctxt->src2.val; -return fastop (ctxt, em_imul); +return em_imul (ctxt); }static int em_cwd (struct x86_emulate_ctxt *ctxt)
@@ -4135,10 +4144,10 @@ static const struct group_dual group7 =static const struct opcode group8[] = {N ,N ,N ,N , -F (DstMem | SrcImmByte | NoWrite, em_bt), -F (DstMem | SrcImmByte | Lock | PageTable, em_bts), -F (DstMem | SrcImmByte | Lock, em_btr), -F (DstMem | SrcImmByte | Lock | PageTable, em_btc), +I (DstMem | SrcImmByte | NoWrite, em_bt), +I (DstMem | SrcImmByte | Lock | PageTable, em_bts), +I (DstMem | SrcImmByte | Lock, em_btr), +I (DstMem | SrcImmByte | Lock | PageTable, em_btc), };/*
@@ -4459,27 +4468,27 @@ static const struct opcode twobyte_table/* 0xA0 - 0xA7 */ I (Stack | Src2FS, em_push_sreg),I (Stack | Src2FS, em_pop_sreg),II (ImplicitOps, em_cpuid, cpuid), -F (DstMem | SrcReg | ModRM | BitOp | NoWrite, em_bt), +I (DstMem | SrcReg | ModRM | BitOp | NoWrite, em_bt),F (DstMem | SrcReg | Src2ImmByte | ModRM, em_shld),F (DstMem | SrcReg | Src2CL | ModRM, em_shld),N ,N ,/* 0xA8 - 0xAF */ I (Stack | Src2GS, em_push_sreg),I (Stack | Src2GS, em_pop_sreg),II (EmulateOnUD | ImplicitOps, em_rsm, rsm), -F (DstMem | SrcReg | ModRM | BitOp | Lock | PageTable, em_bts), +I (DstMem | SrcReg | ModRM | BitOp | Lock | PageTable, em_bts),F (DstMem | SrcReg | Src2ImmByte | ModRM, em_shrd),F (DstMem | SrcReg | Src2CL | ModRM, em_shrd), -GD (0 , &group15),F (DstReg | SrcMem | ModRM, em_imul), +GD (0 , &group15),I (DstReg | SrcMem | ModRM, em_imul),/* 0xB0 - 0xB7 */ I2bv (DstMem | SrcReg | ModRM | Lock | PageTable | SrcWrite, em_cmpxchg),I (DstReg | SrcMemFAddr | ModRM | Src2SS, em_lseg), -F (DstMem | SrcReg | ModRM | BitOp | Lock, em_btr), +I (DstMem | SrcReg | ModRM | BitOp | Lock, em_btr),I (DstReg | SrcMemFAddr | ModRM | Src2FS, em_lseg),I (DstReg | SrcMemFAddr | ModRM | Src2GS, em_lseg),D (DstReg | SrcMem8 | ModRM | Mov),D (DstReg | SrcMem16 | ModRM | Mov),/* 0xB8 - 0xBF */ N ,N ,G (BitOp, group8), -F (DstMem | SrcReg | ModRM | BitOp | Lock | PageTable, em_btc), +I (DstMem | SrcReg | ModRM | BitOp | Lock | PageTable, em_btc),I (DstReg | SrcMem | ModRM, em_bsf_c),I (DstReg | SrcMem | ModRM, em_bsr_c), D(DstReg | SrcMem8 | ModRM | Mov), D(DstReg | SrcMem16 | ModRM | Mov),
Peter Zijlstra
maintainerPATCH v3 6/16 x86/kvm/emulate: Introduce EM_ASM_2CL
In reply to Peter ZijlstraReplace the FASTOP2CL instructions.
--- arch/x86/kvm/emulate.c | 39 ++++++++++++++++++++++++--------------- 1 file changed, 24 insertions(+), 15 deletions(-)
arch/x86/kvm/emulate.c +24 -15
@@ -344,6 +344,15 @@ static int em_##op(struct x86_emulate_ct ON64(case 8: __EM_ASM_2(op##q, rax, rdx); break;) \ EM_ASM_END +/* 2-operand, using "a" (dst) and CL (src2) */ +#define EM_ASM_2CL (op) \ + EM_ASM_START(op) \ + case 1: __EM_ASM_2(op##b, al, cl); break; \ + case 2: __EM_ASM_2(op##w, ax, cl); break; \ + case 4: __EM_ASM_2(op##l, eax, cl); break; \ + ON64(case 8: __EM_ASM_2(op##q, rax, cl); break;) \ + EM_ASM_END +/* * fastop functions have a special calling convention: *
@@ -1080,13 +1089,13 @@ EM_ASM_1(neg);EM_ASM_1 (inc);EM_ASM_1 (dec); -FASTOP2CL (rol); -FASTOP2CL (ror); -FASTOP2CL (rcl); -FASTOP2CL (rcr); -FASTOP2CL (shl); -FASTOP2CL (shr); -FASTOP2CL (sar); +EM_ASM_2CL (rol); +EM_ASM_2CL (ror); +EM_ASM_2CL (rcl); +EM_ASM_2CL (rcr); +EM_ASM_2CL (shl); +EM_ASM_2CL (shr); +EM_ASM_2CL (sar);EM_ASM_2W (bsf);EM_ASM_2W (bsr);
@@ -4079,14 +4088,14 @@ static const struct opcode group1A[] = { };static const struct opcode group2[] = { -F (DstMem | ModRM, em_rol), -F (DstMem | ModRM, em_ror), -F (DstMem | ModRM, em_rcl), -F (DstMem | ModRM, em_rcr), -F (DstMem | ModRM, em_shl), -F (DstMem | ModRM, em_shr), -F (DstMem | ModRM, em_shl), -F (DstMem | ModRM, em_sar), +I (DstMem | ModRM, em_rol), +I (DstMem | ModRM, em_ror), +I (DstMem | ModRM, em_rcl), +I (DstMem | ModRM, em_rcr), +I (DstMem | ModRM, em_shl), +I (DstMem | ModRM, em_shr), +I (DstMem | ModRM, em_shl), +I (DstMem | ModRM, em_sar), };static const struct opcode group3[] = {
Peter Zijlstra
maintainerPATCH v3 7/16 x86/kvm/emulate: Introduce EM_ASM_1SRC2
In reply to Peter ZijlstraReplace the FASTOP1SRC2*() instructions.
--- arch/x86/kvm/emulate.c | 34 ++++++++++++++++++++++++++-------- 1 file changed, 26 insertions(+), 8 deletions(-)
arch/x86/kvm/emulate.c +26 -8
@@ -317,6 +317,24 @@ static int em_##op(struct x86_emulate_ct ON64(case 8: __EM_ASM_1(op##q, rax); break;) \ EM_ASM_END +/* 1-operand, using "c" (src2) */ +#define EM_ASM_1SRC2 (op, name) \ + EM_ASM_START(name) \ + case 1: __EM_ASM_1(op##b, cl); break; \ + case 2: __EM_ASM_1(op##w, cx); break; \ + case 4: __EM_ASM_1(op##l, ecx); break; \ + ON64(case 8: __EM_ASM_1(op##q, rcx); break;) \ + EM_ASM_END + +/* 1-operand, using "c" (src2) with exception */ +#define EM_ASM_1SRC2EX (op, name) \ + EM_ASM_START(name) \ + case 1: __EM_ASM_1_EX(op##b, cl); break; \ + case 2: __EM_ASM_1_EX(op##w, cx); break; \ + case 4: __EM_ASM_1_EX(op##l, ecx); break; \ + ON64(case 8: __EM_ASM_1(op##q, rcx); break;) \ + EM_ASM_END +/* 2-operand, using "a" (dst), "d" (src) */ #define EM_ASM_2 (op) \ EM_ASM_START(op) \
@@ -1074,10 +1092,10 @@ EM_ASM_2(cmp);EM_ASM_2 (test);EM_ASM_2 (xadd); -FASTOP1SRC2 (mul, mul_ex); -FASTOP1SRC2 (imul, imul_ex); -FASTOP1SRC2EX (div, div_ex); -FASTOP1SRC2EX (idiv, idiv_ex); +EM_ASM_1SRC2 (mul, mul_ex); +EM_ASM_1SRC2 (imul, imul_ex); +EM_ASM_1SRC2EX (div, div_ex); +EM_ASM_1SRC2EX (idiv, idiv_ex);FASTOP3WCL (shld);FASTOP3WCL (shrd);
@@ -4103,10 +4121,10 @@ static const struct opcode group3[] = {I (DstMem | SrcImm | NoWrite, em_test),I (DstMem | SrcNone | Lock, em_not),I (DstMem | SrcNone | Lock, em_neg), -F (DstXacc | Src2Mem, em_mul_ex), -F (DstXacc | Src2Mem, em_imul_ex), -F (DstXacc | Src2Mem, em_div_ex), - F(DstXacc | Src2Mem, em_idiv_ex), +I (DstXacc | Src2Mem, em_mul_ex), +I (DstXacc | Src2Mem, em_imul_ex), +I (DstXacc | Src2Mem, em_div_ex), + I(DstXacc | Src2Mem, em_idiv_ex), }; static const struct opcode group4[] = {
Peter Zijlstra
maintainerPATCH v3 15/16 x86/fred: KVM: VMX: Always use FRED for IRQs when CONFIG_X86_FRED=y
In reply to Peter ZijlstraFrom: Sean Christopherson <[email protected]>
Now that FRED provides C-code entry points for handling IRQs, use the FRED infrastructure for forwarding IRQs even if FRED is fully disabled, e.g. isn't supported in hardware. Avoiding the non-FRED assembly trampolines into the IDT handlers for IRQs eliminates the associated non-CFI indirect call (KVM performs a CALL by doing a lookup on the IDT using the IRQ vector).
Keep NMIs on the legacy IDT path, as the FRED NMI entry code relies on FRED's architectural behavior with respect to NMI blocking, i.e. doesn't jump through the myriad hoops needed to deal with IRET "unexpectedly" unmasking NMIs. KVM's NMI path already makes a direct CALL to C-code, i.e. isn't problematic for CFI. KVM does make a short detour through assembly code to build the stack frame, but the "FRED entry from KVM" path does the same.
Force FRED for 64-bit kernels if KVM_INTEL is enabled, as the benefits of eliminating the IRQ trampoline usage far outwieghts the code overhead for FRED.
--- arch/x86/kvm/Kconfig | 1 + arch/x86/kvm/vmx/vmx.c | 8 +++++++- 2 files changed, 8 insertions(+), 1 deletion(-)
arch/x86/kvm/Kconfig +1 -0
@@ -97,6 +97,7 @@ config KVM_INTEL depends on KVM && IA32_FEAT_CTL select KVM_GENERIC_PRIVATE_MEM if INTEL_TDX_HOST select KVM_GENERIC_MEMORY_ATTRIBUTES if INTEL_TDX_HOST + select X86_FRED if X86_64 help Provides support for KVM on processors equipped with Intel's VT extensions, a.k.a. Virtual Machine Extensions (VMX).
arch/x86/kvm/vmx/vmx.c +7 -1
@@ -6989,8 +6989,14 @@ static void handle_external_interrupt_ir "unexpected VM-Exit interrupt info: 0x%x", intr_info)) return; +/* +* Invoke the kernel's IRQ handler for the vector. Use the FRED path +* when it's available even if FRED isn't fully enabled, e.g. even if +* FRED isn't supported in hardware, in order to avoid the indirect +* CALL in the non-FRED path. +*/ kvm_before_interrupt (vcpu,KVM_HANDLING_IRQ ); -if (cpu_feature_enabled (X86_FEATURE_FRED )) +if (IS_ENABLED (CONFIG_X86_FRED ))fred_entry_from_kvm (EVENT_TYPE_EXTINT , vector);else vmx_do_interrupt_irqoff (gate_offset ((gate_desc *)host_idt_base + vector));
Peter Zijlstra
maintainerPATCH v3 1/16 x86/kvm/emulate: Implement test_cc() in C
In reply to Peter ZijlstraCurrent test_cc() uses the fastop infrastructure to test flags using SETcc instructions. However, int3_emulate_jcc() already fully implements the flags->CC mapping, use that.
Removes a pile of gnarly asm.
--- arch/x86/include/asm/text-patching.h | 20 +++++++++++++------- arch/x86/kvm/emulate.c | 34 ++-------------------------------- 2 files changed, 15 insertions(+), 39 deletions(-)
arch/x86/include/asm/text-patching.h +13 -7
@@ -177,9 +177,9 @@ void int3_emulate_ret(struct pt_regs *re }static __always_inline -void int3_emulate_jcc(struct pt_regs *regs, u8 cc, unsigned long ip, unsigned long disp) +bool __emulate_cc(unsigned long flags, u8 cc) { -static const unsigned long jcc_mask[6 ] = { +static const unsigned long cc_mask[6 ] = { [0 ] =X86_EFLAGS_OF , [1 ] =X86_EFLAGS_CF , [2 ] =X86_EFLAGS_ZF ,
@@ -192,15 +192,21 @@ void int3_emulate_jcc(struct pt_regs *rebool match;if (cc <0xc ) { - match = regs->flags & jcc_mask[cc >>1 ]; + match = flags & cc_mask[cc >>1 ]; }else { - match = ((regs->flags &X86_EFLAGS_SF ) >>X86_EFLAGS_SF_BIT ) ^ - ((regs->flags &X86_EFLAGS_OF ) >>X86_EFLAGS_OF_BIT ); + match = ((flags &X86_EFLAGS_SF ) >>X86_EFLAGS_SF_BIT ) ^ + ((flags &X86_EFLAGS_OF ) >>X86_EFLAGS_OF_BIT );if (cc >=0xe ) - match = match || (regs->flags &X86_EFLAGS_ZF ); + match = match || (flags &X86_EFLAGS_ZF ); } -if ((match && !invert) || (!match && invert)) +return (match && !invert) || (!match && invert); +} + +static __always_inline +void int3_emulate_jcc(struct pt_regs *regs, u8 cc, unsigned long ip, unsigned long disp) +{ +if (__emulate_cc (regs->flags, cc)) ip += disp;int3_emulate_jmp (regs, ip);
arch/x86/kvm/emulate.c +2 -32
@@ -26,6 +26,7 @@#include <asm/debugreg.h> #include <asm/nospec-branch.h> #include <asm/ibt.h> +#include <asm/text-patching.h> #include "x86.h" #include "tss.h"
@@ -416,31 +417,6 @@ static int fastop(struct x86_emulate_ctx ON64(FOP3E(op##q, rax, rdx, cl)) \ FOP_END -/* Special case for SETcc - 1 instruction per cc */ -#define FOP_SETCC (op) \ - FOP_FUNC(op) \ - #op " %al \n\t" \ - FOP_RET(op) - -FOP_START(setcc) -FOP_SETCC(seto) -FOP_SETCC(setno) -FOP_SETCC(setc) -FOP_SETCC(setnc) -FOP_SETCC(setz) -FOP_SETCC(setnz) -FOP_SETCC(setbe) -FOP_SETCC(setnbe) -FOP_SETCC(sets) -FOP_SETCC(setns) -FOP_SETCC(setp) -FOP_SETCC(setnp) -FOP_SETCC(setl) -FOP_SETCC(setnl) -FOP_SETCC(setle) -FOP_SETCC(setnle) -FOP_END; - FOP_START(salc) FOP_FUNC(salc)"pushf; sbb %al, %al; popf \n\t"
@@ -1068,13 +1044,7 @@ static int em_bsr_c(struct x86_emulate_c static __always_inline u8 test_cc(unsigned int condition, unsigned long flags) { -u8 rc; -void (*fop)(void ) = (void *)em_setcc +FASTOP_SIZE * (condition &0xf ); - - flags = (flags &EFLAGS_MASK ) |X86_EFLAGS_IF ; - asm("push %[flags]; popf; " CALL_NOSPEC - : "=a"(rc), ASM_CALL_CONSTRAINT : [thunk_target]"r"(fop), [flags]"r"(flags)); -return rc; +return __emulate_cc (flags, condition &0xf ); }static void fetch_register_operand (struct operand *op)
Peter Zijlstra
maintainerPATCH v3 8/16 x86/kvm/emulate: Introduce EM_ASM_3WCL
In reply to Peter ZijlstraReplace the FASTOP3WCL instructions.
--- arch/x86/kvm/emulate.c | 25 +++++++++++++++++++------ 1 file changed, 19 insertions(+), 6 deletions(-)
arch/x86/kvm/emulate.c +19 -6
@@ -302,6 +302,9 @@ static int em_##op(struct x86_emulate_ct#define __EM_ASM_2 (op, dst, src) \ __EM_ASM(#op " %%" #src ", %%" #dst " \n\t") +#define __EM_ASM_3 (op, dst, src, src2) \ + __EM_ASM(#op " %%" #src2 ", %%" #src ", %%" #dst " \n\t") +#define EM_ASM_END \ } \ ctxt->eflags = (ctxt->eflags & ~EFLAGS_MASK) | (flags & EFLAGS_MASK); \
@@ -371,6 +374,16 @@ static int em_##op(struct x86_emulate_ct ON64(case 8: __EM_ASM_2(op##q, rax, cl); break;) \ EM_ASM_END +/* 3-operand, using "a" (dst), "d" (src) and CL (src2) */ +#define EM_ASM_3WCL (op) \ + EM_ASM_START(op) \ + case 1: break; \ + case 2: __EM_ASM_3(op##w, ax, dx, cl); break; \ + case 4: __EM_ASM_3(op##l, eax, edx, cl); break; \ + ON64(case 8: __EM_ASM_3(op##q, rax, rdx, cl); break;) \ + EM_ASM_END + +/* * fastop functions have a special calling convention: *
@@ -1097,8 +1110,8 @@ EM_ASM_1SRC2(imul, imul_ex);EM_ASM_1SRC2EX (div, div_ex);EM_ASM_1SRC2EX (idiv, idiv_ex); -FASTOP3WCL (shld); -FASTOP3WCL (shrd); +EM_ASM_3WCL (shld); +EM_ASM_3WCL (shrd);EM_ASM_2W (imul);
@@ -4496,14 +4509,14 @@ static const struct opcode twobyte_tableI (Stack | Src2FS, em_push_sreg),I (Stack | Src2FS, em_pop_sreg),II (ImplicitOps, em_cpuid, cpuid),I (DstMem | SrcReg | ModRM | BitOp | NoWrite, em_bt), -F (DstMem | SrcReg | Src2ImmByte | ModRM, em_shld), -F (DstMem | SrcReg | Src2CL | ModRM, em_shld),N ,N , +I (DstMem | SrcReg | Src2ImmByte | ModRM, em_shld), +I (DstMem | SrcReg | Src2CL | ModRM, em_shld),N ,N ,/* 0xA8 - 0xAF */ I (Stack | Src2GS, em_push_sreg),I (Stack | Src2GS, em_pop_sreg),II (EmulateOnUD | ImplicitOps, em_rsm, rsm),I (DstMem | SrcReg | ModRM | BitOp | Lock | PageTable, em_bts), -F (DstMem | SrcReg | Src2ImmByte | ModRM, em_shrd), -F (DstMem | SrcReg | Src2CL | ModRM, em_shrd), +I (DstMem | SrcReg | Src2ImmByte | ModRM, em_shrd), +I (DstMem | SrcReg | Src2CL | ModRM, em_shrd),GD (0 , &group15),I (DstReg | SrcMem | ModRM, em_imul),/* 0xB0 - 0xB7 */ I2bv(DstMem | SrcReg | ModRM | Lock | PageTable | SrcWrite, em_cmpxchg),
Peter Zijlstra
maintainerPATCH v3 9/16 x86/kvm/emulate: Convert em_salc() to C
In reply to Peter ZijlstraImplement the SALC (Set AL if Carry) instruction in C.
--- arch/x86/kvm/emulate.c | 15 +++++++++------ 1 file changed, 9 insertions(+), 6 deletions(-)
arch/x86/kvm/emulate.c +9 -6
@@ -529,11 +529,14 @@ static int fastop(struct x86_emulate_ctx ON64(FOP3E(op##q, rax, rdx, cl)) \ FOP_END -FOP_START(salc) -FOP_FUNC(salc) -"pushf; sbb %al, %al; popf \n\t" -FOP_RET(salc) -FOP_END ; +static int em_salc (struct x86_emulate_ctxt *ctxt) +{ +/* +* Set AL 0xFF if CF is set, or 0x00 when clear. +*/ + ctxt->dst.val =0xFF * !!(ctxt->eflags &X86_EFLAGS_CF ); +return X86EMUL_CONTINUE ; +}/* * XXX: inoutclob user must know where the argument is being expanded.
@@ -4423,7 +4426,7 @@ static const struct opcode opcode_table[G (Src2CL | ByteOp, group2),G (Src2CL, group2),I (DstAcc | SrcImmUByte | No64, em_aam),I (DstAcc | SrcImmUByte | No64, em_aad), -F (DstAcc | ByteOp | No64, em_salc), +I (DstAcc | ByteOp | No64, em_salc),I (DstAcc | SrcXLat | ByteOp, em_mov),/* 0xD8 - 0xDF */ N, E(0, &escape_d9), N, E(0, &escape_db), N, E(0, &escape_dd), N, N,
Peter Zijlstra
maintainerPATCH v3 10/16 x86/kvm/emulate: Remove fastops
In reply to Peter ZijlstraNo more FASTOPs, remove the remains.
--- arch/x86/kvm/emulate.c | 172 ------------------------------------------------- 1 file changed, 1 insertion(+), 171 deletions(-)
arch/x86/kvm/emulate.c +1 -171
@@ -167,7 +167,6 @@#define Unaligned ((u64)2 << 41)/* Explicitly unaligned (e.g. MOVDQU) */ #define Avx ((u64)3 << 41)/* Advanced Vector Extensions */ #define Aligned16 ((u64)4 << 41)/* Aligned to 16 byte boundary (e.g. FXSAVE) */ -#define Fastop ((u64)1 << 44)/* Use opcode::u.fastop */ #define NoWrite ((u64)1 << 45)/* No writeback */ #define SrcWrite ((u64)1 << 46)/* Write back src operand */ #define NoMod ((u64)1 << 47)/* Mod field is ignored */
@@ -203,7 +202,6 @@ struct opcode {const struct escape *esc;const struct instr_dual *idual;const struct mode_dual *mdual; -void (*fastop)(struct fastop *fake); } u;int (*check_perm)(struct x86_emulate_ctxt *ctxt); };
@@ -383,152 +381,6 @@ static int em_##op(struct x86_emulate_ct ON64(case 8: __EM_ASM_3(op##q, rax, rdx, cl); break;) \ EM_ASM_END - -/* -* fastop functions have a special calling convention: -* -* dst: rax (in/out) -* src: rdx (in/out) -* src2: rcx (in) -* flags: rflags (in/out) -* ex: rsi (in:fastop pointer, out:zero if exception) -* -* Moreover, they are all exactly FASTOP_SIZE bytes long, so functions for -* different operand sizes can be reached by calculation, rather than a jump -* table (which would be bigger than the code). -* -* The 16 byte alignment, considering 5 bytes for the RET thunk, 3 for ENDBR -* and 1 for the straight line speculation INT3, leaves 7 bytes for the -* body of the function. Currently none is larger than 4. -*/ -static int fastop (struct x86_emulate_ctxt *ctxt,fastop_t fop); - -#define FASTOP_SIZE 16 - -#define __FOP_FUNC (name) \ - ".align " __stringify(FASTOP_SIZE) " \n\t" \ - ".type " name ", @function \n\t" \ - name ":\n\t" \ - ASM_ENDBR \ - IBT_NOSEAL(name) - -#define FOP_FUNC (name) \ - __FOP_FUNC(#name) - -#define __FOP_RET (name) \ - "11: " ASM_RET \ - ".size " name ", .-" name "\n\t" - -#define FOP_RET (name) \ - __FOP_RET(#name) - -#define __FOP_START (op, align) \ - extern void em_##op(struct fastop *fake); \ - asm(".pushsection .text, \"ax\" \n\t" \ - ".global em_" #op " \n\t" \ - ".align " __stringify(align) " \n\t" \ - "em_" #op ":\n\t" - -#define FOP_START (op) __FOP_START(op, FASTOP_SIZE) - -#define FOP_END \ - ".popsection") - -#define __FOPNOP (name) \ - __FOP_FUNC(name) \ - __FOP_RET(name) - -#define FOPNOP () \ - __FOPNOP(__stringify(__UNIQUE_ID(nop))) - -#define FOP1E (op, dst) \ - __FOP_FUNC(#op "_" #dst) \ - "10: " #op " %" #dst " \n\t" \ - __FOP_RET(#op "_" #dst) - -#define FOP1EEX (op, dst) \ - FOP1E(op, dst) _ASM_EXTABLE_TYPE_REG(10b, 11b, EX_TYPE_ZERO_REG, %%esi) - -#define FASTOP1 (op) \ - FOP_START(op) \ - FOP1E(op##b, al) \ - FOP1E(op##w, ax) \ - FOP1E(op##l, eax) \ - ON64(FOP1E(op##q, rax)) \ - FOP_END - -/* 1-operand, using src2 (for MUL/DIV r/m) */ -#define FASTOP1SRC2 (op, name) \ - FOP_START(name) \ - FOP1E(op, cl) \ - FOP1E(op, cx) \ - FOP1E(op, ecx) \ - ON64(FOP1E(op, rcx)) \ - FOP_END - -/* 1-operand, using src2 (for MUL/DIV r/m), with exceptions */ -#define FASTOP1SRC2EX (op, name) \ - FOP_START(name) \ - FOP1EEX(op, cl) \ - FOP1EEX(op, cx) \ - FOP1EEX(op, ecx) \ - ON64(FOP1EEX(op, rcx)) \ - FOP_END - -#define FOP2E (op, dst, src) \ - __FOP_FUNC(#op "_" #dst "_" #src) \ - #op " %" #src ", %" #dst " \n\t" \ - __FOP_RET(#op "_" #dst "_" #src) - -#define FASTOP2 (op) \ - FOP_START(op) \ - FOP2E(op##b, al, dl) \ - FOP2E(op##w, ax, dx) \ - FOP2E(op##l, eax, edx) \ - ON64(FOP2E(op##q, rax, rdx)) \ - FOP_END - -/* 2 operand, word only */ -#define FASTOP2W (op) \ - FOP_START(op) \ - FOPNOP() \ - FOP2E(op##w, ax, dx) \ - FOP2E(op##l, eax, edx) \ - ON64(FOP2E(op##q, rax, rdx)) \ - FOP_END - -/* 2 operand, src is CL */ -#define FASTOP2CL (op) \ - FOP_START(op) \ - FOP2E(op##b, al, cl) \ - FOP2E(op##w, ax, cl) \ - FOP2E(op##l, eax, cl) \ - ON64(FOP2E(op##q, rax, cl)) \ - FOP_END - -/* 2 operand, src and dest are reversed */ -#define FASTOP2R (op, name) \ - FOP_START(name) \ - FOP2E(op##b, dl, al) \ - FOP2E(op##w, dx, ax) \ - FOP2E(op##l, edx, eax) \ - ON64(FOP2E(op##q, rdx, rax)) \ - FOP_END - -#define FOP3E (op, dst, src, src2) \ - __FOP_FUNC(#op "_" #dst "_" #src "_" #src2) \ - #op " %" #src2 ", %" #src ", %" #dst " \n\t"\ - __FOP_RET(#op "_" #dst "_" #src "_" #src2) - -/* 3-operand, word-only, src2=cl */ -#define FASTOP3WCL (op) \ - FOP_START(op) \ - FOPNOP() \ - FOP3E(op##w, ax, dx, cl) \ - FOP3E(op##l, eax, edx, cl) \ - ON64(FOP3E(op##q, rax, rdx, cl)) \ - FOP_END -static int em_salc (struct x86_emulate_ctxt *ctxt) {/*
@@ -4052,7 +3904,6 @@ static int check_perm_out(struct x86_emu#define MD (_f, _m) { .flags = ((_f) | ModeDual), .u.mdual = (_m) }#define E (_f, _e) { .flags = ((_f) | Escape | ModRM), .u.esc = (_e) }#define I (_f, _e) { .flags = (_f), .u.execute = (_e) } -#define F (_f, _e) { .flags = (_f) | Fastop, .u.fastop = (_e) }#define II (_f, _e, _i) \ { .flags = (_f)|Intercept, .u.execute = (_e), .intercept = x86_intercept_##_i }#define IIP (_f, _e, _i, _p) \
@@ -5158,24 +5009,6 @@ static void fetch_possible_mmx_operand(skvm_read_mmx_reg (op->addr.mm, &op->mm_val); } -static int fastop (struct x86_emulate_ctxt *ctxt,fastop_t fop) -{ -ulong flags = (ctxt->eflags &EFLAGS_MASK ) |X86_EFLAGS_IF ; - -if (!(ctxt->d & ByteOp)) - fop +=__ffs (ctxt->dst.bytes) *FASTOP_SIZE ; - - asm("push %[flags]; popf; " CALL_NOSPEC " ; pushf; pop %[flags]\n" - :"+a" (ctxt->dst.val),"+d" (ctxt->src.val), [flags]"+D" (flags), - [thunk_target]"+S"(fop), ASM_CALL_CONSTRAINT - :"c" (ctxt->src2.val)); - - ctxt->eflags = (ctxt->eflags & ~EFLAGS_MASK ) | (flags &EFLAGS_MASK ); -if (!fop)/* exception is returned in fop variable */ -return emulate_de (ctxt); -return X86EMUL_CONTINUE ; -} -void init_decode_cache (struct x86_emulate_ctxt *ctxt) {/* Clear fields that are set conditionally but read without a guard. */
@@ -5340,10 +5173,7 @@ int x86_emulate_insn(struct x86_emulate_ ctxt->eflags &= ~X86_EFLAGS_RF ;if (ctxt->execute) { -if (ctxt->d & Fastop) - rc =fastop (ctxt, ctxt->fop); -else - rc = ctxt->execute (ctxt); + rc = ctxt->execute (ctxt);if (rc !=X86EMUL_CONTINUE ) goto done; goto writeback;
Peter Zijlstra
maintainerPATCH v3 12/16 x86_64,hyperv: Use direct call to hypercall-page
In reply to Peter ZijlstraInstead of using an indirect call to the hypercall page, use a direct call instead. This avoids all CFI problems, including the one where the hypercall page doesn't have IBT on.
--- arch/x86/hyperv/hv_init.c | 61 ++++++++++++++++++++++------------------------ 1 file changed, 30 insertions(+), 31 deletions(-)
arch/x86/hyperv/hv_init.c +30 -31
@@ -17,7 +17,6 @@#include <asm/desc.h> #include <asm/e820/api.h> #include <asm/sev.h> -#include <asm/ibt.h> #include <asm/hypervisor.h> #include <hyperv/hvhdk.h> #include <asm/mshyperv.h>
@@ -38,23 +37,41 @@void *hv_hypercall_pg;#ifdef CONFIG_X86_64 +static u64 __hv_hyperfail (u64 control,u64 param1,u64 param2) +{ +return U64_MAX ; +} + +DEFINE_STATIC_CALL (__hv_hypercall, __hv_hyperfail); +u64 hv_std_hypercall (u64 control,u64 param1,u64 param2) {u64 hv_status; -if (!hv_hypercall_pg) -return U64_MAX ; - register u64 __r8 asm("r8") = param2; - asm volatile (CALL_NOSPEC + asm volatile ("call " STATIC_CALL_TRAMP_STR(__hv_hypercall) : "=a" (hv_status), ASM_CALL_CONSTRAINT, "+c" (control), "+d" (param1), "+r" (__r8) - : THUNK_TARGET(hv_hypercall_pg) - : "cc", "memory", "r9", "r10", "r11"); + : : "cc", "memory", "r9", "r10", "r11");return hv_status; } + +typedef u64 (*hv_hypercall_f )(u64 control,u64 param1,u64 param2); + +static inline void hv_set_hypercall_pg (void *ptr) +{ + hv_hypercall_pg = ptr; + +if (!ptr) + ptr = &__hv_hyperfail; +static_call_update (__hv_hypercall, (hv_hypercall_f )ptr); +}#else +static inline void hv_set_hypercall_pg (void *ptr) +{ + hv_hypercall_pg = ptr; +}EXPORT_SYMBOL_GPL (hv_hypercall_pg);#endif
@@ -349,7 +366,7 @@ static int hv_suspend(void)* pointer is restored on resume. */ hv_hypercall_pg_saved = hv_hypercall_pg; - hv_hypercall_pg =NULL ; +hv_set_hypercall_pg (NULL );/* Disable the hypercall page in the hypervisor */ rdmsrq (HV_X64_MSR_HYPERCALL , hypercall_msr.as_uint64);
@@ -375,7 +392,7 @@ static void hv_resume(void)vmalloc_to_pfn (hv_hypercall_pg_saved);wrmsrq (HV_X64_MSR_HYPERCALL , hypercall_msr.as_uint64); - hv_hypercall_pg = hv_hypercall_pg_saved; +hv_set_hypercall_pg (hv_hypercall_pg_saved); hv_hypercall_pg_saved =NULL ;/*
@@ -529,8 +546,8 @@ void __init hyperv_init(void)if (hv_isolation_type_tdx () && !ms_hyperv.paravisor_present) goto skip_hypercall_pg_init; - hv_hypercall_pg =__vmalloc_node_range (PAGE_SIZE ,1 ,VMALLOC_START , -VMALLOC_END ,GFP_KERNEL ,PAGE_KERNEL_ROX , + hv_hypercall_pg =__vmalloc_node_range (PAGE_SIZE ,1 ,MODULES_VADDR , +MODULES_END ,GFP_KERNEL ,PAGE_KERNEL_ROX ,VM_FLUSH_RESET_PERMS ,NUMA_NO_NODE ,__builtin_return_address (0 ));if (hv_hypercall_pg ==NULL )
@@ -568,27 +585,9 @@ void __init hyperv_init(void)wrmsrq (HV_X64_MSR_HYPERCALL , hypercall_msr.as_uint64); } -skip_hypercall_pg_init: -/* -* Some versions of Hyper-V that provide IBT in guest VMs have a bug -* in that there's no ENDBR64 instruction at the entry to the -* hypercall page. Because hypercalls are invoked via an indirect call -* to the hypercall page, all hypercall attempts fail when IBT is -* enabled, and Linux panics. For such buggy versions, disable IBT. -* -* Fixed versions of Hyper-V always provide ENDBR64 on the hypercall -* page, so if future Linux kernel versions enable IBT for 32-bit -* builds, additional hypercall page hackery will be required here -* to provide an ENDBR32. - */ -#ifdef CONFIG_X86_KERNEL_IBT -if (cpu_feature_enabled (X86_FEATURE_IBT ) && - *(u32 *)hv_hypercall_pg !=gen_endbr ()) { -setup_clear_cpu_cap (X86_FEATURE_IBT ); -pr_warn ("Disabling IBT because of Hyper-V bug\n" ); - } -#endif +hv_set_hypercall_pg (hv_hypercall_pg); +skip_hypercall_pg_init:/* * hyperv_init() is called before LAPIC is initialized: see * apic_intr_mode_init() -> x86_platform.apic_post_init() and
Peter Zijlstra
maintainerPATCH v3 14/16 x86/fred: Play nice with invoking asm_fred_entry_from_kvm() on non-FRED hardware
In reply to Peter ZijlstraFrom: Josh Poimboeuf <[email protected]>
Modify asm_fred_entry_from_kvm() to allow it to be invoked by KVM even when FRED isn't fully enabled, e.g. when running with CONFIG_X86_FRED=y on non-FRED hardware. This will allow forcing KVM to always use the FRED entry points for 64-bit kernels, which in turn will eliminate a rather gross non-CFI indirect call that KVM uses to trampoline IRQs by doing IDT lookups.
The point of asm_fred_entry_from_kvm() is to bridge between C (vmx:handle_external_interrupt_irqoff()) and more C (__fred_entry_from_kvm()) while changing the calling context to appear like an interrupt (pt_regs). Making the whole thing bound by C ABI.
All that remains for non-FRED hardware is to restore RSP (to undo the redzone and alignment). However the trivial change would result in code like:
push %rbp
mov %rsp, %rbp
sub $REDZONE, %rsp
and $MASK, %rsp
PUSH_AND_CLEAR_REGS
push %rbp
POP_REGS
pop %rbp <-- *objtool fail*
mov %rbp, %rsp
pop %rbp
ret
And this will confuse objtool something wicked -- it gets confused by the extra pop %rbp, not realizing the push and pop preserve the value.
Rather than trying to each objtool about this, recognise that since the code is bound by C ABI on both ends and interrupts are not allowed to change pt_regs (only exceptions are) it is sufficient to PUSH_REGS in order to create pt_regs, but there is no reason to POP_REGS -- provided the callee-saved registers are preserved.
So avoid clearing callee-saved regs and skip POP_REGS.
[Original patch by Sean; much of this version by Josh; Changelog, comments and final form by Peterz]
--- arch/x86/entry/calling.h | 11 +++++------ arch/x86/entry/entry_64_fred.S | 33 ++++++++++++++++++++++++++------- arch/x86/kernel/asm-offsets.c | 1 + 3 files changed, 32 insertions(+), 13 deletions(-)
arch/x86/entry/calling.h +5 -6
@@ -99,7 +99,7 @@ For 32-bit we have the following convent .endif .endm -.macro CLEAR_REGS clear_bp=1 +.macro CLEAR_REGS clear_callee=1/* * Sanitize registers of values that a speculation attack might * otherwise want to exploit. The lower registers are likely clobbered
@@ -113,20 +113,19 @@ For 32-bit we have the following convent xorl %r9d, %r9d /* nospec r9 */ xorl %r10d, %r10d /* nospec r10 */ xorl %r11d, %r11d /* nospec r11 */ + .if \clear_callee xorl %ebx, %ebx /* nospec rbx */ - .if \clear_bp xorl %ebp, %ebp /* nospec rbp */ - .endif xorl %r12d, %r12d /* nospec r12 */ xorl %r13d, %r13d /* nospec r13 */ xorl %r14d, %r14d /* nospec r14 */ xorl %r15d, %r15d /* nospec r15 */ - + .endif .endm -.macro PUSH_AND_CLEAR_REGS rdx=%rdx rcx=%rcx rax=%rax save_ret=0 clear_bp=1 unwind_hint=1 +.macro PUSH_AND_CLEAR_REGS rdx=%rdx rcx=%rcx rax=%rax save_ret=0 clear_callee=1 unwind_hint=1 PUSH_REGS rdx=\rdx, rcx=\rcx, rax=\rax, save_ret=\save_ret unwind_hint=\unwind_hint - CLEAR_REGS clear_bp=\clear_bp + CLEAR_REGS clear_callee=\clear_callee .endm .macro POP_REGS pop_rdi=1
arch/x86/entry/entry_64_fred.S +26 -7
@@ -112,18 +112,37 @@ SYM_FUNC_START(asm_fred_entry_from_kvm) push %rax /* Return RIP */ push $0 /* Error code, 0 for IRQ/NMI */ - PUSH_AND_CLEAR_REGS clear_bp=0 unwind_hint=0 + PUSH_AND_CLEAR_REGS clear_callee=0 unwind_hint=0 + movq %rsp, %rdi /* %rdi -> pt_regs */ + /* + * At this point: {rdi, rsi, rdx, rcx, r8, r9}, {r10, r11}, {rax, rdx} + * are clobbered, which corresponds to: arguments, extra caller-saved + * and return. All registers a C function is allowed to clobber. + * + * Notably, the callee-saved registers: {rbx, r12, r13, r14, r15} + * are untouched, with the exception of rbp, which carries the stack + * frame and will be restored before exit. + * + * Further calling another C function will not alter this state. + */ call __fred_entry_from_kvm /* Call the C entry point */ - POP_REGS - ERETS -1: + +1: /* + * When FRED, use ERETS to potentially clear NMIs, otherwise simply + * restore the stack pointer. + */ + ALTERNATIVE "nop; nop; mov %rbp, %rsp", \ + __stringify(add $C_PTREGS_SIZE, %rsp; ERETS), \ + X86_FEATURE_FRED + /* - * Objtool doesn't understand what ERETS does, this hint tells it that - * yes, we'll reach here and with what stack state. A save/restore pair - * isn't strictly needed, but it's the simplest form. + * Objtool doesn't understand ERETS, and the cfi register state is + * different from initial_func_cfi due to PUSH_REGS. Tell it the state + * is similar to where UNWIND_HINT_SAVE is. */ UNWIND_HINT_RESTORE + pop %rbp RET
arch/x86/kernel/asm-offsets.c +1 -0
@@ -102,6 +102,7 @@ static void __used common(void)BLANK ();DEFINE (PTREGS_SIZE ,sizeof (struct pt_regs )); +OFFSET (C_PTREGS_SIZE , pt_regs, orig_ax);/* TLB state for the entry code */ OFFSET (TLB_STATE_user_pcid_flush_mask, tlb_state, user_pcid_flush_mask);
Peter Zijlstra
maintainerPATCH v3 16/16 objtool: Validate kCFI calls
In reply to Peter ZijlstraValidate that all indirect calls adhere to kCFI rules. Notably doing nocfi indirect call to a cfi function is broken.
Apparently some Rust 'core' code violates this and explodes when ran with FineIBT.
All the ANNOTATE_NOCFI_SYM sites are prime targets for attackers.
- runtime EFI is especially henous because it also needs to disable IBT. Basically calling unknown code without CFI protection at runtime is a massice security issue.
- Kexec image handover; if you can exploit this, you get to keep it :-)
--- arch/x86/kernel/machine_kexec_64.c | 4 +++ arch/x86/kvm/vmx/vmenter.S | 4 +++ arch/x86/platform/efi/efi_stub_64.S | 4 +++ drivers/misc/lkdtm/perms.c | 5 ++++ include/linux/objtool.h | 10 ++++++++ include/linux/objtool_types.h | 1 tools/include/linux/objtool_types.h | 1 tools/objtool/check.c | 42 ++++++++++++++++++++++++++++++++++++ tools/objtool/include/objtool/elf.h | 1 9 files changed, 72 insertions(+)
arch/x86/kernel/machine_kexec_64.c +4 -0
@@ -453,6 +453,10 @@ void __nocfi machine_kexec(struct kimage__ftrace_enabled_restore (save_ftrace_enabled); } +/* +* Handover to the next kernel, no CFI concern. +*/ +ANNOTATE_NOCFI_SYM (machine_kexec);/* arch-dependent functionality related to kexec file-based syscall */
arch/x86/kvm/vmx/vmenter.S +4 -0
@@ -361,6 +361,10 @@ SYM_FUNC_END(vmread_error_trampoline) .section .text, "ax" +#ifndef CONFIG_X86_FRED + SYM_FUNC_START(vmx_do_interrupt_irqoff) VMX_DO_EVENT_IRQOFF CALL_NOSPEC _ASM_ARG1 SYM_FUNC_END(vmx_do_interrupt_irqoff) + +#endif
arch/x86/platform/efi/efi_stub_64.S +4 -0
@@ -11,6 +11,10 @@ #include <asm/nospec-branch.h> SYM_FUNC_START(__efi_call) + /* + * The EFI code doesn't have any CFI, annotate away the CFI violation. + */ + ANNOTATE_NOCFI_SYM pushq %rbp movq %rsp, %rbp and $~0xf, %rsp
drivers/misc/lkdtm/perms.c +5 -0
@@ -9,6 +9,7 @@#include <linux/vmalloc.h> #include <linux/mman.h> #include <linux/uaccess.h> +#include <linux/objtool.h> #include <asm/cacheflush.h> #include <asm/sections.h>
@@ -86,6 +87,10 @@ static noinline __nocfi void execute_locfunc ();pr_err ("FAIL: func returned\n" ); } +/* +* Explicitly doing the wrong thing for testing. +*/ +ANNOTATE_NOCFI_SYM (execute_location);static void execute_user_location (void *dst) {
include/linux/objtool.h +10 -0
@@ -184,6 +184,15 @@* WARN using UD2. */ #define ANNOTATE_REACHABLE (label) __ASM_ANNOTATE(label, ANNOTYPE_REACHABLE) +/* +* This should not be used; it annotates away CFI violations. There are a few +* valid use cases like kexec handover to the next kernel image, and there is +* no security concern there. +* +* There are also a few real issues annotated away, like EFI because we can't +* control the EFI code. +*/ +#define ANNOTATE_NOCFI_SYM (sym) asm(__ASM_ANNOTATE(sym, ANNOTYPE_NOCFI))#else #define ANNOTATE_NOENDBR ANNOTATE type=ANNOTYPE_NOENDBR
@@ -194,6 +203,7 @@#define ANNOTATE_INTRA_FUNCTION_CALL ANNOTATE type=ANNOTYPE_INTRA_FUNCTION_CALL#define ANNOTATE_UNRET_BEGIN ANNOTATE type=ANNOTYPE_UNRET_BEGIN#define ANNOTATE_REACHABLE ANNOTATE type=ANNOTYPE_REACHABLE +#define ANNOTATE_NOCFI_SYM ANNOTATE type=ANNOTYPE_NOCFI#endif #if defined(CONFIG_NOINSTR_VALIDATION) && \
include/linux/objtool_types.h +1 -0
@@ -65,5 +65,6 @@ struct unwind_hint {#define ANNOTYPE_IGNORE_ALTS 6#define ANNOTYPE_INTRA_FUNCTION_CALL 7#define ANNOTYPE_REACHABLE 8 +#define ANNOTYPE_NOCFI 9#endif /* _LINUX_OBJTOOL_TYPES_H */
tools/include/linux/objtool_types.h +1 -0
@@ -65,5 +65,6 @@ struct unwind_hint {#define ANNOTYPE_IGNORE_ALTS 6#define ANNOTYPE_INTRA_FUNCTION_CALL 7#define ANNOTYPE_REACHABLE 8 +#define ANNOTYPE_NOCFI 9#endif /* _LINUX_OBJTOOL_TYPES_H */
tools/objtool/check.c +42 -0
60 lines not on this page · raw message
tools/objtool/include/objtool/elf.h +1 -0
@@ -70,6 +70,7 @@ struct symbol { u8 local_label : 1; u8 frame_pointer : 1; u8 ignore : 1; + u8 nocfi : 1;struct list_head pv_target;struct reloc *relocs;struct section *group_sec;
Peter Zijlstra
maintainerv3 cover objtool: Detect and warn about indirect calls in __nocfi functions
Hi!
On kCFI (CONFIG_CFI_CLANG=y) builds all indirect calls should have the CFI check on (with very few exceptions). Not having the CFI checks undermines the protection provided by CFI and will make these sites candidates for people wanting to steal your cookies.
Specifically the ABI changes are so that doing indirect calls without the CFI magic, to a CFI adorned function is not compatible (although it happens to work for some setups, it very much does not for FineIBT).
Rust people tripped over this the other day, since their 'core' happened to have some no_sanitize(kcfi) bits in, which promptly exploded when ran with FineIBT on.
Since this is very much not a supported model -- on purpose, have objtool detect and warn about such constructs.
This effort [1] found all existing [2] non-cfi indirect calls in the kernel.
Notably the KVM fastop emulation stuff -- which is completely rewritten -- the generated code doesn't look horrific, but is slightly more verbose. I'm running on the assumption that instruction emulation is not super performance critical these days of zero VM-exit VMs etc. Paolo noted that pre-Westmere (2010) cares about this.
KVM has another; the VMX interrupt injection stuff calls the IDT handler directly. This is rewritten to to use the FRED dispatch table, which moves it all into C.
HyperV hypercall page stuff, which I've previously suggested use direct calls, and which I've now converted (after getting properly annoyed with that code).
Also available at:
git://git.kernel.org/pub/scm/linux/kernel/git/peterz/queue.git x86/core
Changes since v2:
- renamed COP to EM_ASM
- reworked the KVM-IDT stuff (Sean, Josh)
[1] https://lkml.kernel.org/r/[email protected]
[2] https://lkml.kernel.org/r/[email protected]
Peter Zijlstra
maintainerPATCH v3 3/16 x86/kvm/emulate: Introduce EM_ASM_2
Replace the FASTOP2 instructions.
--- arch/x86/kvm/emulate.c | 85 +++++++++++++++++++++++++++---------------------- 1 file changed, 47 insertions(+), 38 deletions(-)
arch/x86/kvm/emulate.c +47 -38
@@ -317,6 +317,15 @@ static int em_##op(struct x86_emulate_ct ON64(case 8: __EM_ASM_1(op##q, rax); break;) \ EM_ASM_END +/* 2-operand, using "a" (dst), "d" (src) */ +#define EM_ASM_2 (op) \ + EM_ASM_START(op) \ + case 1: __EM_ASM_2(op##b, al, dl); break; \ + case 2: __EM_ASM_2(op##w, ax, dx); break; \ + case 4: __EM_ASM_2(op##l, eax, edx); break; \ + ON64(case 8: __EM_ASM_2(op##q, rax, rdx); break;) \ + EM_ASM_END +/* * fastop functions have a special calling convention: *
@@ -1027,15 +1036,16 @@ static int read_descriptor(struct x86_emreturn rc; } -FASTOP2 (add); -FASTOP2 (or); -FASTOP2 (adc); -FASTOP2 (sbb); -FASTOP2 (and); -FASTOP2 (sub); -FASTOP2 (xor); -FASTOP2 (cmp); -FASTOP2 (test); +EM_ASM_2 (add); +EM_ASM_2 (or); +EM_ASM_2 (adc); +EM_ASM_2 (sbb); +EM_ASM_2 (and); +EM_ASM_2 (sub); +EM_ASM_2 (xor); +EM_ASM_2 (cmp); +EM_ASM_2 (test); +EM_ASM_2 (xadd);FASTOP1SRC2 (mul, mul_ex);FASTOP1SRC2 (imul, imul_ex);
@@ -1067,7 +1077,6 @@ FASTOP2W(bts);FASTOP2W (btr);FASTOP2W (btc); -FASTOP2 (xadd);FASTOP2R (cmp, cmp_r);
@@ -2304,7 +2313,7 @@ static int em_cmpxchg(struct x86_emulate ctxt->dst.val =reg_read (ctxt,VCPU_REGS_RAX ); ctxt->src.orig_val = ctxt->src.val; ctxt->src.val = ctxt->dst.orig_val; -fastop (ctxt, em_cmp); +em_cmp (ctxt);if (ctxt->eflags &X86_EFLAGS_ZF ) {/* Success: write back to memory; no update of EAX */
@@ -3069,7 +3078,7 @@ static int em_das(struct x86_emulate_ctx ctxt->src.type =OP_IMM ; ctxt->src.val =0 ; ctxt->src.bytes =1 ; -fastop (ctxt, em_or); +em_or (ctxt); ctxt->eflags &= ~(X86_EFLAGS_AF |X86_EFLAGS_CF );if (cf) ctxt->eflags |=X86_EFLAGS_CF ;
@@ -3095,7 +3104,7 @@ static int em_aam(struct x86_emulate_ctx ctxt->src.type =OP_IMM ; ctxt->src.val =0 ; ctxt->src.bytes =1 ; -fastop (ctxt, em_or); +em_or (ctxt);return X86EMUL_CONTINUE ; }
@@ -3113,7 +3122,7 @@ static int em_aad(struct x86_emulate_ctx ctxt->src.type =OP_IMM ; ctxt->src.val =0 ; ctxt->src.bytes =1 ; -fastop (ctxt, em_or); +em_or (ctxt);return X86EMUL_CONTINUE ; }
@@ -3998,9 +4007,9 @@ static int check_perm_out(struct x86_emu#define I2bvIP (_f, _e, _i, _p) \ IIP((_f) | ByteOp, _e, _i, _p), IIP(_f, _e, _i, _p) -#define F6ALU (_f, _e) F2bv((_f) | DstMem | SrcReg | ModRM, _e), \ - F2bv(((_f) | DstReg | SrcMem | ModRM) & ~Lock, _e), \ - F2bv(((_f) & ~Lock) | DstAcc | SrcImm, _e) +#define I6ALU (_f, _e) I2bv((_f) | DstMem | SrcReg | ModRM, _e), \ + I2bv(((_f) | DstReg | SrcMem | ModRM) & ~Lock, _e), \ + I2bv(((_f) & ~Lock) | DstAcc | SrcImm, _e)static const struct opcode group7_rm0[] = {N ,
@@ -4038,14 +4047,14 @@ static const struct opcode group7_rm7[] };static const struct opcode group1[] = { -F (Lock, em_add), -F (Lock | PageTable, em_or), -F (Lock, em_adc), -F (Lock, em_sbb), -F (Lock | PageTable, em_and), -F (Lock, em_sub), -F (Lock, em_xor), -F (NoWrite, em_cmp), +I (Lock, em_add), +I (Lock | PageTable, em_or), +I (Lock, em_adc), +I (Lock, em_sbb), +I (Lock | PageTable, em_and), +I (Lock, em_sub), +I (Lock, em_xor), +I (NoWrite, em_cmp), };static const struct opcode group1A[] = {
@@ -4064,8 +4073,8 @@ static const struct opcode group2[] = { };static const struct opcode group3[] = { -F (DstMem | SrcImm | NoWrite, em_test), -F (DstMem | SrcImm | NoWrite, em_test), +I (DstMem | SrcImm | NoWrite, em_test), +I (DstMem | SrcImm | NoWrite, em_test),I (DstMem | SrcNone | Lock, em_not),I (DstMem | SrcNone | Lock, em_neg),F (DstXacc | Src2Mem, em_mul_ex),
@@ -4258,29 +4267,29 @@ static const struct instr_dual instr_duastatic const struct opcode opcode_table[256 ] = {/* 0x00 - 0x07 */ -F6ALU (Lock, em_add), +I6ALU (Lock, em_add),I (ImplicitOps | Stack | No64 | Src2ES, em_push_sreg),I (ImplicitOps | Stack | No64 | Src2ES, em_pop_sreg),/* 0x08 - 0x0F */ -F6ALU (Lock | PageTable, em_or), +I6ALU (Lock | PageTable, em_or),I (ImplicitOps | Stack | No64 | Src2CS, em_push_sreg),N ,/* 0x10 - 0x17 */ -F6ALU (Lock, em_adc), +I6ALU (Lock, em_adc),I (ImplicitOps | Stack | No64 | Src2SS, em_push_sreg),I (ImplicitOps | Stack | No64 | Src2SS, em_pop_sreg),/* 0x18 - 0x1F */ -F6ALU (Lock, em_sbb), +I6ALU (Lock, em_sbb),I (ImplicitOps | Stack | No64 | Src2DS, em_push_sreg),I (ImplicitOps | Stack | No64 | Src2DS, em_pop_sreg),/* 0x20 - 0x27 */ -F6ALU (Lock | PageTable, em_and),N ,N , +I6ALU (Lock | PageTable, em_and),N ,N ,/* 0x28 - 0x2F */ -F6ALU (Lock, em_sub),N ,I (ByteOp | DstAcc | No64, em_das), +I6ALU (Lock, em_sub),N ,I (ByteOp | DstAcc | No64, em_das),/* 0x30 - 0x37 */ -F6ALU (Lock, em_xor),N ,N , +I6ALU (Lock, em_xor),N ,N ,/* 0x38 - 0x3F */ -F6ALU (NoWrite, em_cmp),N ,N , +I6ALU (NoWrite, em_cmp),N ,N ,/* 0x40 - 0x4F */ X8 (I (DstReg, em_inc)),X8 (I (DstReg, em_dec)),/* 0x50 - 0x57 */
@@ -4306,7 +4315,7 @@ static const struct opcode opcode_table[G (DstMem | SrcImm, group1),G (ByteOp | DstMem | SrcImm | No64, group1),G (DstMem | SrcImmByte, group1), -F2bv (DstMem | SrcReg | ModRM | NoWrite, em_test), +I2bv (DstMem | SrcReg | ModRM | NoWrite, em_test),I2bv (DstMem | SrcReg | ModRM | Lock | PageTable, em_xchg),/* 0x88 - 0x8F */ I2bv(DstMem | SrcReg | ModRM | Mov | PageTable, em_mov),
@@ -4329,7 +4338,7 @@ static const struct opcode opcode_table[I2bv (SrcSI | DstDI | Mov | String | TwoMemOp, em_mov),F2bv (SrcSI | DstDI | String | NoWrite | TwoMemOp, em_cmp_r),/* 0xA8 - 0xAF */ -F2bv (DstAcc | SrcImm | NoWrite, em_test), +I2bv (DstAcc | SrcImm | NoWrite, em_test),I2bv (SrcAcc | DstDI | Mov | String, em_mov),I2bv (SrcSI | DstAcc | Mov | String, em_mov), F2bv(SrcAcc | DstDI | String | NoWrite, em_cmp_r),
@@ -4467,7 +4476,7 @@ static const struct opcode twobyte_tableI (DstReg | SrcMem | ModRM, em_bsr_c),D (DstReg | SrcMem8 | ModRM | Mov),D (DstReg | SrcMem16 | ModRM | Mov),/* 0xC0 - 0xC7 */ -F2bv (DstMem | SrcReg | ModRM | SrcWrite | Lock, em_xadd), +I2bv (DstMem | SrcReg | ModRM | SrcWrite | Lock, em_xadd),N ,ID (0 , &instr_dual_0f_c3), N, N, N, GD(0, &group9),/* 0xC8 - 0xCF */
Peter Zijlstra
maintainerPATCH v3 13/16 x86/fred: Install system vector handlers even if FRED isnt fully enabled
In reply to Peter ZijlstraFrom: Sean Christopherson <[email protected]>
Install the system vector IRQ handlers for FRED even if FRED isn't fully enabled in hardware. This will allow KVM to use the FRED IRQ path even on non-FRED hardware, which in turn will eliminate a non-CFI indirect CALL (KVM currently invokes the IRQ handler via an IDT lookup on the vector).
--- arch/x86/include/asm/idtentry.h | 9 ++------- arch/x86/kernel/irqinit.c | 6 ++++-- 2 files changed, 6 insertions(+), 9 deletions(-)
arch/x86/include/asm/idtentry.h +2 -7
@@ -460,17 +460,12 @@ __visible noinstr void func(struct pt_re#endif void idt_install_sysvec (unsigned int n,const void *function); - -#ifdef CONFIG_X86_FRED void fred_install_sysvec (unsigned int vector,const idtentry_t function); -#else -static inline void fred_install_sysvec (unsigned int vector,const idtentry_t function) { } -#endif #define sysvec_install (vector, function) { \ - if (cpu_feature_enabled(X86_FEATURE_FRED)) \ + if (IS_ENABLED(CONFIG_X86_FRED)) \ fred_install_sysvec(vector, function); \ - else \ + if (!cpu_feature_enabled(X86_FEATURE_FRED)) \ idt_install_sysvec(vector, asm_##function); \ }
arch/x86/kernel/irqinit.c +4 -2
@@ -97,9 +97,11 @@ void __init native_init_IRQ(void)/* Execute any quirks before the call gates are initialised: */ x86_init.irqs.pre_vector_init (); -if (cpu_feature_enabled (X86_FEATURE_FRED )) +/* FRED's IRQ path may be used even if FRED isn't fully enabled. */ +if (IS_ENABLED (CONFIG_X86_FRED ))fred_complete_exception_setup (); -else + +if (!cpu_feature_enabled (X86_FEATURE_FRED ))idt_setup_apic_and_irq_gates ();lapic_assign_system_vectors ();
Peter Zijlstra
maintainerPATCH v3 2/16 x86/kvm/emulate: Introduce EM_ASM_1
In reply to Peter ZijlstraReplace fastops with C based stubs. There are a bunch of problems with the current fastop infrastructure, most all related to their special calling convention, which bypasses the normal C-ABI.
There are two immediate problems with this at present:
- it relies on RET preserving EFLAGS; whereas C-ABI does not.
- it circumvents compiler based control-flow-integrity checking because its all asm magic.
The first is a problem for some mitigations where the x86_indirect_return_thunk needs to include non-trivial work that clobbers EFLAGS (eg. the Skylake call depth tracking thing).
The second is a problem because it presents a 'naked' indirect call on kCFI builds, making it a prime target for control flow hijacking.
Additionally, given that a large chunk of virtual machine performance relies on absolutely avoiding vmexit these days, this emulation stuff just isn't that critical for performance anymore.
As such, replace the fastop calls with normal C functions using the 'execute' member.
As noted by Paolo: this code was performance critical for pre-Westmere (2010) and only when running big real mode code.
--- arch/x86/kvm/emulate.c | 71 ++++++++++++++++++++++++++++++++++++++++--------- 1 file changed, 58 insertions(+), 13 deletions(-)
arch/x86/kvm/emulate.c +58 -13
@@ -267,11 +267,56 @@ static void invalidate_registers(struct X86_EFLAGS_PF|X86_EFLAGS_CF)#ifdef CONFIG_X86_64 -#define ON64 (x) x +#define ON64(x...) x#else -#define ON64 (x) +#define ON64(x...)#endif +#define EM_ASM_START (op) \ +static int em_##op(struct x86_emulate_ctxt *ctxt) \ +{ \ + unsigned long flags = (ctxt->eflags & EFLAGS_MASK) | X86_EFLAGS_IF; \ + int bytes = 1, ok = 1; \ + if (!(ctxt->d & ByteOp)) \ + bytes = ctxt->dst.bytes; \ + switch (bytes) { + +#define __EM_ASM (str) \ + asm("push %[flags]; popf \n\t" \ + "10: " str \ + "pushf; pop %[flags] \n\t" \ + "11: \n\t" \ + : "+a" (ctxt->dst.val), \ + "+d" (ctxt->src.val), \ + [flags] "+D" (flags), \ + "+S" (ok) \ + : "c" (ctxt->src2.val)) + +#define __EM_ASM_1 (op, dst) \ + __EM_ASM(#op " %%" #dst " \n\t") + +#define __EM_ASM_1_EX (op, dst) \ + __EM_ASM(#op " %%" #dst " \n\t" \ + _ASM_EXTABLE_TYPE_REG(10b, 11f, EX_TYPE_ZERO_REG, %%esi)) + +#define __EM_ASM_2 (op, dst, src) \ + __EM_ASM(#op " %%" #src ", %%" #dst " \n\t") + +#define EM_ASM_END \ + } \ + ctxt->eflags = (ctxt->eflags & ~EFLAGS_MASK) | (flags & EFLAGS_MASK); \ + return !ok ? emulate_de(ctxt) : X86EMUL_CONTINUE; \ +} + +/* 1-operand, using "a" (dst) */ +#define EM_ASM_1 (op) \ + EM_ASM_START(op) \ + case 1: __EM_ASM_1(op##b, al); break; \ + case 2: __EM_ASM_1(op##w, ax); break; \ + case 4: __EM_ASM_1(op##l, eax); break; \ + ON64(case 8: __EM_ASM_1(op##q, rax); break;) \ + EM_ASM_END +/* * fastop functions have a special calling convention: *
@@ -1002,10 +1047,10 @@ FASTOP3WCL(shrd);FASTOP2W (imul); -FASTOP1 (not); -FASTOP1 (neg); -FASTOP1 (inc); -FASTOP1 (dec); +EM_ASM_1 (not); +EM_ASM_1 (neg); +EM_ASM_1 (inc); +EM_ASM_1 (dec);FASTOP2CL (rol);FASTOP2CL (ror);
@@ -4021,8 +4066,8 @@ static const struct opcode group2[] = {static const struct opcode group3[] = {F (DstMem | SrcImm | NoWrite, em_test),F (DstMem | SrcImm | NoWrite, em_test), -F (DstMem | SrcNone | Lock, em_not), -F (DstMem | SrcNone | Lock, em_neg), +I (DstMem | SrcNone | Lock, em_not), +I (DstMem | SrcNone | Lock, em_neg),F (DstXacc | Src2Mem, em_mul_ex),F (DstXacc | Src2Mem, em_imul_ex),F (DstXacc | Src2Mem, em_div_ex),
@@ -4030,14 +4075,14 @@ static const struct opcode group3[] = { };static const struct opcode group4[] = { -F (ByteOp | DstMem | SrcNone | Lock, em_inc), -F (ByteOp | DstMem | SrcNone | Lock, em_dec), +I (ByteOp | DstMem | SrcNone | Lock, em_inc), +I (ByteOp | DstMem | SrcNone | Lock, em_dec),N ,N ,N ,N ,N ,N , };static const struct opcode group5[] = { -F (DstMem | SrcNone | Lock, em_inc), -F (DstMem | SrcNone | Lock, em_dec), +I (DstMem | SrcNone | Lock, em_inc), +I (DstMem | SrcNone | Lock, em_dec),I (SrcMem | NearBranch | IsBranch, em_call_near_abs),I (SrcMemFAddr | ImplicitOps | IsBranch, em_call_far),I (SrcMem | NearBranch | IsBranch, em_jmp_abs),
@@ -4237,7 +4282,7 @@ static const struct opcode opcode_table[/* 0x38 - 0x3F */ F6ALU (NoWrite, em_cmp),N ,N ,/* 0x40 - 0x4F */ -X8 (F (DstReg, em_inc)),X8 (F (DstReg, em_dec)), +X8 (I (DstReg, em_inc)),X8 (I (DstReg, em_dec)),/* 0x50 - 0x57 */ X8(I(SrcReg | Stack, em_push)),/* 0x58 - 0x5F */
Peter Zijlstra
maintainerPATCH v3 11/16 x86,hyperv: Clean up hv_do_hypercall()
In reply to Peter ZijlstraWhat used to be a simple few instructions has turned into a giant mess (for x86_64). Not only does it use static_branch wrong, it mixes it with dynamic branches for no apparent reason.
Notably it uses static_branch through an out-of-line function call, which completely defeats the purpose, since instead of a simple JMP/NOP site, you get a CALL+RET+TEST+Jcc sequence in return, which is absolutely idiotic.
Add to that a dynamic test of hyperv_paravisor_present, something which is set once and never changed.
Replace all this idiocy with a single direct function call to the right hypercall variant.
--- arch/x86/hyperv/hv_init.c | 20 +++++ arch/x86/hyperv/ivm.c | 15 ++++ arch/x86/include/asm/mshyperv.h | 137 +++++++++++----------------------------- arch/x86/kernel/cpu/mshyperv.c | 19 +++-- 4 files changed, 89 insertions(+), 102 deletions(-)
arch/x86/hyperv/hv_init.c +20 -0
@@ -36,7 +36,27 @@#include <linux/highmem.h> void *hv_hypercall_pg; + +#ifdef CONFIG_X86_64 +u64 hv_std_hypercall (u64 control,u64 param1,u64 param2) +{ +u64 hv_status; + +if (!hv_hypercall_pg) +return U64_MAX ; + + register u64 __r8 asm("r8") = param2; + asm volatile (CALL_NOSPEC + : "=a" (hv_status), ASM_CALL_CONSTRAINT, + "+c" (control), "+d" (param1), "+r" (__r8) + : THUNK_TARGET(hv_hypercall_pg) + : "cc", "memory", "r9", "r10", "r11"); + +return hv_status; +} +#else EXPORT_SYMBOL_GPL (hv_hypercall_pg); +#endif union hv_ghcb *__percpu *hv_ghcb_pg;
arch/x86/hyperv/ivm.c +15 -0
@@ -377,9 +377,23 @@ int hv_snp_boot_ap(u32 cpu, unsigned lonreturn ret; } +u64 hv_snp_hypercall (u64 control,u64 param1,u64 param2) +{ +u64 hv_status; + + register u64 __r8 asm("r8") = param2; + asmvolatile ("vmmcall" + : "=a" (hv_status), ASM_CALL_CONSTRAINT, +"+c" (control),"+d" (param1),"+r" (__r8) + : :"cc" ,"memory" ,"r9" ,"r10" ,"r11" ); + +return hv_status; +} +#else static inline void hv_ghcb_msr_write (u64 msr,u64 value) {}static inline void hv_ghcb_msr_read (u64 msr,u64 *value) {} +u64 hv_snp_hypercall (u64 control,u64 param1,u64 param2) {return U64_MAX ; }#endif /* CONFIG_AMD_MEM_ENCRYPT */ #ifdef CONFIG_INTEL_TDX_GUEST
@@ -429,6 +443,7 @@ u64 hv_tdx_hypercall(u64 control, u64 pa#else static inline void hv_tdx_msr_write (u64 msr,u64 value) {}static inline void hv_tdx_msr_read (u64 msr,u64 *value) {} +u64 hv_tdx_hypercall (u64 control,u64 param1,u64 param2) {return U64_MAX ; }#endif /* CONFIG_INTEL_TDX_GUEST */ #if defined(CONFIG_AMD_MEM_ENCRYPT ) || defined(CONFIG_INTEL_TDX_GUEST )
arch/x86/include/asm/mshyperv.h +41 -96
194 lines not on this page · raw message
arch/x86/kernel/cpu/mshyperv.c +13 -6
@@ -38,10 +38,6 @@bool hv_nested;struct ms_hyperv_info ms_hyperv; -/* Used in modules via hv_do_hypercall(): see arch/x86/include/asm/mshyperv.h */ -bool hyperv_paravisor_present__ro_after_init ; -EXPORT_SYMBOL_GPL (hyperv_paravisor_present); -#if IS_ENABLED (CONFIG_HYPERV )static inline unsigned int hv_get_nested_msr (unsigned int reg) {
@@ -288,8 +284,18 @@ static void __init x86_setup_ops_for_tsc old_restore_sched_clock_state = x86_platform.restore_sched_clock_state; x86_platform.restore_sched_clock_state = hv_restore_sched_clock_state; } + +#ifdef CONFIG_X86_64 +DEFINE_STATIC_CALL (hv_hypercall, hv_std_hypercall); +EXPORT_STATIC_CALL_TRAMP_GPL (hv_hypercall); +#define hypercall_update (hc) static_call_update(hv_hypercall, hc) +#endif #endif /* CONFIG_HYPERV */ +#ifndef hypercall_update +#define hypercall_update (hc) (void)hc +#endif +static uint32_t __init ms_hyperv_platform (void ) {u32 eax;
@@ -484,14 +490,14 @@ static void __init ms_hyperv_init_platfo ms_hyperv.shared_gpa_boundary =BIT_ULL (ms_hyperv.shared_gpa_boundary_bits); - hyperv_paravisor_present = !!ms_hyperv.paravisor_present; -pr_info ("Hyper-V: Isolation Config: Group A 0x%x, Group B 0x%x\n" , ms_hyperv.isolation_config_a, ms_hyperv.isolation_config_b);if (hv_get_isolation_type () ==HV_ISOLATION_TYPE_SNP ) {static_branch_enable (&isolation_type_snp); +if (!ms_hyperv.paravisor_present) +hypercall_update (hv_snp_hypercall); }else if (hv_get_isolation_type () ==HV_ISOLATION_TYPE_TDX ) {static_branch_enable (&isolation_type_tdx);
@@ -499,6 +505,7 @@ static void __init ms_hyperv_init_platfo ms_hyperv.hints &= ~HV_X64_APIC_ACCESS_RECOMMENDED ;if (!ms_hyperv.paravisor_present) { +hypercall_update (hv_tdx_hypercall);/* * Mark the Hyper-V TSC page feature as disabled * in a TDX VM without paravisor so that the
Peter Zijlstra
maintainerRe: [PATCH v3 16/16] objtool: Validate kCFI calls
In reply to Peter Zijlstraarch/x86/platform/efi/efi_stub_64.S line 20
Quoted 13 lines · Peter Zijlstra
arch/x86/platform/efi/efi_stub_64.S +4
-0@@ -11,6 +11,10 @@ #include <asm/nospec-branch.h> SYM_FUNC_START(__efi_call) + /* + * The EFI code doesn't have any CFI, annotate away the CFI violation. + */ + ANNOTATE_NOCFI_SYM pushq %rbp movq %rsp, %rbp and $~0xf, %rsp
FWIW, we should probably do something like this as well.
---
arch/x86/kernel/cpu/common.c +7 -0
@@ -562,6 +562,13 @@ __noendbr u64 ibt_save(bool disable) {u64 msr =0 ; +/* +* Firmware code will not provide the same level of +* control-flow-integriry. Taint the kernel to let the user know. +*/ +if (disable || (IS_ENABLED (CONFIG_CFI_CLANG ) && cfi_mode !=CFI_OFF )) +add_taint (TAINT_CFI ,LOCKDEP_STILL_OK ); +if (cpu_feature_enabled (X86_FEATURE_IBT )) {rdmsrq (MSR_IA32_S_CET , msr);if (disable)
include/linux/panic.h +2 -1
@@ -73,7 +73,8 @@ static inline void set_arch_panic_timeou#define TAINT_RANDSTRUCT 17#define TAINT_TEST 18#define TAINT_FWCTL 19 -#define TAINT_FLAGS_COUNT 20 +#define TAINT_CFI 20 +#define TAINT_FLAGS_COUNT 21#define TAINT_FLAGS_MAX ((1UL << TAINT_FLAGS_COUNT) - 1)struct taint_flag {
Peter Zijlstra
maintainerRe: [PATCH v3 16/16] objtool: Validate kCFI calls
Quoted 23 lines · Peter Zijlstra
arch/x86/platform/efi/efi_stub_64.S line 20
Quoted 13 lines · Peter Zijlstra
arch/x86/platform/efi/efi_stub_64.S +4
-0@@ -11,6 +11,10 @@ #include <asm/nospec-branch.h> SYM_FUNC_START(__efi_call) + /* + * The EFI code doesn't have any CFI, annotate away the CFI violation. + */ + ANNOTATE_NOCFI_SYM pushq %rbp movq %rsp, %rbp and $~0xf, %rspFWIW, we should probably do something like this as well.
---
arch/x86/kernel/cpu/common.c __noendbr u64 ibt_save(bool disable) · line 570
Quoted 9 lines
arch/x86/kernel/cpu/common.c +6
-0@@ -562,6 +562,13 @@ __noendbr u64 ibt_save(bool disable) {u64 msr =0 ; +/* +* Firmware code will not provide the same level of +* control-flow-integriry. Taint the kernel to let the user know. +*/ +if (disable || (IS_ENABLED (CONFIG_CFI_CLANG ) && cfi_mode !=CFI_OFF )) +add_taint (TAINT_CFI ,LOCKDEP_STILL_OK );
Or perhaps:
WARN_TAINT_ONCE(disable || IS_ENABLED(CONFIG_CFI_CLANG) && cfi_mode != CFI_OFF), TAINT_CFI, "Firmware has weaker CFI");
Quoted 16 lines
arch/x86/kernel/cpu/common.c +1
-0+if (cpu_feature_enabled (X86_FEATURE_IBT )) {rdmsrq (MSR_IA32_S_CET , msr);if (disable)include/linux/panic.h +2
-1@@ -73,7 +73,8 @@ static inline void set_arch_panic_timeou#define TAINT_RANDSTRUCT 17#define TAINT_TEST 18#define TAINT_FWCTL 19 -#define TAINT_FLAGS_COUNT 20 +#define TAINT_CFI 20 +#define TAINT_FLAGS_COUNT 21#define TAINT_FLAGS_MAX ((1UL << TAINT_FLAGS_COUNT) - 1)struct taint_flag {
Re: [PATCH v3 16/16] objtool: Validate kCFI calls
In reply to Peter ZijlstraQuoted 3 lines · Peter Zijlstra
Apparently some Rust 'core' code violates this and explodes when ran with FineIBT.
I think this was fixed in Rust 1.88 (latest version), right? Or is there an issue still?
5595c31c3709 ("x86/Kconfig: make CFI_AUTO_DEFAULT depend on !RUSTor Rust >= 1.88")
Quoted 3 lines
- runtime EFI is especially henous because it also needs to disable IBT. Basically calling unknown code without CFI protection at runtime is a massice security issue.
heinous
massive
Cheers,
Miguel
Re: [PATCH v3 11/16] x86,hyperv: Clean up hv_do_hypercall()
In reply to Peter ZijlstraQuoted 12 lines · Peter Zijlstra
What used to be a simple few instructions has turned into a giant mess (for
x86_64). Not only does it usestatic_branchwrong, it mixes it with dynamic branches for no apparent reason.Notably it uses
static_branchthrough an out-of-line function call, which completely defeats the purpose, since instead of a simple JMP/NOP site, you get a CALL+RET+TEST+Jcc sequence in return, which is absolutely idiotic.Add to that a dynamic test of
hyperv_paravisor_present, something which is set once and never changed.
Quoted 6 lines
Replace all this idiocy with a single direct function call to the right hypercall variant.
Signed-off-by Peter Zijlstra (Intel) <[email protected]>Tested-by Michael Kelley <[email protected]>Reviewed-by Michael Kelley <[email protected]>
Re: [PATCH v3 12/16] x86_64,hyperv: Use direct call to hypercall-page
In reply to Peter ZijlstraQuoted 6 lines · Peter Zijlstra
Instead of using an indirect call to the hypercall page, use a direct call instead. This avoids all CFI problems, including the one where the hypercall page doesn't have IBT on.
Signed-off-by Peter Zijlstra (Intel) <[email protected]>Reviewed-by Michael Kelley <[email protected]>
Peter Zijlstra
maintainerRe: [PATCH v3 16/16] objtool: Validate kCFI calls
In reply to Miguel OjedaQuoted 10 lines · Miguel Ojeda
Quoted 3 lines · Peter Zijlstra
Apparently some Rust 'core' code violates this and explodes when ran with FineIBT.
I think this was fixed in Rust 1.88 (latest version), right? Or is there an issue still?
5595c31c3709 ("x86/Kconfig: make CFI_AUTO_DEFAULT depend on !RUSTor Rust >= 1.88")
Oh yeah, it got fixed. Clearly I failed to update the Changelog.
Quoted 6 lines
Quoted 3 lines
- runtime EFI is especially henous because it also needs to disable IBT. Basically calling unknown code without CFI protection at runtime is a massice security issue.
heinous
massive
Typing hard; Thanks!
RE: [PATCH v3 11/16] x86,hyperv: Clean up hv_do_hypercall()
In reply to Peter ZijlstraFrom: Peter Zijlstra <[email protected]> Sent: Monday, July 14, 2025 3:20 AM
Quoted 1 line
Nit: For the patch "Subject", the prefix should be "x86/hyperv:" for consistency with past practice. There's definitely been some variation over time, but "x86/hyperv:" has been the most consistently used, and that's what I try to remind people to use.
Michael
Quoted 349 lines
What used to be a simple few instructions has turned into a giant mess (for
x86_64). Not only does it usestatic_branchwrong, it mixes it with dynamic branches for no apparent reason.Notably it uses
static_branchthrough an out-of-line function call, which completely defeats the purpose, since instead of a simple JMP/NOP site, you get a CALL+RET+TEST+Jcc sequence in return, which is absolutely idiotic.Add to that a dynamic test of
hyperv_paravisor_present, something which is set once and never changed.Replace all this idiocy with a single direct function call to the right hypercall variant.
Signed-off-by Peter Zijlstra (Intel) <[email protected]>Tested-by Michael Kelley <[email protected]>Reviewed-by Michael Kelley <[email protected]>--- arch/x86/hyperv/hv_init.c | 20 +++++ arch/x86/hyperv/ivm.c | 15 ++++ arch/x86/include/asm/mshyperv.h | 137 +++++++++++----------------------------- arch/x86/kernel/cpu/mshyperv.c | 19 +++-- 4 files changed, 89 insertions(+), 102 deletions(-)arch/x86/hyperv/hv_init.c +20
-0@@ -36,7 +36,27 @@#include <linux/highmem.h> void *hv_hypercall_pg; + +#ifdef CONFIG_X86_64 +u64 hv_std_hypercall (u64 control,u64 param1,u64 param2) +{ +u64 hv_status; + +if (!hv_hypercall_pg) +return U64_MAX ; + + register u64 __r8 asm("r8") = param2; + asm volatile (CALL_NOSPEC + : "=a" (hv_status), ASM_CALL_CONSTRAINT, + "+c" (control), "+d" (param1), "+r" (__r8) + : THUNK_TARGET(hv_hypercall_pg) + : "cc", "memory", "r9", "r10", "r11"); + +return hv_status; +} +#else EXPORT_SYMBOL_GPL (hv_hypercall_pg); +#endif union hv_ghcb *__percpu *hv_ghcb_pg;arch/x86/hyperv/ivm.c +15
-0@@ -377,9 +377,23 @@ int hv_snp_boot_ap(u32 cpu, unsigned lonreturn ret; } +u64 hv_snp_hypercall (u64 control,u64 param1,u64 param2) +{ +u64 hv_status; + + register u64 __r8 asm("r8") = param2; + asmvolatile ("vmmcall" + : "=a" (hv_status), ASM_CALL_CONSTRAINT, +"+c" (control),"+d" (param1),"+r" (__r8) + : :"cc" ,"memory" ,"r9" ,"r10" ,"r11" ); + +return hv_status; +} +#else static inline void hv_ghcb_msr_write (u64 msr,u64 value) {}static inline void hv_ghcb_msr_read (u64 msr,u64 *value) {} +u64 hv_snp_hypercall (u64 control,u64 param1,u64 param2) {return U64_MAX ; }#endif /* CONFIG_AMD_MEM_ENCRYPT */ #ifdef CONFIG_INTEL_TDX_GUEST @@ -429,6 +443,7 @@ u64 hv_tdx_hypercall(u64 control, u64 pa#else static inline void hv_tdx_msr_write (u64 msr,u64 value) {}static inline void hv_tdx_msr_read (u64 msr,u64 *value) {} +u64 hv_tdx_hypercall (u64 control,u64 param1,u64 param2) {return U64_MAX ; }#endif /* CONFIG_INTEL_TDX_GUEST */ #if defined(CONFIG_AMD_MEM_ENCRYPT ) || defined(CONFIG_INTEL_TDX_GUEST )arch/x86/include/asm/mshyperv.h +41
-96@@ -6,6 +6,7 @@#include <linux/nmi.h> #include <linux/msi.h> #include <linux/io.h> +#include <linux/static_call.h> #include <asm/nospec-branch.h> #include <asm/paravirt.h> #include <asm/msr.h> @@ -39,16 +40,21 @@ static inline unsigned char hv_get_nmi_rreturn 0 ; } -#if IS_ENABLED (CONFIG_HYPERV ) -extern bool hyperv_paravisor_present; +extern u64 hv_tdx_hypercall (u64 control,u64 param1,u64 param2); +extern u64 hv_snp_hypercall (u64 control,u64 param1,u64 param2); +extern u64 hv_std_hypercall (u64 control,u64 param1,u64 param2); +#if IS_ENABLED (CONFIG_HYPERV )extern void *hv_hypercall_pg;extern union hv_ghcb *__percpu *hv_ghcb_pg;bool hv_isolation_type_snp (void );bool hv_isolation_type_tdx (void ); -u64 hv_tdx_hypercall (u64 control,u64 param1,u64 param2); + +#ifdef CONFIG_X86_64 +DECLARE_STATIC_CALL (hv_hypercall, hv_std_hypercall); +#endif /* * DEFAULT INIT GPAT and SEGMENT LIMIT value in struct VMSA @@ -65,37 +71,15 @@ static inline u64 hv_do_hypercall(u64 co {u64 input_address = input ?virt_to_phys (input) :0 ;u64 output_address = output ?virt_to_phys (output) :0 ; -u64 hv_status;#ifdef CONFIG_X86_64 -if (hv_isolation_type_tdx () && !hyperv_paravisor_present) -return hv_tdx_hypercall (control, input_address, output_address); - -if (hv_isolation_type_snp () && !hyperv_paravisor_present) { - __asm__ __volatile__("mov %[output_address], %%r8\n" -"vmmcall" - : "=a" (hv_status), ASM_CALL_CONSTRAINT, -"+c" (control),"+d" (input_address) - : [output_address]"r" (output_address) - :"cc" ,"memory" ,"r8" ,"r9" ,"r10" ,"r11" ); -return hv_status; - } - -if (!hv_hypercall_pg) -return U64_MAX ; - - __asm__ __volatile__("mov %[output_address], %%r8\n" -CALL_NOSPEC - : "=a" (hv_status), ASM_CALL_CONSTRAINT, -"+c" (control),"+d" (input_address) - : [output_address] "r" (output_address), - THUNK_TARGET(hv_hypercall_pg) - :"cc" ,"memory" ,"r8" ,"r9" ,"r10" ,"r11" ); +return static_call_mod (hv_hypercall)(control, input_address, output_address);#else u32 input_address_hi =upper_32_bits (input_address);u32 input_address_lo =lower_32_bits (input_address);u32 output_address_hi =upper_32_bits (output_address);u32 output_address_lo =lower_32_bits (output_address); +u64 hv_status;if (!hv_hypercall_pg)return U64_MAX ;@@ -108,8 +92,8 @@ static inline u64 hv_do_hypercall(u64 co "D"(output_address_hi), "S"(output_address_lo), THUNK_TARGET(hv_hypercall_pg) : "cc", "memory"); -#endif /* !x86_64 */ return hv_status; +#endif /* !x86_64 */ }/* Hypercall to the L0 hypervisor */ @@ -121,41 +105,23 @@ static inline u64 hv_do_nested_hypercall/* Fast hypercall with 8 bytes of input and no output */ static inline u64 _hv_do_fast_hypercall8 (u64 control,u64 input1) { -u64 hv_status; -#ifdef CONFIG_X86_64 -if (hv_isolation_type_tdx () && !hyperv_paravisor_present) -return hv_tdx_hypercall (control, input1,0 ); - -if (hv_isolation_type_snp () && !hyperv_paravisor_present) { - __asm__ __volatile__( -"vmmcall" - : "=a" (hv_status), ASM_CALL_CONSTRAINT, -"+c" (control),"+d" (input1) - ::"cc" ,"r8" ,"r9" ,"r10" ,"r11" ); - }else { - __asm__ __volatile__(CALL_NOSPEC - : "=a" (hv_status), ASM_CALL_CONSTRAINT, - "+c" (control), "+d" (input1) - : THUNK_TARGET(hv_hypercall_pg) - : "cc", "r8", "r9", "r10", "r11"); - } +return static_call_mod (hv_hypercall)(control, input1,0 );#else - { -u32 input1_hi =upper_32_bits (input1); -u32 input1_lo =lower_32_bits (input1); - - __asm__ __volatile__ (CALL_NOSPEC - : "=A"(hv_status), - "+c"(input1_lo), - ASM_CALL_CONSTRAINT - : "A" (control), - "b" (input1_hi), - THUNK_TARGET(hv_hypercall_pg) - : "cc", "edi", "esi"); - } -#endif +u32 input1_hi =upper_32_bits (input1); +u32 input1_lo =lower_32_bits (input1); +u64 hv_status; + + __asm__ __volatile__ (CALL_NOSPEC + : "=A"(hv_status), + "+c"(input1_lo), + ASM_CALL_CONSTRAINT + : "A" (control), + "b" (input1_hi), + THUNK_TARGET(hv_hypercall_pg) + : "cc", "edi", "esi");return hv_status; +#endif }static inline u64 hv_do_fast_hypercall8 (u16 code,u64 input1)@@ -175,45 +141,24 @@ static inline u64 hv_do_fast_nested_hype/* Fast hypercall with 16 bytes of input */ static inline u64 _hv_do_fast_hypercall16 (u64 control,u64 input1,u64 input2) { -u64 hv_status; -#ifdef CONFIG_X86_64 -if (hv_isolation_type_tdx () && !hyperv_paravisor_present) -return hv_tdx_hypercall (control, input1, input2); - -if (hv_isolation_type_snp () && !hyperv_paravisor_present) { - __asm__ __volatile__("mov %[input2], %%r8\n" -"vmmcall" - : "=a" (hv_status), ASM_CALL_CONSTRAINT, -"+c" (control),"+d" (input1) - : [input2]"r" (input2) - :"cc" ,"r8" ,"r9" ,"r10" ,"r11" ); - }else { - __asm__ __volatile__("mov %[input2], %%r8\n" -CALL_NOSPEC - : "=a" (hv_status), ASM_CALL_CONSTRAINT, -"+c" (control),"+d" (input1) - : [input2] "r" (input2), - THUNK_TARGET(hv_hypercall_pg) - :"cc" ,"r8" ,"r9" ,"r10" ,"r11" ); - } +return static_call_mod (hv_hypercall)(control, input1, input2);#else - { -u32 input1_hi =upper_32_bits (input1); -u32 input1_lo =lower_32_bits (input1); -u32 input2_hi =upper_32_bits (input2); -u32 input2_lo =lower_32_bits (input2); - - __asm__ __volatile__ (CALL_NOSPEC - : "=A"(hv_status), - "+c"(input1_lo), ASM_CALL_CONSTRAINT - : "A" (control), "b" (input1_hi), - "D"(input2_hi), "S"(input2_lo), - THUNK_TARGET(hv_hypercall_pg) - : "cc"); - } -#endif +u32 input1_hi =upper_32_bits (input1); +u32 input1_lo =lower_32_bits (input1); +u32 input2_hi =upper_32_bits (input2); +u32 input2_lo =lower_32_bits (input2); +u64 hv_status; + + __asm__ __volatile__ (CALL_NOSPEC + : "=A"(hv_status), + "+c"(input1_lo), ASM_CALL_CONSTRAINT + : "A" (control), "b" (input1_hi), + "D"(input2_hi), "S"(input2_lo), + THUNK_TARGET(hv_hypercall_pg) + : "cc");return hv_status; +#endif }static inline u64 hv_do_fast_hypercall16 (u16 code,u64 input1,u64 input2)arch/x86/kernel/cpu/mshyperv.c +13
-6@@ -38,10 +38,6 @@bool hv_nested;struct ms_hyperv_info ms_hyperv; -/* Used in modules via hv_do_hypercall(): see arch/x86/include/asm/mshyperv.h */ -bool hyperv_paravisor_present__ro_after_init ; -EXPORT_SYMBOL_GPL (hyperv_paravisor_present); -#if IS_ENABLED (CONFIG_HYPERV )static inline unsigned int hv_get_nested_msr (unsigned int reg) {@@ -288,8 +284,18 @@ static void __init x86_setup_ops_for_tsc old_restore_sched_clock_state = x86_platform.restore_sched_clock_state; x86_platform.restore_sched_clock_state = hv_restore_sched_clock_state; } + +#ifdef CONFIG_X86_64 +DEFINE_STATIC_CALL (hv_hypercall, hv_std_hypercall); +EXPORT_STATIC_CALL_TRAMP_GPL (hv_hypercall); +#define hypercall_update (hc) static_call_update(hv_hypercall, hc) +#endif #endif /* CONFIG_HYPERV */ +#ifndef hypercall_update +#define hypercall_update (hc) (void)hc +#endif +static uint32_t __init ms_hyperv_platform (void ) {u32 eax;@@ -484,14 +490,14 @@ static void __init ms_hyperv_init_platfo ms_hyperv.shared_gpa_boundary =BIT_ULL (ms_hyperv.shared_gpa_boundary_bits); - hyperv_paravisor_present = !!ms_hyperv.paravisor_present; -pr_info ("Hyper-V: Isolation Config: Group A 0x%x, Group B 0x%x\n" , ms_hyperv.isolation_config_a, ms_hyperv.isolation_config_b);if (hv_get_isolation_type () ==HV_ISOLATION_TYPE_SNP ) {static_branch_enable (&isolation_type_snp); +if (!ms_hyperv.paravisor_present) +hypercall_update (hv_snp_hypercall); }else if (hv_get_isolation_type () ==HV_ISOLATION_TYPE_TDX ) {static_branch_enable (&isolation_type_tdx);@@ -499,6 +505,7 @@ static void __init ms_hyperv_init_platfo ms_hyperv.hints &= ~HV_X64_APIC_ACCESS_RECOMMENDED ;if (!ms_hyperv.paravisor_present) { +hypercall_update (hv_tdx_hypercall);/* * Mark the Hyper-V TSC page feature as disabled * in a TDX VM without paravisor so that the
RE: [PATCH v3 12/16] x86_64,hyperv: Use direct call to hypercall-page
In reply to Peter ZijlstraFrom: Peter Zijlstra <[email protected]> Sent: Monday, July 14, 2025 3:20 AM
Quoted 1 line
Same nit here: Use "x86/hyperv:" as the Subject prefix, for consistency with past practice.
Michael
Quoted 128 lines
Instead of using an indirect call to the hypercall page, use a direct call instead. This avoids all CFI problems, including the one where the hypercall page doesn't have IBT on.
Signed-off-by Peter Zijlstra (Intel) <[email protected]>Reviewed-by Michael Kelley <[email protected]>--- arch/x86/hyperv/hv_init.c | 61 ++++++++++++++++++++++------------------------ 1 file changed, 30 insertions(+), 31 deletions(-)arch/x86/hyperv/hv_init.c +30
-31@@ -17,7 +17,6 @@#include <asm/desc.h> #include <asm/e820/api.h> #include <asm/sev.h> -#include <asm/ibt.h> #include <asm/hypervisor.h> #include <hyperv/hvhdk.h> #include <asm/mshyperv.h> @@ -38,23 +37,41 @@void *hv_hypercall_pg;#ifdef CONFIG_X86_64 +static u64 __hv_hyperfail (u64 control,u64 param1,u64 param2) +{ +return U64_MAX ; +} + +DEFINE_STATIC_CALL (__hv_hypercall, __hv_hyperfail); +u64 hv_std_hypercall (u64 control,u64 param1,u64 param2) {u64 hv_status; -if (!hv_hypercall_pg) -return U64_MAX ; - register u64 __r8 asm("r8") = param2; - asm volatile (CALL_NOSPEC + asm volatile ("call " STATIC_CALL_TRAMP_STR(__hv_hypercall) : "=a" (hv_status), ASM_CALL_CONSTRAINT, "+c" (control), "+d" (param1), "+r" (__r8) - : THUNK_TARGET(hv_hypercall_pg) - : "cc", "memory", "r9", "r10", "r11"); + : : "cc", "memory", "r9", "r10", "r11");return hv_status; } + +typedef u64 (*hv_hypercall_f )(u64 control,u64 param1,u64 param2); + +static inline void hv_set_hypercall_pg (void *ptr) +{ + hv_hypercall_pg = ptr; + +if (!ptr) + ptr = &__hv_hyperfail; +static_call_update (__hv_hypercall, (hv_hypercall_f )ptr); +}#else +static inline void hv_set_hypercall_pg (void *ptr) +{ + hv_hypercall_pg = ptr; +}EXPORT_SYMBOL_GPL (hv_hypercall_pg);#endif @@ -349,7 +366,7 @@ static int hv_suspend(void)* pointer is restored on resume. */ hv_hypercall_pg_saved = hv_hypercall_pg; - hv_hypercall_pg =NULL ; +hv_set_hypercall_pg (NULL );/* Disable the hypercall page in the hypervisor */ rdmsrq (HV_X64_MSR_HYPERCALL , hypercall_msr.as_uint64);@@ -375,7 +392,7 @@ static void hv_resume(void)vmalloc_to_pfn (hv_hypercall_pg_saved);wrmsrq (HV_X64_MSR_HYPERCALL , hypercall_msr.as_uint64); - hv_hypercall_pg = hv_hypercall_pg_saved; +hv_set_hypercall_pg (hv_hypercall_pg_saved); hv_hypercall_pg_saved =NULL ;/* @@ -529,8 +546,8 @@ void __init hyperv_init(void)if (hv_isolation_type_tdx () && !ms_hyperv.paravisor_present) goto skip_hypercall_pg_init; - hv_hypercall_pg =__vmalloc_node_range (PAGE_SIZE ,1 ,VMALLOC_START , -VMALLOC_END ,GFP_KERNEL ,PAGE_KERNEL_ROX , + hv_hypercall_pg =__vmalloc_node_range (PAGE_SIZE ,1 ,MODULES_VADDR , +MODULES_END ,GFP_KERNEL ,PAGE_KERNEL_ROX ,VM_FLUSH_RESET_PERMS ,NUMA_NO_NODE ,__builtin_return_address (0 ));if (hv_hypercall_pg ==NULL )@@ -568,27 +585,9 @@ void __init hyperv_init(void)wrmsrq (HV_X64_MSR_HYPERCALL , hypercall_msr.as_uint64); } -skip_hypercall_pg_init: -/* -* Some versions of Hyper-V that provide IBT in guest VMs have a bug -* in that there's no ENDBR64 instruction at the entry to the -* hypercall page. Because hypercalls are invoked via an indirect call -* to the hypercall page, all hypercall attempts fail when IBT is -* enabled, and Linux panics. For such buggy versions, disable IBT. -* -* Fixed versions of Hyper-V always provide ENDBR64 on the hypercall -* page, so if future Linux kernel versions enable IBT for 32-bit -* builds, additional hypercall page hackery will be required here -* to provide an ENDBR32. - */ -#ifdef CONFIG_X86_KERNEL_IBT -if (cpu_feature_enabled (X86_FEATURE_IBT ) && - *(u32 *)hv_hypercall_pg !=gen_endbr ()) { -setup_clear_cpu_cap (X86_FEATURE_IBT ); -pr_warn ("Disabling IBT because of Hyper-V bug\n" ); - } -#endif +hv_set_hypercall_pg (hv_hypercall_pg); +skip_hypercall_pg_init:/* * hyperv_init() is called before LAPIC is initialized: see * apic_intr_mode_init() -> x86_platform.apic_post_init() and
Re: [PATCH v3 16/16] objtool: Validate kCFI calls
In reply to Peter ZijlstraQuoted 8 lines · Peter Zijlstra
Validate that all indirect calls adhere to kCFI rules. Notably doing nocfi indirect call to a cfi function is broken.
Apparently some Rust 'core' code violates this and explodes when ran with FineIBT.
All the
ANNOTATE_NOCFI_SYMsites are prime targets for attackers.
Quoted 7 lines
- runtime EFI is especially henous because it also needs to disable IBT. Basically calling unknown code without CFI protection at runtime is a massice security issue.
- Kexec image handover; if you can exploit this, you get to keep it :-)
Signed-off-by Peter Zijlstra (Intel) <[email protected]>
Signature · 1 line
Josh
Sean Christopherson
maintainerRe: [PATCH v3 07/16] x86/kvm/emulate: Introduce EM_ASM_1SRC2
In reply to Peter ZijlstraFor all of the KVM patches, please use
KVM: x86:
"x86/kvm" is used for guest-side code, and while I hope no one will conflate this with guest code, the consistency is helpful.
Quoted 19 lines · Peter Zijlstra
Replace the FASTOP1SRC2*() instructions.
Signed-off-by Peter Zijlstra (Intel) <[email protected]>--- arch/x86/kvm/emulate.c | 34 ++++++++++++++++++++++++++-------- 1 file changed, 26 insertions(+), 8 deletions(-)arch/x86/kvm/emulate.c +6
-0@@ -317,6 +317,24 @@ static int em_##op(struct x86_emulate_ct ON64(case 8: __EM_ASM_1(op##q, rax); break;) \ EM_ASM_END +/* 1-operand, using "c" (src2) */ +#define EM_ASM_1SRC2 (op, name) \ + EM_ASM_START(name) \ + case 1: __EM_ASM_1(op##b, cl); break; \ + case 2: __EM_ASM_1(op##w, cx); break; \ + case 4: __EM_ASM_1(op##l, ecx); break; \
arch/x86/kvm/emulate.c static int em_##op(struct x86_emulate_ct · line 335
Quoted 10 lines
arch/x86/kvm/emulate.c +10
-0+ ON64(case 8: __EM_ASM_1(op##q, rcx); break;) \ + EM_ASM_END + +/* 1-operand, using "c" (src2) with exception */ +#define EM_ASM_1SRC2EX (op, name) \ + EM_ASM_START(name) \ + case 1: __EM_ASM_1_EX(op##b, cl); break; \ + case 2: __EM_ASM_1_EX(op##w, cx); break; \ + case 4: __EM_ASM_1_EX(op##l, ecx); break; \ + ON64(case 8: __EM_ASM_1(op##q, rcx); break;) \
This needs to be __EM_ASM_1_EX(). Luckily, KVM-Unit-Tests actually has testcase for divq (somewhere in the morass of testcases). I also now have an extension to the fastops selftest to explicitly test all four flavors of div-by-zero; I'll get it posted tomorrow.
(also, don't also me how long it took me to spot the copy+paste typo; I was full
on debugging the exception fixup code before I realized my local diff looked "odd", *sigh*)
Quoted 19 lines · Peter Zijlstra
Hi!
On kCFI (
CONFIG_CFI_CLANG=y) builds all indirect calls should have the CFI check on (with very few exceptions). Not having the CFI checks undermines the protection provided by CFI and will make these sites candidates for people wanting to steal your cookies.Specifically the ABI changes are so that doing indirect calls without the CFI magic, to a CFI adorned function is not compatible (although it happens to work for some setups, it very much does not for FineIBT).
Rust people tripped over this the other day, since their 'core' happened to have some
no_sanitize(kcfi)bits in, which promptly exploded when ran with FineIBT on.Since this is very much not a supported model -- on purpose, have objtool detect and warn about such constructs.
Quoted 7 lines
This effort [1] found all existing [2] non-cfi indirect calls in the kernel.
Notably the KVM fastop emulation stuff -- which is completely rewritten -- the generated code doesn't look horrific, but is slightly more verbose. I'm running on the assumption that instruction emulation is not super performance critical these days of zero VM-exit VMs etc. Paolo noted that pre-Westmere (2010) cares about this.
Yeah, I'm confident the fastop stuff isn't performance critical. I'm skeptical that fastops were _ever_ about raw performance.
Running with EPT disabled to force emulation of Big RM, with OVMF and a 64-bit Linux guest, I get literally zero hits on fastop(). With SeaBIOS and a 32-bit Linux guest, booting a 24 vCPU VM hits <40 fastops.
Maybe there are some super legacy workloads that still heavily utilize Big RM, but if they exist, I've no idea what they are, and AFAICT that was never the motivation.
As highlighted in the original cover letter[*], fastops reduced the code footprint of kvm/emulate.o by ~2500 bytes. And as called out by commit e28bbd44dad1 ("KVM:
x86 emulator: framework for streamlining arithmetic opcodes"), executing a proxy for the to-be-emulated instruction is all about functional correctness, e.g. to ensure arithmetic RFLAGS match exactly. Nothing suggests that performance was ever a motivating factor.
I strongly suspect that the "fastop" name was a fairly arbitrary choice, and the framework needed to be called _something_. And then everyone since has assumed that the motivation for fastops was to go fast, when in fact that was just a happy side effect of the implementation.
https://lore.kernel.org/all/[email protected]
So, with the _EX goof fixed, and "KVM: x86:" for all the relevant KVM patches:
P.S. Thanks a ton for cleaning this up!
Sean Christopherson
maintainerRe: [PATCH v3 16/16] objtool: Validate kCFI calls
In reply to Peter Zijlstraarch/x86/kvm/vmx/vmenter.S SYM_FUNC_END(vmread_error_trampoline) · line 370
Quoted 13 lines · Peter Zijlstra
arch/x86/kvm/vmx/vmenter.S +4
-0@@ -361,6 +361,10 @@ SYM_FUNC_END(vmread_error_trampoline) .section .text, "ax" +#ifndef CONFIG_X86_FRED + SYM_FUNC_START(vmx_do_interrupt_irqoff) VMX_DO_EVENT_IRQOFF CALL_NOSPEC _ASM_ARG1 SYM_FUNC_END(vmx_do_interrupt_irqoff) + +#endif
This can go in the previous patch, "x86/fred: KVM: VMX: Always use FRED for IRQs when CONFIG_X86_FRED=y".
Xin Li
maintainerRe: [PATCH v3 16/16] objtool: Validate kCFI calls
Quoted 7 lines · Sean Christopherson
Quoted 6 lines · Peter Zijlstra
arch/x86/kvm/vmx/vmenter.S +0
-0@@ -361,6 +361,10 @@ SYM_FUNC_END(vmread_error_trampoline) .section .text, "ax"
Quoted 11 lines
arch/x86/kvm/vmx/vmenter.S SYM_FUNC_END(vmread_error_trampoline) · line 370
Quoted 7 lines
arch/x86/kvm/vmx/vmenter.S +4
-0+#ifndef CONFIG_X86_FRED + SYM_FUNC_START(vmx_do_interrupt_irqoff) VMX_DO_EVENT_IRQOFF CALL_NOSPEC _ASM_ARG1 SYM_FUNC_END(vmx_do_interrupt_irqoff) + +#endifThis can go in the previous patch, "x86/fred: KVM: VMX: Always use FRED for IRQs when
CONFIG_X86_FRED=y".
I'm going to test patch 13~15, plus this change in patch 16.
BTW, there is a declaration for vmx_do_interrupt_irqoff() in arch/x86/kvm/vmx/vmx.c, so we'd better also do:
arch/x86/kvm/vmx/vmx.c +2 -0
@@ -6945,7 +6945,9 @@ void vmx_load_eoi_exitmap(struct kvm_vcpu *vcpu, u64 *eoi_exit_bitmap)vmcs_write64 (EOI_EXIT_BITMAP3 , eoi_exit_bitmap[3 ]); } +#ifndef CONFIG_X86_FRED void vmx_do_interrupt_irqoff (unsigned long entry); +#endif void vmx_do_nmi_irqoff (void );static void handle_nm_fault_irqoff (struct kvm_vcpu *vcpu)
Sean Christopherson
maintainerRe: [PATCH v3 16/16] objtool: Validate kCFI calls
Quoted 26 lines · Xin Li
Quoted 16 lines · Sean Christopherson
arch/x86/kvm/vmx/vmenter.S SYM_FUNC_END(vmread_error_trampoline) · line 368
Quoted 11 lines · Peter Zijlstra
arch/x86/kvm/vmx/vmenter.S +4
-0@@ -361,6 +361,10 @@ SYM_FUNC_END(vmread_error_trampoline) .section .text, "ax" +#ifndef CONFIG_X86_FRED + SYM_FUNC_START(vmx_do_interrupt_irqoff) VMX_DO_EVENT_IRQOFF CALL_NOSPEC _ASM_ARG1 SYM_FUNC_END(vmx_do_interrupt_irqoff) + +#endifThis can go in the previous patch, "x86/fred: KVM: VMX: Always use FRED for IRQs when
CONFIG_X86_FRED=y".I'm going to test patch 13~15, plus this change in patch 16.
BTW, there is a declaration for
vmx_do_interrupt_irqoff()inarch/x86/kvm/vmx/vmx.c, so we'd better also do:
arch/x86/kvm/vmx/vmx.c void vmx_load_eoi_exitmap(struct kvm_vcpu *vcpu, u64 · line 6951
Quoted 7 lines
arch/x86/kvm/vmx/vmx.c +2
-0@@ -6945,7 +6945,9 @@ void vmx_load_eoi_exitmap(struct kvm_vcpu *vcpu, u64 *eoi_exit_bitmap)vmcs_write64 (EOI_EXIT_BITMAP3 , eoi_exit_bitmap[3 ]); } +#ifndef CONFIG_X86_FRED void vmx_do_interrupt_irqoff (unsigned long entry); +#endif
No, we want to keep the declaration. Unconditionally decaring the symbol allows KVM to use IS_ENABLED():
if (IS_ENABLED(CONFIG_X86_FRED))
fred_entry_from_kvm(EVENT_TYPE_EXTINT, vector);
Hiding the declaration would require that to be a "proper" #ifdef, which would be a net negative for readability. The extra declaration won't hurt anything for CONFIG_X86_FRED=n, as "bad" usage will still fail at link time.
Quoted 3 lines
arch/x86/kvm/vmx/vmx.c +0
-0void vmx_do_nmi_irqoff (void );static void
handle_nm_fault_irqoff(struct kvm_vcpu *vcpu)
Xin Li
maintainerRe: [PATCH v3 16/16] objtool: Validate kCFI calls
Quoted 17 lines · Sean Christopherson
Quoted 7 lines
BTW, there is a declaration for
vmx_do_interrupt_irqoff()inarch/x86/kvm/vmx/vmx.c, so we'd better also do:arch/x86/kvm/vmx/vmx.c void vmx_load_eoi_exitmap(struct kvm_vcpu *vcpu, u64 · line 6951
Quoted 7 lines
arch/x86/kvm/vmx/vmx.c +2
-0@@ -6945,7 +6945,9 @@ void vmx_load_eoi_exitmap(struct kvm_vcpu *vcpu, u64 *eoi_exit_bitmap)vmcs_write64 (EOI_EXIT_BITMAP3 , eoi_exit_bitmap[3 ]); } +#ifndef CONFIG_X86_FRED void vmx_do_interrupt_irqoff (unsigned long entry); +#endif No, we want to keep the declaration. Unconditionally decaring the symbol allows KVM to use
IS_ENABLED():
Quoted 6 lines
if (IS_ENABLED(CONFIG_X86_FRED))
fred_entry_from_kvm(EVENT_TYPE_EXTINT, vector);Hiding the declaration would require that to be a "proper" #ifdef, which would be a net negative for readability. The extra declaration won't hurt anything for
CONFIG_X86_FRED=n, as "bad" usage will still fail at link time.
I did hit a compilation error, so yes, we have to keep it.
Xin Li
maintainerRe: [PATCH v3 14/16] x86/fred: Play nice with invoking asm_fred_entry_from_kvm() on non-FRED hardware
In reply to Peter Zijlstraarch/x86/entry/entry_64_fred.S SYM_FUNC_START(asm_fred_entry_from_kvm) · line 131
Quoted 6 lines · Peter Zijlstra
call
__fred_entry_from_kvm/* Call the C entry point */arch/x86/entry/entry_64_fred.S +2
-3- POP_REGS - ERETS -1: + +1: /*
The symbol "1" is misplaced; it needs to be put after the ERETS instruction.
arch/x86/entry/entry_64_fred.S SYM_FUNC_START(asm_fred_entry_from_kvm) · line 135
Quoted 4 lines
arch/x86/entry/entry_64_fred.S +4
-0+ * When FRED, use ERETS to potentially clear NMIs, otherwise simply + * restore the stack pointer. + */ + ALTERNATIVE "nop; nop; mov %rbp, %rsp", \
Why explicitly add two nops here?
ALTERNATIVE will still pad three-byte nop after the MOV instruction.
Quoted 16 lines
arch/x86/entry/entry_64_fred.S +7
-3+ __stringify(add $C_PTREGS_SIZE, %rsp; ERETS), \ + X86_FEATURE_FRED + /* - * Objtool doesn't understand what ERETS does, this hint tells it that - * yes, we'll reach here and with what stack state. A save/restore pair - * isn't strictly needed, but it's the simplest form. + * Objtool doesn't understand ERETS, and the cfi register state is + * different from initial_func_cfi due to PUSH_REGS. Tell it the state + * is similar to where UNWIND_HINT_SAVE is. */ UNWIND_HINT_RESTORE + pop %rbp RET
Peter Zijlstra
maintainerRe: [PATCH v3 07/16] x86/kvm/emulate: Introduce EM_ASM_1SRC2
In reply to Sean ChristophersonQuoted 6 lines · Sean Christopherson
For all of the KVM patches, please use
KVM: x86:
"x86/kvm" is used for guest-side code, and while I hope no one will conflate this with guest code, the consistency is helpful.
Sure.
Quoted 36 lines · Peter Zijlstra
Quoted 19 lines
Replace the FASTOP1SRC2*() instructions.
Signed-off-by Peter Zijlstra (Intel) <[email protected]>--- arch/x86/kvm/emulate.c | 34 ++++++++++++++++++++++++++-------- 1 file changed, 26 insertions(+), 8 deletions(-)arch/x86/kvm/emulate.c +6
-0@@ -317,6 +317,24 @@ static int em_##op(struct x86_emulate_ct ON64(case 8: __EM_ASM_1(op##q, rax); break;) \ EM_ASM_END +/* 1-operand, using "c" (src2) */ +#define EM_ASM_1SRC2 (op, name) \ + EM_ASM_START(name) \ + case 1: __EM_ASM_1(op##b, cl); break; \ + case 2: __EM_ASM_1(op##w, cx); break; \ + case 4: __EM_ASM_1(op##l, ecx); break; \arch/x86/kvm/emulate.c static int em_##op(struct x86_emulate_ct · line 335
Quoted 10 lines
arch/x86/kvm/emulate.c +10
-0+ ON64(case 8: __EM_ASM_1(op##q, rcx); break;) \ + EM_ASM_END + +/* 1-operand, using "c" (src2) with exception */ +#define EM_ASM_1SRC2EX (op, name) \ + EM_ASM_START(name) \ + case 1: __EM_ASM_1_EX(op##b, cl); break; \ + case 2: __EM_ASM_1_EX(op##w, cx); break; \ + case 4: __EM_ASM_1_EX(op##l, ecx); break; \ + ON64(case 8: __EM_ASM_1(op##q, rcx); break;) \This needs to be
__EM_ASM_1_EX(). Luckily, KVM-Unit-Tests actually has testcase for divq (somewhere in the morass of testcases). I also now have an extension to the fastops selftest to explicitly test all four flavors of div-by-zero; I'll get it posted tomorrow.
Quoted 3 lines
(also, don't also me how long it took me to spot the copy+paste typo; I was full
on debugging the exception fixup code before I realized my local diff looked "odd", *sigh*)
Urgh, sorry about that. Typically I use regex for these things, clearly I messed up here.
Thanks and fixed!
Peter Zijlstra
maintainerRe: [PATCH v3 12/16] x86_64,hyperv: Use direct call to hypercall-page
In reply to Michael KelleyQuoted 5 lines · Michael Kelley
From: Peter Zijlstra <[email protected]> Sent: Monday, July 14, 2025 3:20 AM
Quoted 1 line
Same nit here: Use "x86/hyperv:" as the Subject prefix, for consistency with past practice.
Sure, done. Thanks!
Peter Zijlstra
maintainerRe: [PATCH v3 14/16] x86/fred: Play nice with invoking asm_fred_entry_from_kvm() on non-FRED hardware
In reply to Xin LiQuoted 10 lines · Xin Li
arch/x86/entry/entry_64_fred.S SYM_FUNC_START(asm_fred_entry_from_kvm) · line 131
Quoted 6 lines · Peter Zijlstra
call
__fred_entry_from_kvm/* Call the C entry point */arch/x86/entry/entry_64_fred.S +2
-3- POP_REGS - ERETS -1: + +1: /*The symbol "1" is misplaced; it needs to be put after the ERETS instruction.
Doh, fixed.
Quoted 6 lines
arch/x86/entry/entry_64_fred.S SYM_FUNC_START(asm_fred_entry_from_kvm) · line 135
Quoted 4 lines
arch/x86/entry/entry_64_fred.S +4
-0+ * When FRED, use ERETS to potentially clear NMIs, otherwise simply + * restore the stack pointer. + */ + ALTERNATIVE "nop; nop; mov %rbp, %rsp", \Why explicitly add two nops here?
Because the CFI information for all alternative code flows must be the same. So by playing games with instruction offsets you can have conflicting CFI inside the alternative.
Specifically, we have:
0: 90 nop 1: 90 nop 2: 48 89 ec mov %rbp, %rsp
0: 48 83 c4 0c add $12, %rsp
4: f2 0f 01 ca erets
This gets us CFI updates on 0, 2 and 4, without conflicts.
Quoted 18 lines
ALTERNATIVE will still pad three-byte nop after the MOV instruction.
Quoted 15 lines
arch/x86/entry/entry_64_fred.S +7
-3+ __stringify(add $C_PTREGS_SIZE, %rsp; ERETS), \ + X86_FEATURE_FRED + /* - * Objtool doesn't understand what ERETS does, this hint tells it that - * yes, we'll reach here and with what stack state. A save/restore pair - * isn't strictly needed, but it's the simplest form. + * Objtool doesn't understand ERETS, and the cfi register state is + * different from initial_func_cfi due to PUSH_REGS. Tell it the state + * is similar to where UNWIND_HINT_SAVE is. */ UNWIND_HINT_RESTORE + pop %rbp RET
[tip: x86/core] x86/fred: KVM: VMX: Always use FRED for IRQs when CONFIG_X86_FRED=y
In reply to Peter ZijlstraThe following commit has been merged into the x86/core branch of tip:
The following commit has been merged into the x86/core branch of tip:
Commit-ID: 28d11e4548b75d0960429344f12d5f6cc9cee25b Gitweb: https://git.kernel.org/tip/28d11e4548b75d0960429344f12d5f6cc9cee25b Author: Sean Christopherson <[email protected]> AuthorDate: Thu, 01 May 2025 11:10:39 -07:00 Committer: Peter Zijlstra <[email protected]> CommitterDate: Mon, 18 Aug 2025 14:23:08 +02:00
x86/fred: KVM: VMX: Always use FRED for IRQs when CONFIG_X86_FRED=y
Now that FRED provides C-code entry points for handling IRQs, use the FRED infrastructure for forwarding IRQs even if FRED is fully disabled, e.g. isn't supported in hardware. Avoiding the non-FRED assembly trampolines into the IDT handlers for IRQs eliminates the associated non-CFI indirect call (KVM performs a CALL by doing a lookup on the IDT using the IRQ vector).
Keep NMIs on the legacy IDT path, as the FRED NMI entry code relies on FRED's architectural behavior with respect to NMI blocking, i.e. doesn't jump through the myriad hoops needed to deal with IRET "unexpectedly" unmasking NMIs. KVM's NMI path already makes a direct CALL to C-code, i.e. isn't problematic for CFI. KVM does make a short detour through assembly code to build the stack frame, but the "FRED entry from KVM" path does the same.
Force FRED for 64-bit kernels if KVM_INTEL is enabled, as the benefits of eliminating the IRQ trampoline usage far outwieghts the code overhead for FRED.
--- arch/x86/kvm/Kconfig | 1 + arch/x86/kvm/vmx/vmx.c | 8 +++++++- 2 files changed, 8 insertions(+), 1 deletion(-)
arch/x86/kvm/Kconfig +1 -0
@@ -97,6 +97,7 @@ config KVM_INTEL depends on KVM && IA32_FEAT_CTL select KVM_GENERIC_PRIVATE_MEM if INTEL_TDX_HOST select KVM_GENERIC_MEMORY_ATTRIBUTES if INTEL_TDX_HOST + select X86_FRED if X86_64 help Provides support for KVM on processors equipped with Intel's VT extensions, a.k.a. Virtual Machine Extensions (VMX).
arch/x86/kvm/vmx/vmx.c +7 -1
@@ -6913,8 +6913,14 @@ static void handle_external_interrupt_irqoff(struct kvm_vcpu *vcpu, "unexpected VM-Exit interrupt info: 0x%x", intr_info)) return; +/* +* Invoke the kernel's IRQ handler for the vector. Use the FRED path +* when it's available even if FRED isn't fully enabled, e.g. even if +* FRED isn't supported in hardware, in order to avoid the indirect +* CALL in the non-FRED path. +*/ kvm_before_interrupt (vcpu,KVM_HANDLING_IRQ ); -if (cpu_feature_enabled (X86_FEATURE_FRED )) +if (IS_ENABLED (CONFIG_X86_FRED ))fred_entry_from_kvm (EVENT_TYPE_EXTINT , vector);else vmx_do_interrupt_irqoff (gate_offset ((gate_desc *)host_idt_base + vector));
[tip: x86/core] objtool: Validate kCFI calls
In reply to Peter ZijlstraThe following commit has been merged into the x86/core branch of tip:
The following commit has been merged into the x86/core branch of tip:
Commit-ID: 894af4a1cde61c3401f237184fb770f72ff12df8 Gitweb: https://git.kernel.org/tip/894af4a1cde61c3401f237184fb770f72ff12df8 Author: Peter Zijlstra <[email protected]> AuthorDate: Sat, 12 Apr 2025 13:56:01 +02:00 Committer: Peter Zijlstra <[email protected]> CommitterDate: Mon, 18 Aug 2025 14:23:09 +02:00
objtool: Validate kCFI calls
Validate that all indirect calls adhere to kCFI rules. Notably doing nocfi indirect call to a cfi function is broken.
Apparently some Rust 'core' code violates this and explodes when ran with FineIBT.
All the ANNOTATE_NOCFI_SYM sites are prime targets for attackers.
- runtime EFI is especially henous because it also needs to disable IBT. Basically calling unknown code without CFI protection at runtime is a massice security issue.
- Kexec image handover; if you can exploit this, you get to keep it :-)
--- arch/x86/kernel/machine_kexec_64.c | 4 +++- arch/x86/kvm/vmx/vmenter.S | 4 +++- arch/x86/platform/efi/efi_stub_64.S | 4 +++- drivers/misc/lkdtm/perms.c | 5 +++- include/linux/objtool.h | 10 +++++++- include/linux/objtool_types.h | 1 +- tools/include/linux/objtool_types.h | 1 +- tools/objtool/check.c | 42 ++++++++++++++++++++++++++++- tools/objtool/include/objtool/elf.h | 1 +- 9 files changed, 72 insertions(+)
arch/x86/kernel/machine_kexec_64.c +4 -0
@@ -453,6 +453,10 @@ void __nocfi machine_kexec(struct kimage *image)__ftrace_enabled_restore (save_ftrace_enabled); } +/* +* Handover to the next kernel, no CFI concern. +*/ +ANNOTATE_NOCFI_SYM (machine_kexec);/* arch-dependent functionality related to kexec file-based syscall */
arch/x86/kvm/vmx/vmenter.S +4 -0
@@ -361,6 +361,10 @@ SYM_FUNC_END(vmread_error_trampoline) .section .text, "ax" +#ifndef CONFIG_X86_FRED + SYM_FUNC_START(vmx_do_interrupt_irqoff) VMX_DO_EVENT_IRQOFF CALL_NOSPEC _ASM_ARG1 SYM_FUNC_END(vmx_do_interrupt_irqoff) + +#endif
arch/x86/platform/efi/efi_stub_64.S +4 -0
@@ -11,6 +11,10 @@ #include <asm/nospec-branch.h> SYM_FUNC_START(__efi_call) + /* + * The EFI code doesn't have any CFI, annotate away the CFI violation. + */ + ANNOTATE_NOCFI_SYM pushq %rbp movq %rsp, %rbp and $~0xf, %rsp
drivers/misc/lkdtm/perms.c +5 -0
@@ -9,6 +9,7 @@#include <linux/vmalloc.h> #include <linux/mman.h> #include <linux/uaccess.h> +#include <linux/objtool.h> #include <asm/cacheflush.h> #include <asm/sections.h>
@@ -86,6 +87,10 @@ static noinline __nocfi void execute_location(void *dst, bool write)func ();pr_err ("FAIL: func returned\n" ); } +/* +* Explicitly doing the wrong thing for testing. +*/ +ANNOTATE_NOCFI_SYM (execute_location);static void execute_user_location (void *dst) {
include/linux/objtool.h +10 -0
@@ -184,6 +184,15 @@* WARN using UD2. */ #define ANNOTATE_REACHABLE (label) __ASM_ANNOTATE(label, ANNOTYPE_REACHABLE) +/* +* This should not be used; it annotates away CFI violations. There are a few +* valid use cases like kexec handover to the next kernel image, and there is +* no security concern there. +* +* There are also a few real issues annotated away, like EFI because we can't +* control the EFI code. +*/ +#define ANNOTATE_NOCFI_SYM (sym) asm(__ASM_ANNOTATE(sym, ANNOTYPE_NOCFI))#else #define ANNOTATE_NOENDBR ANNOTATE type=ANNOTYPE_NOENDBR
@@ -194,6 +203,7 @@#define ANNOTATE_INTRA_FUNCTION_CALL ANNOTATE type=ANNOTYPE_INTRA_FUNCTION_CALL#define ANNOTATE_UNRET_BEGIN ANNOTATE type=ANNOTYPE_UNRET_BEGIN#define ANNOTATE_REACHABLE ANNOTATE type=ANNOTYPE_REACHABLE +#define ANNOTATE_NOCFI_SYM ANNOTATE type=ANNOTYPE_NOCFI#endif #if defined(CONFIG_NOINSTR_VALIDATION) && \
include/linux/objtool_types.h +1 -0
@@ -65,5 +65,6 @@ struct unwind_hint {#define ANNOTYPE_IGNORE_ALTS 6#define ANNOTYPE_INTRA_FUNCTION_CALL 7#define ANNOTYPE_REACHABLE 8 +#define ANNOTYPE_NOCFI 9#endif /* _LINUX_OBJTOOL_TYPES_H */
tools/include/linux/objtool_types.h +1 -0
@@ -65,5 +65,6 @@ struct unwind_hint {#define ANNOTYPE_IGNORE_ALTS 6#define ANNOTYPE_INTRA_FUNCTION_CALL 7#define ANNOTYPE_REACHABLE 8 +#define ANNOTYPE_NOCFI 9#endif /* _LINUX_OBJTOOL_TYPES_H */
tools/objtool/check.c +42 -0
60 lines not on this page · raw message
tools/objtool/include/objtool/elf.h +1 -0
@@ -70,6 +70,7 @@ struct symbol { u8 local_label : 1; u8 frame_pointer : 1; u8 ignore : 1; + u8 nocfi : 1;struct list_head pv_target;struct reloc *relocs;struct section *group_sec;
[tip: x86/core] x86/fred: Play nice with invoking asm_fred_entry_from_kvm() on non-FRED hardware
In reply to Peter ZijlstraThe following commit has been merged into the x86/core branch of tip:
The following commit has been merged into the x86/core branch of tip:
Commit-ID: deed19b9b28724bd32e85063c60718c0a6803906 Gitweb: https://git.kernel.org/tip/deed19b9b28724bd32e85063c60718c0a6803906 Author: Josh Poimboeuf <[email protected]> AuthorDate: Fri, 06 Jun 2025 13:04:25 +02:00 Committer: Peter Zijlstra <[email protected]> CommitterDate: Mon, 18 Aug 2025 14:23:08 +02:00
x86/fred: Play nice with invoking asm_fred_entry_from_kvm() on non-FRED hardware
Modify asm_fred_entry_from_kvm() to allow it to be invoked by KVM even when FRED isn't fully enabled, e.g. when running with CONFIG_X86_FRED=y on non-FRED hardware. This will allow forcing KVM to always use the FRED entry points for 64-bit kernels, which in turn will eliminate a rather gross non-CFI indirect call that KVM uses to trampoline IRQs by doing IDT lookups.
The point of asm_fred_entry_from_kvm() is to bridge between C (vmx:handle_external_interrupt_irqoff()) and more C (__fred_entry_from_kvm()) while changing the calling context to appear like an interrupt (pt_regs). Making the whole thing bound by C ABI.
All that remains for non-FRED hardware is to restore RSP (to undo the redzone and alignment). However the trivial change would result in code like:
push %rbp
mov %rsp, %rbp
sub $REDZONE, %rsp
and $MASK, %rsp
PUSH_AND_CLEAR_REGS
push %rbp
POP_REGS
pop %rbp <-- *objtool fail*
mov %rbp, %rsp
pop %rbp
ret
And this will confuse objtool something wicked -- it gets confused by the extra pop %rbp, not realizing the push and pop preserve the value.
Rather than trying to each objtool about this, recognise that since the code is bound by C ABI on both ends and interrupts are not allowed to change pt_regs (only exceptions are) it is sufficient to PUSH_REGS in order to create pt_regs, but there is no reason to POP_REGS -- provided the callee-saved registers are preserved.
So avoid clearing callee-saved regs and skip POP_REGS.
[Original patch by Sean; much of this version by Josh; Changelog, comments and final form by Peterz]
--- arch/x86/entry/calling.h | 11 +++++------ arch/x86/entry/entry_64_fred.S | 33 ++++++++++++++++++++++++++------- arch/x86/kernel/asm-offsets.c | 1 +- 3 files changed, 32 insertions(+), 13 deletions(-)
arch/x86/entry/calling.h +5 -6
@@ -99,7 +99,7 @@ For 32-bit we have the following conventions - kernel is built with .endif .endm -.macro CLEAR_REGS clear_bp=1 +.macro CLEAR_REGS clear_callee=1/* * Sanitize registers of values that a speculation attack might * otherwise want to exploit. The lower registers are likely clobbered
@@ -113,20 +113,19 @@ For 32-bit we have the following conventions - kernel is built with xorl %r9d, %r9d /* nospec r9 */ xorl %r10d, %r10d /* nospec r10 */ xorl %r11d, %r11d /* nospec r11 */ + .if \clear_callee xorl %ebx, %ebx /* nospec rbx */ - .if \clear_bp xorl %ebp, %ebp /* nospec rbp */ - .endif xorl %r12d, %r12d /* nospec r12 */ xorl %r13d, %r13d /* nospec r13 */ xorl %r14d, %r14d /* nospec r14 */ xorl %r15d, %r15d /* nospec r15 */ - + .endif .endm -.macro PUSH_AND_CLEAR_REGS rdx=%rdx rcx=%rcx rax=%rax save_ret=0 clear_bp=1 unwind_hint=1 +.macro PUSH_AND_CLEAR_REGS rdx=%rdx rcx=%rcx rax=%rax save_ret=0 clear_callee=1 unwind_hint=1 PUSH_REGS rdx=\rdx, rcx=\rcx, rax=\rax, save_ret=\save_ret unwind_hint=\unwind_hint - CLEAR_REGS clear_bp=\clear_bp + CLEAR_REGS clear_callee=\clear_callee .endm .macro POP_REGS pop_rdi=1
arch/x86/entry/entry_64_fred.S +26 -7
@@ -112,18 +112,37 @@ SYM_FUNC_START(asm_fred_entry_from_kvm) push %rax /* Return RIP */ push $0 /* Error code, 0 for IRQ/NMI */ - PUSH_AND_CLEAR_REGS clear_bp=0 unwind_hint=0 + PUSH_AND_CLEAR_REGS clear_callee=0 unwind_hint=0 + movq %rsp, %rdi /* %rdi -> pt_regs */ + /* + * At this point: {rdi, rsi, rdx, rcx, r8, r9}, {r10, r11}, {rax, rdx} + * are clobbered, which corresponds to: arguments, extra caller-saved + * and return. All registers a C function is allowed to clobber. + * + * Notably, the callee-saved registers: {rbx, r12, r13, r14, r15} + * are untouched, with the exception of rbp, which carries the stack + * frame and will be restored before exit. + * + * Further calling another C function will not alter this state. + */ call __fred_entry_from_kvm /* Call the C entry point */ - POP_REGS - ERETS -1: + /* - * Objtool doesn't understand what ERETS does, this hint tells it that - * yes, we'll reach here and with what stack state. A save/restore pair - * isn't strictly needed, but it's the simplest form. + * When FRED, use ERETS to potentially clear NMIs, otherwise simply + * restore the stack pointer. + */ + ALTERNATIVE "nop; nop; mov %rbp, %rsp", \ + __stringify(add $C_PTREGS_SIZE, %rsp; ERETS), \ + X86_FEATURE_FRED + +1: /* + * Objtool doesn't understand ERETS, and the cfi register state is + * different from initial_func_cfi due to PUSH_REGS. Tell it the state + * is similar to where UNWIND_HINT_SAVE is. */ UNWIND_HINT_RESTORE + pop %rbp RET
arch/x86/kernel/asm-offsets.c +1 -0
@@ -102,6 +102,7 @@ static void __used common(void)BLANK ();DEFINE (PTREGS_SIZE ,sizeof (struct pt_regs )); +OFFSET (C_PTREGS_SIZE , pt_regs, orig_ax);/* TLB state for the entry code */ OFFSET (TLB_STATE_user_pcid_flush_mask, tlb_state, user_pcid_flush_mask);
[tip: x86/core] x86/fred: Install system vector handlers even if FRED isn't fully enabled
In reply to Peter ZijlstraThe following commit has been merged into the x86/core branch of tip:
The following commit has been merged into the x86/core branch of tip:
Commit-ID: 2d1435b742156f6d4adc8b9ab0f24b35d879266e Gitweb: https://git.kernel.org/tip/2d1435b742156f6d4adc8b9ab0f24b35d879266e Author: Sean Christopherson <[email protected]> AuthorDate: Fri, 02 May 2025 07:24:01 -07:00 Committer: Peter Zijlstra <[email protected]> CommitterDate: Mon, 18 Aug 2025 14:23:08 +02:00
x86/fred: Install system vector handlers even if FRED isn't fully enabled
Install the system vector IRQ handlers for FRED even if FRED isn't fully enabled in hardware. This will allow KVM to use the FRED IRQ path even on non-FRED hardware, which in turn will eliminate a non-CFI indirect CALL (KVM currently invokes the IRQ handler via an IDT lookup on the vector).
--- arch/x86/include/asm/idtentry.h | 9 ++------- arch/x86/kernel/irqinit.c | 6 ++++-- 2 files changed, 6 insertions(+), 9 deletions(-)
arch/x86/include/asm/idtentry.h +2 -7
@@ -460,17 +460,12 @@ __visible noinstr void func(struct pt_regs *regs, \#endif void idt_install_sysvec (unsigned int n,const void *function); - -#ifdef CONFIG_X86_FRED void fred_install_sysvec (unsigned int vector,const idtentry_t function); -#else -static inline void fred_install_sysvec (unsigned int vector,const idtentry_t function) { } -#endif #define sysvec_install (vector, function) { \ - if (cpu_feature_enabled(X86_FEATURE_FRED)) \ + if (IS_ENABLED(CONFIG_X86_FRED)) \ fred_install_sysvec(vector, function); \ - else \ + if (!cpu_feature_enabled(X86_FEATURE_FRED)) \ idt_install_sysvec(vector, asm_##function); \ }
arch/x86/kernel/irqinit.c +4 -2
@@ -97,9 +97,11 @@ void __init native_init_IRQ(void)/* Execute any quirks before the call gates are initialised: */ x86_init.irqs.pre_vector_init (); -if (cpu_feature_enabled (X86_FEATURE_FRED )) +/* FRED's IRQ path may be used even if FRED isn't fully enabled. */ +if (IS_ENABLED (CONFIG_X86_FRED ))fred_complete_exception_setup (); -else + +if (!cpu_feature_enabled (X86_FEATURE_FRED ))idt_setup_apic_and_irq_gates ();lapic_assign_system_vectors ();
[tip: x86/core] x86/hyperv: Use direct call to hypercall-page
In reply to Peter ZijlstraThe following commit has been merged into the x86/core branch of tip:
The following commit has been merged into the x86/core branch of tip:
Commit-ID: c8ed0812646e1335c80a8f204c1b92b2f9d76119 Gitweb: https://git.kernel.org/tip/c8ed0812646e1335c80a8f204c1b92b2f9d76119 Author: Peter Zijlstra <[email protected]> AuthorDate: Sat, 12 Apr 2025 13:55:55 +02:00 Committer: Peter Zijlstra <[email protected]> CommitterDate: Mon, 18 Aug 2025 14:23:07 +02:00
x86/hyperv: Use direct call to hypercall-page
Instead of using an indirect call to the hypercall page, use a direct call instead. This avoids all CFI problems, including the one where the hypercall page doesn't have IBT on.
--- arch/x86/hyperv/hv_init.c | 61 ++++++++++++++++++-------------------- 1 file changed, 30 insertions(+), 31 deletions(-)
arch/x86/hyperv/hv_init.c +30 -31
@@ -17,7 +17,6 @@#include <asm/desc.h> #include <asm/e820/api.h> #include <asm/sev.h> -#include <asm/ibt.h> #include <asm/hypervisor.h> #include <hyperv/hvhdk.h> #include <asm/mshyperv.h>
@@ -39,23 +38,41 @@void *hv_hypercall_pg;#ifdef CONFIG_X86_64 +static u64 __hv_hyperfail (u64 control,u64 param1,u64 param2) +{ +return U64_MAX ; +} + +DEFINE_STATIC_CALL (__hv_hypercall, __hv_hyperfail); +u64 hv_std_hypercall (u64 control,u64 param1,u64 param2) {u64 hv_status; -if (!hv_hypercall_pg) -return U64_MAX ; - register u64 __r8 asm("r8") = param2; - asm volatile (CALL_NOSPEC + asm volatile ("call " STATIC_CALL_TRAMP_STR(__hv_hypercall) : "=a" (hv_status), ASM_CALL_CONSTRAINT, "+c" (control), "+d" (param1), "+r" (__r8) - : THUNK_TARGET(hv_hypercall_pg) - : "cc", "memory", "r9", "r10", "r11"); + : : "cc", "memory", "r9", "r10", "r11");return hv_status; } + +typedef u64 (*hv_hypercall_f )(u64 control,u64 param1,u64 param2); + +static inline void hv_set_hypercall_pg (void *ptr) +{ + hv_hypercall_pg = ptr; + +if (!ptr) + ptr = &__hv_hyperfail; +static_call_update (__hv_hypercall, (hv_hypercall_f )ptr); +}#else +static inline void hv_set_hypercall_pg (void *ptr) +{ + hv_hypercall_pg = ptr; +}EXPORT_SYMBOL_GPL (hv_hypercall_pg);#endif
@@ -350,7 +367,7 @@ static int hv_suspend(void)* pointer is restored on resume. */ hv_hypercall_pg_saved = hv_hypercall_pg; - hv_hypercall_pg =NULL ; +hv_set_hypercall_pg (NULL );/* Disable the hypercall page in the hypervisor */ rdmsrq (HV_X64_MSR_HYPERCALL , hypercall_msr.as_uint64);
@@ -376,7 +393,7 @@ static void hv_resume(void)vmalloc_to_pfn (hv_hypercall_pg_saved);wrmsrq (HV_X64_MSR_HYPERCALL , hypercall_msr.as_uint64); - hv_hypercall_pg = hv_hypercall_pg_saved; +hv_set_hypercall_pg (hv_hypercall_pg_saved); hv_hypercall_pg_saved =NULL ;/*
@@ -496,8 +513,8 @@ void __init hyperv_init(void)if (hv_isolation_type_tdx () && !ms_hyperv.paravisor_present) goto skip_hypercall_pg_init; - hv_hypercall_pg =__vmalloc_node_range (PAGE_SIZE ,1 ,VMALLOC_START , -VMALLOC_END ,GFP_KERNEL ,PAGE_KERNEL_ROX , + hv_hypercall_pg =__vmalloc_node_range (PAGE_SIZE ,1 ,MODULES_VADDR , +MODULES_END ,GFP_KERNEL ,PAGE_KERNEL_ROX ,VM_FLUSH_RESET_PERMS ,NUMA_NO_NODE ,__builtin_return_address (0 ));if (hv_hypercall_pg ==NULL )
@@ -535,27 +552,9 @@ void __init hyperv_init(void)wrmsrq (HV_X64_MSR_HYPERCALL , hypercall_msr.as_uint64); } -skip_hypercall_pg_init: -/* -* Some versions of Hyper-V that provide IBT in guest VMs have a bug -* in that there's no ENDBR64 instruction at the entry to the -* hypercall page. Because hypercalls are invoked via an indirect call -* to the hypercall page, all hypercall attempts fail when IBT is -* enabled, and Linux panics. For such buggy versions, disable IBT. -* -* Fixed versions of Hyper-V always provide ENDBR64 on the hypercall -* page, so if future Linux kernel versions enable IBT for 32-bit -* builds, additional hypercall page hackery will be required here -* to provide an ENDBR32. - */ -#ifdef CONFIG_X86_KERNEL_IBT -if (cpu_feature_enabled (X86_FEATURE_IBT ) && - *(u32 *)hv_hypercall_pg !=gen_endbr ()) { -setup_clear_cpu_cap (X86_FEATURE_IBT ); -pr_warn ("Disabling IBT because of Hyper-V bug\n" ); - } -#endif +hv_set_hypercall_pg (hv_hypercall_pg); +skip_hypercall_pg_init:/* * hyperv_init() is called before LAPIC is initialized: see * apic_intr_mode_init() -> x86_platform.apic_post_init() and
[tip: x86/core] KVM: x86: Convert em_salc() to C
In reply to Peter ZijlstraThe following commit has been merged into the x86/core branch of tip:
The following commit has been merged into the x86/core branch of tip:
Commit-ID: 77892e1fef34db650aa3de4be1d27267ca7f1330 Gitweb: https://git.kernel.org/tip/77892e1fef34db650aa3de4be1d27267ca7f1330 Author: Peter Zijlstra <[email protected]> AuthorDate: Sat, 26 Apr 2025 13:07:14 +02:00 Committer: Peter Zijlstra <[email protected]> CommitterDate: Mon, 18 Aug 2025 14:23:06 +02:00
KVM: x86: Convert em_salc() to C
Implement the SALC (Set AL if Carry) instruction in C.
--- arch/x86/kvm/emulate.c | 15 +++++++++------ 1 file changed, 9 insertions(+), 6 deletions(-)
arch/x86/kvm/emulate.c +9 -6
@@ -529,11 +529,14 @@ static int fastop(struct x86_emulate_ctxt *ctxt, fastop_t fop); ON64(FOP3E(op##q, rax, rdx, cl)) \ FOP_END -FOP_START(salc) -FOP_FUNC(salc) -"pushf; sbb %al, %al; popf \n\t" -FOP_RET(salc) -FOP_END ; +static int em_salc (struct x86_emulate_ctxt *ctxt) +{ +/* +* Set AL 0xFF if CF is set, or 0x00 when clear. +*/ + ctxt->dst.val =0xFF * !!(ctxt->eflags &X86_EFLAGS_CF ); +return X86EMUL_CONTINUE ; +}/* * XXX: inoutclob user must know where the argument is being expanded.
@@ -4423,7 +4426,7 @@ static const struct opcode opcode_table[256] = {G (Src2CL | ByteOp, group2),G (Src2CL, group2),I (DstAcc | SrcImmUByte | No64, em_aam),I (DstAcc | SrcImmUByte | No64, em_aad), -F (DstAcc | ByteOp | No64, em_salc), +I (DstAcc | ByteOp | No64, em_salc),I (DstAcc | SrcXLat | ByteOp, em_mov),/* 0xD8 - 0xDF */ N, E(0, &escape_d9), N, E(0, &escape_db), N, E(0, &escape_dd), N, N,
[tip: x86/core] KVM: x86: Remove fastops
In reply to Peter ZijlstraThe following commit has been merged into the x86/core branch of tip:
The following commit has been merged into the x86/core branch of tip:
Commit-ID: a1d34a444ce8c9a6ec4376247f80f0b777c6d3fe Gitweb: https://git.kernel.org/tip/a1d34a444ce8c9a6ec4376247f80f0b777c6d3fe Author: Peter Zijlstra <[email protected]> AuthorDate: Sat, 26 Apr 2025 12:55:35 +02:00 Committer: Peter Zijlstra <[email protected]> CommitterDate: Mon, 18 Aug 2025 14:23:07 +02:00
KVM: x86: Remove fastops
No more FASTOPs, remove the remains.
--- arch/x86/kvm/emulate.c | 172 +---------------------------------------- 1 file changed, 1 insertion(+), 171 deletions(-)
arch/x86/kvm/emulate.c +1 -171
@@ -167,7 +167,6 @@#define Unaligned ((u64)2 << 41)/* Explicitly unaligned (e.g. MOVDQU) */ #define Avx ((u64)3 << 41)/* Advanced Vector Extensions */ #define Aligned16 ((u64)4 << 41)/* Aligned to 16 byte boundary (e.g. FXSAVE) */ -#define Fastop ((u64)1 << 44)/* Use opcode::u.fastop */ #define NoWrite ((u64)1 << 45)/* No writeback */ #define SrcWrite ((u64)1 << 46)/* Write back src operand */ #define NoMod ((u64)1 << 47)/* Mod field is ignored */
@@ -203,7 +202,6 @@ struct opcode {const struct escape *esc;const struct instr_dual *idual;const struct mode_dual *mdual; -void (*fastop)(struct fastop *fake); } u;int (*check_perm)(struct x86_emulate_ctxt *ctxt); };
@@ -383,152 +381,6 @@ static int em_##op(struct x86_emulate_ctxt *ctxt) \ ON64(case 8: __EM_ASM_3(op##q, rax, rdx, cl); break;) \ EM_ASM_END - -/* -* fastop functions have a special calling convention: -* -* dst: rax (in/out) -* src: rdx (in/out) -* src2: rcx (in) -* flags: rflags (in/out) -* ex: rsi (in:fastop pointer, out:zero if exception) -* -* Moreover, they are all exactly FASTOP_SIZE bytes long, so functions for -* different operand sizes can be reached by calculation, rather than a jump -* table (which would be bigger than the code). -* -* The 16 byte alignment, considering 5 bytes for the RET thunk, 3 for ENDBR -* and 1 for the straight line speculation INT3, leaves 7 bytes for the -* body of the function. Currently none is larger than 4. -*/ -static int fastop (struct x86_emulate_ctxt *ctxt,fastop_t fop); - -#define FASTOP_SIZE 16 - -#define __FOP_FUNC (name) \ - ".align " __stringify(FASTOP_SIZE) " \n\t" \ - ".type " name ", @function \n\t" \ - name ":\n\t" \ - ASM_ENDBR \ - IBT_NOSEAL(name) - -#define FOP_FUNC (name) \ - __FOP_FUNC(#name) - -#define __FOP_RET (name) \ - "11: " ASM_RET \ - ".size " name ", .-" name "\n\t" - -#define FOP_RET (name) \ - __FOP_RET(#name) - -#define __FOP_START (op, align) \ - extern void em_##op(struct fastop *fake); \ - asm(".pushsection .text, \"ax\" \n\t" \ - ".global em_" #op " \n\t" \ - ".align " __stringify(align) " \n\t" \ - "em_" #op ":\n\t" - -#define FOP_START (op) __FOP_START(op, FASTOP_SIZE) - -#define FOP_END \ - ".popsection") - -#define __FOPNOP (name) \ - __FOP_FUNC(name) \ - __FOP_RET(name) - -#define FOPNOP () \ - __FOPNOP(__stringify(__UNIQUE_ID(nop))) - -#define FOP1E (op, dst) \ - __FOP_FUNC(#op "_" #dst) \ - "10: " #op " %" #dst " \n\t" \ - __FOP_RET(#op "_" #dst) - -#define FOP1EEX (op, dst) \ - FOP1E(op, dst) _ASM_EXTABLE_TYPE_REG(10b, 11b, EX_TYPE_ZERO_REG, %%esi) - -#define FASTOP1 (op) \ - FOP_START(op) \ - FOP1E(op##b, al) \ - FOP1E(op##w, ax) \ - FOP1E(op##l, eax) \ - ON64(FOP1E(op##q, rax)) \ - FOP_END - -/* 1-operand, using src2 (for MUL/DIV r/m) */ -#define FASTOP1SRC2 (op, name) \ - FOP_START(name) \ - FOP1E(op, cl) \ - FOP1E(op, cx) \ - FOP1E(op, ecx) \ - ON64(FOP1E(op, rcx)) \ - FOP_END - -/* 1-operand, using src2 (for MUL/DIV r/m), with exceptions */ -#define FASTOP1SRC2EX (op, name) \ - FOP_START(name) \ - FOP1EEX(op, cl) \ - FOP1EEX(op, cx) \ - FOP1EEX(op, ecx) \ - ON64(FOP1EEX(op, rcx)) \ - FOP_END - -#define FOP2E (op, dst, src) \ - __FOP_FUNC(#op "_" #dst "_" #src) \ - #op " %" #src ", %" #dst " \n\t" \ - __FOP_RET(#op "_" #dst "_" #src) - -#define FASTOP2 (op) \ - FOP_START(op) \ - FOP2E(op##b, al, dl) \ - FOP2E(op##w, ax, dx) \ - FOP2E(op##l, eax, edx) \ - ON64(FOP2E(op##q, rax, rdx)) \ - FOP_END - -/* 2 operand, word only */ -#define FASTOP2W (op) \ - FOP_START(op) \ - FOPNOP() \ - FOP2E(op##w, ax, dx) \ - FOP2E(op##l, eax, edx) \ - ON64(FOP2E(op##q, rax, rdx)) \ - FOP_END - -/* 2 operand, src is CL */ -#define FASTOP2CL (op) \ - FOP_START(op) \ - FOP2E(op##b, al, cl) \ - FOP2E(op##w, ax, cl) \ - FOP2E(op##l, eax, cl) \ - ON64(FOP2E(op##q, rax, cl)) \ - FOP_END - -/* 2 operand, src and dest are reversed */ -#define FASTOP2R (op, name) \ - FOP_START(name) \ - FOP2E(op##b, dl, al) \ - FOP2E(op##w, dx, ax) \ - FOP2E(op##l, edx, eax) \ - ON64(FOP2E(op##q, rdx, rax)) \ - FOP_END - -#define FOP3E (op, dst, src, src2) \ - __FOP_FUNC(#op "_" #dst "_" #src "_" #src2) \ - #op " %" #src2 ", %" #src ", %" #dst " \n\t"\ - __FOP_RET(#op "_" #dst "_" #src "_" #src2) - -/* 3-operand, word-only, src2=cl */ -#define FASTOP3WCL (op) \ - FOP_START(op) \ - FOPNOP() \ - FOP3E(op##w, ax, dx, cl) \ - FOP3E(op##l, eax, edx, cl) \ - ON64(FOP3E(op##q, rax, rdx, cl)) \ - FOP_END -static int em_salc (struct x86_emulate_ctxt *ctxt) {/*
@@ -4052,7 +3904,6 @@ static int check_perm_out(struct x86_emulate_ctxt *ctxt)#define MD (_f, _m) { .flags = ((_f) | ModeDual), .u.mdual = (_m) }#define E (_f, _e) { .flags = ((_f) | Escape | ModRM), .u.esc = (_e) }#define I (_f, _e) { .flags = (_f), .u.execute = (_e) } -#define F (_f, _e) { .flags = (_f) | Fastop, .u.fastop = (_e) }#define II (_f, _e, _i) \ { .flags = (_f)|Intercept, .u.execute = (_e), .intercept = x86_intercept_##_i }#define IIP (_f, _e, _i, _p) \
@@ -5158,24 +5009,6 @@ static void fetch_possible_mmx_operand(struct operand *op)kvm_read_mmx_reg (op->addr.mm, &op->mm_val); } -static int fastop (struct x86_emulate_ctxt *ctxt,fastop_t fop) -{ -ulong flags = (ctxt->eflags &EFLAGS_MASK ) |X86_EFLAGS_IF ; - -if (!(ctxt->d & ByteOp)) - fop +=__ffs (ctxt->dst.bytes) *FASTOP_SIZE ; - - asm("push %[flags]; popf; " CALL_NOSPEC " ; pushf; pop %[flags]\n" - :"+a" (ctxt->dst.val),"+d" (ctxt->src.val), [flags]"+D" (flags), - [thunk_target]"+S"(fop), ASM_CALL_CONSTRAINT - :"c" (ctxt->src2.val)); - - ctxt->eflags = (ctxt->eflags & ~EFLAGS_MASK ) | (flags &EFLAGS_MASK ); -if (!fop)/* exception is returned in fop variable */ -return emulate_de (ctxt); -return X86EMUL_CONTINUE ; -} -void init_decode_cache (struct x86_emulate_ctxt *ctxt) {/* Clear fields that are set conditionally but read without a guard. */
@@ -5340,10 +5173,7 @@ special_insn: ctxt->eflags &= ~X86_EFLAGS_RF ;if (ctxt->execute) { -if (ctxt->d & Fastop) - rc =fastop (ctxt, ctxt->fop); -else - rc = ctxt->execute (ctxt); + rc = ctxt->execute (ctxt);if (rc !=X86EMUL_CONTINUE ) goto done; goto writeback;
[tip: x86/core] x86/hyperv: Clean up hv_do_hypercall()
In reply to Peter ZijlstraThe following commit has been merged into the x86/core branch of tip:
The following commit has been merged into the x86/core branch of tip:
Commit-ID: 0e20f1f4c2cb77130cfe903a058a08883645dc4b Gitweb: https://git.kernel.org/tip/0e20f1f4c2cb77130cfe903a058a08883645dc4b Author: Peter Zijlstra <[email protected]> AuthorDate: Sat, 12 Apr 2025 13:55:49 +02:00 Committer: Peter Zijlstra <[email protected]> CommitterDate: Mon, 18 Aug 2025 14:23:07 +02:00
x86/hyperv: Clean up hv_do_hypercall()
What used to be a simple few instructions has turned into a giant mess (for x86_64). Not only does it use static_branch wrong, it mixes it with dynamic branches for no apparent reason.
Notably it uses static_branch through an out-of-line function call, which completely defeats the purpose, since instead of a simple JMP/NOP site, you get a CALL+RET+TEST+Jcc sequence in return, which is absolutely idiotic.
Add to that a dynamic test of hyperv_paravisor_present, something which is set once and never changed.
Replace all this idiocy with a single direct function call to the right hypercall variant.
--- arch/x86/hyperv/hv_init.c | 20 +++++- arch/x86/hyperv/ivm.c | 15 +++- arch/x86/include/asm/mshyperv.h | 137 +++++++++---------------------- arch/x86/kernel/cpu/mshyperv.c | 19 ++-- 4 files changed, 89 insertions(+), 102 deletions(-)
arch/x86/hyperv/hv_init.c +20 -0
@@ -37,7 +37,27 @@#include <linux/export.h> void *hv_hypercall_pg; + +#ifdef CONFIG_X86_64 +u64 hv_std_hypercall (u64 control,u64 param1,u64 param2) +{ +u64 hv_status; + +if (!hv_hypercall_pg) +return U64_MAX ; + + register u64 __r8 asm("r8") = param2; + asm volatile (CALL_NOSPEC + : "=a" (hv_status), ASM_CALL_CONSTRAINT, + "+c" (control), "+d" (param1), "+r" (__r8) + : THUNK_TARGET(hv_hypercall_pg) + : "cc", "memory", "r9", "r10", "r11"); + +return hv_status; +} +#else EXPORT_SYMBOL_GPL (hv_hypercall_pg); +#endif union hv_ghcb *__percpu *hv_ghcb_pg;
arch/x86/hyperv/ivm.c +15 -0
@@ -385,9 +385,23 @@ int hv_snp_boot_ap(u32 apic_id, unsigned long start_ip, unsigned int cpu)return ret; } +u64 hv_snp_hypercall (u64 control,u64 param1,u64 param2) +{ +u64 hv_status; + + register u64 __r8 asm("r8") = param2; + asmvolatile ("vmmcall" + : "=a" (hv_status), ASM_CALL_CONSTRAINT, +"+c" (control),"+d" (param1),"+r" (__r8) + : :"cc" ,"memory" ,"r9" ,"r10" ,"r11" ); + +return hv_status; +} +#else static inline void hv_ghcb_msr_write (u64 msr,u64 value) {}static inline void hv_ghcb_msr_read (u64 msr,u64 *value) {} +u64 hv_snp_hypercall (u64 control,u64 param1,u64 param2) {return U64_MAX ; }#endif /* CONFIG_AMD_MEM_ENCRYPT */ #ifdef CONFIG_INTEL_TDX_GUEST
@@ -437,6 +451,7 @@ u64 hv_tdx_hypercall(u64 control, u64 param1, u64 param2)#else static inline void hv_tdx_msr_write (u64 msr,u64 value) {}static inline void hv_tdx_msr_read (u64 msr,u64 *value) {} +u64 hv_tdx_hypercall (u64 control,u64 param1,u64 param2) {return U64_MAX ; }#endif /* CONFIG_INTEL_TDX_GUEST */ #if defined(CONFIG_AMD_MEM_ENCRYPT ) || defined(CONFIG_INTEL_TDX_GUEST )
arch/x86/include/asm/mshyperv.h +41 -96
193 lines not on this page · raw message
arch/x86/kernel/cpu/mshyperv.c +13 -6
@@ -38,10 +38,6 @@bool hv_nested;struct ms_hyperv_info ms_hyperv; -/* Used in modules via hv_do_hypercall(): see arch/x86/include/asm/mshyperv.h */ -bool hyperv_paravisor_present__ro_after_init ; -EXPORT_SYMBOL_GPL (hyperv_paravisor_present); -#if IS_ENABLED (CONFIG_HYPERV )static inline unsigned int hv_get_nested_msr (unsigned int reg) {
@@ -288,8 +284,18 @@ static void __init x86_setup_ops_for_tsc_pg_clock(void) old_restore_sched_clock_state = x86_platform.restore_sched_clock_state; x86_platform.restore_sched_clock_state = hv_restore_sched_clock_state; } + +#ifdef CONFIG_X86_64 +DEFINE_STATIC_CALL (hv_hypercall, hv_std_hypercall); +EXPORT_STATIC_CALL_TRAMP_GPL (hv_hypercall); +#define hypercall_update (hc) static_call_update(hv_hypercall, hc) +#endif #endif /* CONFIG_HYPERV */ +#ifndef hypercall_update +#define hypercall_update (hc) (void)hc +#endif +static uint32_t __init ms_hyperv_platform (void ) {u32 eax;
@@ -484,14 +490,14 @@ static void __init ms_hyperv_init_platform(void) ms_hyperv.shared_gpa_boundary =BIT_ULL (ms_hyperv.shared_gpa_boundary_bits); - hyperv_paravisor_present = !!ms_hyperv.paravisor_present; -pr_info ("Hyper-V: Isolation Config: Group A 0x%x, Group B 0x%x\n" , ms_hyperv.isolation_config_a, ms_hyperv.isolation_config_b);if (hv_get_isolation_type () ==HV_ISOLATION_TYPE_SNP ) {static_branch_enable (&isolation_type_snp); +if (!ms_hyperv.paravisor_present) +hypercall_update (hv_snp_hypercall); }else if (hv_get_isolation_type () ==HV_ISOLATION_TYPE_TDX ) {static_branch_enable (&isolation_type_tdx);
@@ -499,6 +505,7 @@ static void __init ms_hyperv_init_platform(void) ms_hyperv.hints &= ~HV_X64_APIC_ACCESS_RECOMMENDED ;if (!ms_hyperv.paravisor_present) { +hypercall_update (hv_tdx_hypercall);/* * Mark the Hyper-V TSC page feature as disabled * in a TDX VM without paravisor so that the
[tip: x86/core] KVM: x86: Introduce EM_ASM_1SRC2
In reply to Peter ZijlstraThe following commit has been merged into the x86/core branch of tip:
The following commit has been merged into the x86/core branch of tip:
Commit-ID: af4fe07aa1e07610d3208a85210b896c87e2105e Gitweb: https://git.kernel.org/tip/af4fe07aa1e07610d3208a85210b896c87e2105e Author: Peter Zijlstra <[email protected]> AuthorDate: Sat, 26 Apr 2025 12:40:12 +02:00 Committer: Peter Zijlstra <[email protected]> CommitterDate: Mon, 18 Aug 2025 14:23:06 +02:00
KVM: x86: Introduce EM_ASM_1SRC2
Replace the FASTOP1SRC2*() instructions.
--- arch/x86/kvm/emulate.c | 34 ++++++++++++++++++++++++++-------- 1 file changed, 26 insertions(+), 8 deletions(-)
arch/x86/kvm/emulate.c +26 -8
@@ -317,6 +317,24 @@ static int em_##op(struct x86_emulate_ctxt *ctxt) \ ON64(case 8: __EM_ASM_1(op##q, rax); break;) \ EM_ASM_END +/* 1-operand, using "c" (src2) */ +#define EM_ASM_1SRC2 (op, name) \ + EM_ASM_START(name) \ + case 1: __EM_ASM_1(op##b, cl); break; \ + case 2: __EM_ASM_1(op##w, cx); break; \ + case 4: __EM_ASM_1(op##l, ecx); break; \ + ON64(case 8: __EM_ASM_1(op##q, rcx); break;) \ + EM_ASM_END + +/* 1-operand, using "c" (src2) with exception */ +#define EM_ASM_1SRC2EX (op, name) \ + EM_ASM_START(name) \ + case 1: __EM_ASM_1_EX(op##b, cl); break; \ + case 2: __EM_ASM_1_EX(op##w, cx); break; \ + case 4: __EM_ASM_1_EX(op##l, ecx); break; \ + ON64(case 8: __EM_ASM_1_EX(op##q, rcx); break;) \ + EM_ASM_END +/* 2-operand, using "a" (dst), "d" (src) */ #define EM_ASM_2 (op) \ EM_ASM_START(op) \
@@ -1074,10 +1092,10 @@ EM_ASM_2(cmp);EM_ASM_2 (test);EM_ASM_2 (xadd); -FASTOP1SRC2 (mul, mul_ex); -FASTOP1SRC2 (imul, imul_ex); -FASTOP1SRC2EX (div, div_ex); -FASTOP1SRC2EX (idiv, idiv_ex); +EM_ASM_1SRC2 (mul, mul_ex); +EM_ASM_1SRC2 (imul, imul_ex); +EM_ASM_1SRC2EX (div, div_ex); +EM_ASM_1SRC2EX (idiv, idiv_ex);FASTOP3WCL (shld);FASTOP3WCL (shrd);
@@ -4103,10 +4121,10 @@ static const struct opcode group3[] = {I (DstMem | SrcImm | NoWrite, em_test),I (DstMem | SrcNone | Lock, em_not),I (DstMem | SrcNone | Lock, em_neg), -F (DstXacc | Src2Mem, em_mul_ex), -F (DstXacc | Src2Mem, em_imul_ex), -F (DstXacc | Src2Mem, em_div_ex), - F(DstXacc | Src2Mem, em_idiv_ex), +I (DstXacc | Src2Mem, em_mul_ex), +I (DstXacc | Src2Mem, em_imul_ex), +I (DstXacc | Src2Mem, em_div_ex), + I(DstXacc | Src2Mem, em_idiv_ex), }; static const struct opcode group4[] = {
[tip: x86/core] KVM: x86: Introduce EM_ASM_3WCL
In reply to Peter ZijlstraThe following commit has been merged into the x86/core branch of tip:
The following commit has been merged into the x86/core branch of tip:
Commit-ID: 2d82acd7d6a7eba2ed2bf84ab7440c1ad7c27958 Gitweb: https://git.kernel.org/tip/2d82acd7d6a7eba2ed2bf84ab7440c1ad7c27958 Author: Peter Zijlstra <[email protected]> AuthorDate: Sat, 26 Apr 2025 12:47:50 +02:00 Committer: Peter Zijlstra <[email protected]> CommitterDate: Mon, 18 Aug 2025 14:23:06 +02:00
KVM: x86: Introduce EM_ASM_3WCL
Replace the FASTOP3WCL instructions.
--- arch/x86/kvm/emulate.c | 25 +++++++++++++++++++------ 1 file changed, 19 insertions(+), 6 deletions(-)
arch/x86/kvm/emulate.c +19 -6
@@ -302,6 +302,9 @@ static int em_##op(struct x86_emulate_ctxt *ctxt) \#define __EM_ASM_2 (op, dst, src) \ __EM_ASM(#op " %%" #src ", %%" #dst " \n\t") +#define __EM_ASM_3 (op, dst, src, src2) \ + __EM_ASM(#op " %%" #src2 ", %%" #src ", %%" #dst " \n\t") +#define EM_ASM_END \ } \ ctxt->eflags = (ctxt->eflags & ~EFLAGS_MASK) | (flags & EFLAGS_MASK); \
@@ -371,6 +374,16 @@ static int em_##op(struct x86_emulate_ctxt *ctxt) \ ON64(case 8: __EM_ASM_2(op##q, rax, cl); break;) \ EM_ASM_END +/* 3-operand, using "a" (dst), "d" (src) and CL (src2) */ +#define EM_ASM_3WCL (op) \ + EM_ASM_START(op) \ + case 1: break; \ + case 2: __EM_ASM_3(op##w, ax, dx, cl); break; \ + case 4: __EM_ASM_3(op##l, eax, edx, cl); break; \ + ON64(case 8: __EM_ASM_3(op##q, rax, rdx, cl); break;) \ + EM_ASM_END + +/* * fastop functions have a special calling convention: *
@@ -1097,8 +1110,8 @@ EM_ASM_1SRC2(imul, imul_ex);EM_ASM_1SRC2EX (div, div_ex);EM_ASM_1SRC2EX (idiv, idiv_ex); -FASTOP3WCL (shld); -FASTOP3WCL (shrd); +EM_ASM_3WCL (shld); +EM_ASM_3WCL (shrd);EM_ASM_2W (imul);
@@ -4496,14 +4509,14 @@ static const struct opcode twobyte_table[256] = {I (Stack | Src2FS, em_push_sreg),I (Stack | Src2FS, em_pop_sreg),II (ImplicitOps, em_cpuid, cpuid),I (DstMem | SrcReg | ModRM | BitOp | NoWrite, em_bt), -F (DstMem | SrcReg | Src2ImmByte | ModRM, em_shld), -F (DstMem | SrcReg | Src2CL | ModRM, em_shld),N ,N , +I (DstMem | SrcReg | Src2ImmByte | ModRM, em_shld), +I (DstMem | SrcReg | Src2CL | ModRM, em_shld),N ,N ,/* 0xA8 - 0xAF */ I (Stack | Src2GS, em_push_sreg),I (Stack | Src2GS, em_pop_sreg),II (EmulateOnUD | ImplicitOps, em_rsm, rsm),I (DstMem | SrcReg | ModRM | BitOp | Lock | PageTable, em_bts), -F (DstMem | SrcReg | Src2ImmByte | ModRM, em_shrd), -F (DstMem | SrcReg | Src2CL | ModRM, em_shrd), +I (DstMem | SrcReg | Src2ImmByte | ModRM, em_shrd), +I (DstMem | SrcReg | Src2CL | ModRM, em_shrd),GD (0 , &group15),I (DstReg | SrcMem | ModRM, em_imul),/* 0xB0 - 0xB7 */ I2bv(DstMem | SrcReg | ModRM | Lock | PageTable | SrcWrite, em_cmpxchg),
[tip: x86/core] KVM: x86: Introduce EM_ASM_2W
In reply to Peter ZijlstraThe following commit has been merged into the x86/core branch of tip:
The following commit has been merged into the x86/core branch of tip:
Commit-ID: 975e51783cfd0abc303817e8bf1ab2f630834c2a Gitweb: https://git.kernel.org/tip/975e51783cfd0abc303817e8bf1ab2f630834c2a Author: Peter Zijlstra <[email protected]> AuthorDate: Sat, 26 Apr 2025 12:30:37 +02:00 Committer: Peter Zijlstra <[email protected]> CommitterDate: Mon, 18 Aug 2025 14:23:05 +02:00
KVM: x86: Introduce EM_ASM_2W
Replace the FASTOP2W instructions.
--- arch/x86/kvm/emulate.c | 47 ++++++++++++++++++++++++----------------- 1 file changed, 28 insertions(+), 19 deletions(-)
arch/x86/kvm/emulate.c +28 -19
@@ -335,6 +335,15 @@ static int em_##op(struct x86_emulate_ctxt *ctxt) \ ON64(case 8: __EM_ASM_2(op##q, rdx, rax); break;) \ EM_ASM_END +/* 2-operand, word only (no byte op) */ +#define EM_ASM_2W (op) \ + EM_ASM_START(op) \ + case 1: break; \ + case 2: __EM_ASM_2(op##w, ax, dx); break; \ + case 4: __EM_ASM_2(op##l, eax, edx); break; \ + ON64(case 8: __EM_ASM_2(op##q, rax, rdx); break;) \ + EM_ASM_END +/* * fastop functions have a special calling convention: *
@@ -1064,7 +1073,7 @@ FASTOP1SRC2EX(idiv, idiv_ex);FASTOP3WCL (shld);FASTOP3WCL (shrd); -FASTOP2W (imul); +EM_ASM_2W (imul);EM_ASM_1 (not);EM_ASM_1 (neg);
@@ -1079,12 +1088,12 @@ FASTOP2CL(shl);FASTOP2CL (shr);FASTOP2CL (sar); -FASTOP2W (bsf); -FASTOP2W (bsr); -FASTOP2W (bt); -FASTOP2W (bts); -FASTOP2W (btr); -FASTOP2W (btc); +EM_ASM_2W (bsf); +EM_ASM_2W (bsr); +EM_ASM_2W (bt); +EM_ASM_2W (bts); +EM_ASM_2W (btr); +EM_ASM_2W (btc);EM_ASM_2R (cmp, cmp_r);
@@ -1093,7 +1102,7 @@ static int em_bsf_c(struct x86_emulate_ctxt *ctxt)/* If src is zero, do not writeback, but update flags */ if (ctxt->src.val ==0 ) ctxt->dst.type =OP_NONE ; -return fastop (ctxt, em_bsf); +return em_bsf (ctxt); }static int em_bsr_c (struct x86_emulate_ctxt *ctxt)
@@ -1101,7 +1110,7 @@ static int em_bsr_c(struct x86_emulate_ctxt *ctxt)/* If src is zero, do not writeback, but update flags */ if (ctxt->src.val ==0 ) ctxt->dst.type =OP_NONE ; -return fastop (ctxt, em_bsr); +return em_bsr (ctxt); } static __always_inline u8 test_cc(unsigned int condition, unsigned long flags)
@@ -3221,7 +3230,7 @@ static int em_xchg(struct x86_emulate_ctxt *ctxt)static int em_imul_3op (struct x86_emulate_ctxt *ctxt) { ctxt->dst.val = ctxt->src2.val; -return fastop (ctxt, em_imul); +return em_imul (ctxt); }static int em_cwd (struct x86_emulate_ctxt *ctxt)
@@ -4135,10 +4144,10 @@ static const struct group_dual group7 = { {static const struct opcode group8[] = {N ,N ,N ,N , -F (DstMem | SrcImmByte | NoWrite, em_bt), -F (DstMem | SrcImmByte | Lock | PageTable, em_bts), -F (DstMem | SrcImmByte | Lock, em_btr), -F (DstMem | SrcImmByte | Lock | PageTable, em_btc), +I (DstMem | SrcImmByte | NoWrite, em_bt), +I (DstMem | SrcImmByte | Lock | PageTable, em_bts), +I (DstMem | SrcImmByte | Lock, em_btr), +I (DstMem | SrcImmByte | Lock | PageTable, em_btc), };/*
@@ -4459,27 +4468,27 @@ static const struct opcode twobyte_table[256] = {/* 0xA0 - 0xA7 */ I (Stack | Src2FS, em_push_sreg),I (Stack | Src2FS, em_pop_sreg),II (ImplicitOps, em_cpuid, cpuid), -F (DstMem | SrcReg | ModRM | BitOp | NoWrite, em_bt), +I (DstMem | SrcReg | ModRM | BitOp | NoWrite, em_bt),F (DstMem | SrcReg | Src2ImmByte | ModRM, em_shld),F (DstMem | SrcReg | Src2CL | ModRM, em_shld),N ,N ,/* 0xA8 - 0xAF */ I (Stack | Src2GS, em_push_sreg),I (Stack | Src2GS, em_pop_sreg),II (EmulateOnUD | ImplicitOps, em_rsm, rsm), -F (DstMem | SrcReg | ModRM | BitOp | Lock | PageTable, em_bts), +I (DstMem | SrcReg | ModRM | BitOp | Lock | PageTable, em_bts),F (DstMem | SrcReg | Src2ImmByte | ModRM, em_shrd),F (DstMem | SrcReg | Src2CL | ModRM, em_shrd), -GD (0 , &group15),F (DstReg | SrcMem | ModRM, em_imul), +GD (0 , &group15),I (DstReg | SrcMem | ModRM, em_imul),/* 0xB0 - 0xB7 */ I2bv (DstMem | SrcReg | ModRM | Lock | PageTable | SrcWrite, em_cmpxchg),I (DstReg | SrcMemFAddr | ModRM | Src2SS, em_lseg), -F (DstMem | SrcReg | ModRM | BitOp | Lock, em_btr), +I (DstMem | SrcReg | ModRM | BitOp | Lock, em_btr),I (DstReg | SrcMemFAddr | ModRM | Src2FS, em_lseg),I (DstReg | SrcMemFAddr | ModRM | Src2GS, em_lseg),D (DstReg | SrcMem8 | ModRM | Mov),D (DstReg | SrcMem16 | ModRM | Mov),/* 0xB8 - 0xBF */ N ,N ,G (BitOp, group8), -F (DstMem | SrcReg | ModRM | BitOp | Lock | PageTable, em_btc), +I (DstMem | SrcReg | ModRM | BitOp | Lock | PageTable, em_btc),I (DstReg | SrcMem | ModRM, em_bsf_c),I (DstReg | SrcMem | ModRM, em_bsr_c), D(DstReg | SrcMem8 | ModRM | Mov), D(DstReg | SrcMem16 | ModRM | Mov),
[tip: x86/core] KVM: x86: Introduce EM_ASM_2CL
In reply to Peter ZijlstraThe following commit has been merged into the x86/core branch of tip:
The following commit has been merged into the x86/core branch of tip:
Commit-ID: 67e944aff63cb6b9c236d9cc2b4d870aeefd306f Gitweb: https://git.kernel.org/tip/67e944aff63cb6b9c236d9cc2b4d870aeefd306f Author: Peter Zijlstra <[email protected]> AuthorDate: Sat, 26 Apr 2025 12:34:42 +02:00 Committer: Peter Zijlstra <[email protected]> CommitterDate: Mon, 18 Aug 2025 14:23:06 +02:00
KVM: x86: Introduce EM_ASM_2CL
Replace the FASTOP2CL instructions.
--- arch/x86/kvm/emulate.c | 39 ++++++++++++++++++++++++--------------- 1 file changed, 24 insertions(+), 15 deletions(-)
arch/x86/kvm/emulate.c +24 -15
@@ -344,6 +344,15 @@ static int em_##op(struct x86_emulate_ctxt *ctxt) \ ON64(case 8: __EM_ASM_2(op##q, rax, rdx); break;) \ EM_ASM_END +/* 2-operand, using "a" (dst) and CL (src2) */ +#define EM_ASM_2CL (op) \ + EM_ASM_START(op) \ + case 1: __EM_ASM_2(op##b, al, cl); break; \ + case 2: __EM_ASM_2(op##w, ax, cl); break; \ + case 4: __EM_ASM_2(op##l, eax, cl); break; \ + ON64(case 8: __EM_ASM_2(op##q, rax, cl); break;) \ + EM_ASM_END +/* * fastop functions have a special calling convention: *
@@ -1080,13 +1089,13 @@ EM_ASM_1(neg);EM_ASM_1 (inc);EM_ASM_1 (dec); -FASTOP2CL (rol); -FASTOP2CL (ror); -FASTOP2CL (rcl); -FASTOP2CL (rcr); -FASTOP2CL (shl); -FASTOP2CL (shr); -FASTOP2CL (sar); +EM_ASM_2CL (rol); +EM_ASM_2CL (ror); +EM_ASM_2CL (rcl); +EM_ASM_2CL (rcr); +EM_ASM_2CL (shl); +EM_ASM_2CL (shr); +EM_ASM_2CL (sar);EM_ASM_2W (bsf);EM_ASM_2W (bsr);
@@ -4079,14 +4088,14 @@ static const struct opcode group1A[] = { };static const struct opcode group2[] = { -F (DstMem | ModRM, em_rol), -F (DstMem | ModRM, em_ror), -F (DstMem | ModRM, em_rcl), -F (DstMem | ModRM, em_rcr), -F (DstMem | ModRM, em_shl), -F (DstMem | ModRM, em_shr), -F (DstMem | ModRM, em_shl), -F (DstMem | ModRM, em_sar), +I (DstMem | ModRM, em_rol), +I (DstMem | ModRM, em_ror), +I (DstMem | ModRM, em_rcl), +I (DstMem | ModRM, em_rcr), +I (DstMem | ModRM, em_shl), +I (DstMem | ModRM, em_shr), +I (DstMem | ModRM, em_shl), +I (DstMem | ModRM, em_sar), };static const struct opcode group3[] = {
[tip: x86/core] KVM: x86: Introduce EM_ASM_2R
In reply to Peter ZijlstraThe following commit has been merged into the x86/core branch of tip:
The following commit has been merged into the x86/core branch of tip:
Commit-ID: 2df2b52c98db03c10ebe93fa270b1e2baf47f8a0 Gitweb: https://git.kernel.org/tip/2df2b52c98db03c10ebe93fa270b1e2baf47f8a0 Author: Peter Zijlstra <[email protected]> AuthorDate: Sat, 26 Apr 2025 12:52:05 +02:00 Committer: Peter Zijlstra <[email protected]> CommitterDate: Mon, 18 Aug 2025 14:23:05 +02:00
KVM: x86: Introduce EM_ASM_2R
Replace the FASTOP2R instruction.
--- arch/x86/kvm/emulate.c | 16 ++++++++++++---- 1 file changed, 12 insertions(+), 4 deletions(-)
arch/x86/kvm/emulate.c +12 -4
@@ -326,6 +326,15 @@ static int em_##op(struct x86_emulate_ctxt *ctxt) \ ON64(case 8: __EM_ASM_2(op##q, rax, rdx); break;) \ EM_ASM_END +/* 2-operand, reversed */ +#define EM_ASM_2R (op, name) \ + EM_ASM_START(name) \ + case 1: __EM_ASM_2(op##b, dl, al); break; \ + case 2: __EM_ASM_2(op##w, dx, ax); break; \ + case 4: __EM_ASM_2(op##l, edx, eax); break; \ + ON64(case 8: __EM_ASM_2(op##q, rdx, rax); break;) \ + EM_ASM_END +/* * fastop functions have a special calling convention: *
@@ -1077,8 +1086,7 @@ FASTOP2W(bts);FASTOP2W (btr);FASTOP2W (btc); - -FASTOP2R (cmp, cmp_r); +EM_ASM_2R (cmp, cmp_r);static int em_bsf_c (struct x86_emulate_ctxt *ctxt) {
@@ -4336,12 +4344,12 @@ static const struct opcode opcode_table[256] = {I2bv (DstAcc | SrcMem | Mov | MemAbs, em_mov),I2bv (DstMem | SrcAcc | Mov | MemAbs | PageTable, em_mov),I2bv (SrcSI | DstDI | Mov | String | TwoMemOp, em_mov), -F2bv (SrcSI | DstDI | String | NoWrite | TwoMemOp, em_cmp_r), +I2bv (SrcSI | DstDI | String | NoWrite | TwoMemOp, em_cmp_r),/* 0xA8 - 0xAF */ I2bv (DstAcc | SrcImm | NoWrite, em_test),I2bv (SrcAcc | DstDI | Mov | String, em_mov),I2bv (SrcSI | DstAcc | Mov | String, em_mov), -F2bv (SrcAcc | DstDI | String | NoWrite, em_cmp_r), +I2bv (SrcAcc | DstDI | String | NoWrite, em_cmp_r),/* 0xB0 - 0xB7 */ X8(I(ByteOp | DstReg | SrcImm | Mov, em_mov)),/* 0xB8 - 0xBF */
[tip: x86/core] KVM: x86: Introduce EM_ASM_2
In reply to Peter ZijlstraThe following commit has been merged into the x86/core branch of tip:
The following commit has been merged into the x86/core branch of tip:
Commit-ID: b26deb0cad4af9f522496fd25dc3ebca759e65a4 Gitweb: https://git.kernel.org/tip/b26deb0cad4af9f522496fd25dc3ebca759e65a4 Author: Peter Zijlstra <[email protected]> AuthorDate: Sat, 26 Apr 2025 12:25:21 +02:00 Committer: Peter Zijlstra <[email protected]> CommitterDate: Mon, 18 Aug 2025 14:23:05 +02:00
KVM: x86: Introduce EM_ASM_2
Replace the FASTOP2 instructions.
--- arch/x86/kvm/emulate.c | 85 ++++++++++++++++++++++------------------- 1 file changed, 47 insertions(+), 38 deletions(-)
arch/x86/kvm/emulate.c +47 -38
@@ -317,6 +317,15 @@ static int em_##op(struct x86_emulate_ctxt *ctxt) \ ON64(case 8: __EM_ASM_1(op##q, rax); break;) \ EM_ASM_END +/* 2-operand, using "a" (dst), "d" (src) */ +#define EM_ASM_2 (op) \ + EM_ASM_START(op) \ + case 1: __EM_ASM_2(op##b, al, dl); break; \ + case 2: __EM_ASM_2(op##w, ax, dx); break; \ + case 4: __EM_ASM_2(op##l, eax, edx); break; \ + ON64(case 8: __EM_ASM_2(op##q, rax, rdx); break;) \ + EM_ASM_END +/* * fastop functions have a special calling convention: *
@@ -1027,15 +1036,16 @@ static int read_descriptor(struct x86_emulate_ctxt *ctxt,return rc; } -FASTOP2 (add); -FASTOP2 (or); -FASTOP2 (adc); -FASTOP2 (sbb); -FASTOP2 (and); -FASTOP2 (sub); -FASTOP2 (xor); -FASTOP2 (cmp); -FASTOP2 (test); +EM_ASM_2 (add); +EM_ASM_2 (or); +EM_ASM_2 (adc); +EM_ASM_2 (sbb); +EM_ASM_2 (and); +EM_ASM_2 (sub); +EM_ASM_2 (xor); +EM_ASM_2 (cmp); +EM_ASM_2 (test); +EM_ASM_2 (xadd);FASTOP1SRC2 (mul, mul_ex);FASTOP1SRC2 (imul, imul_ex);
@@ -1067,7 +1077,6 @@ FASTOP2W(bts);FASTOP2W (btr);FASTOP2W (btc); -FASTOP2 (xadd);FASTOP2R (cmp, cmp_r);
@@ -2304,7 +2313,7 @@ static int em_cmpxchg(struct x86_emulate_ctxt *ctxt) ctxt->dst.val =reg_read (ctxt,VCPU_REGS_RAX ); ctxt->src.orig_val = ctxt->src.val; ctxt->src.val = ctxt->dst.orig_val; -fastop (ctxt, em_cmp); +em_cmp (ctxt);if (ctxt->eflags &X86_EFLAGS_ZF ) {/* Success: write back to memory; no update of EAX */
@@ -3069,7 +3078,7 @@ static int em_das(struct x86_emulate_ctxt *ctxt) ctxt->src.type =OP_IMM ; ctxt->src.val =0 ; ctxt->src.bytes =1 ; -fastop (ctxt, em_or); +em_or (ctxt); ctxt->eflags &= ~(X86_EFLAGS_AF |X86_EFLAGS_CF );if (cf) ctxt->eflags |=X86_EFLAGS_CF ;
@@ -3095,7 +3104,7 @@ static int em_aam(struct x86_emulate_ctxt *ctxt) ctxt->src.type =OP_IMM ; ctxt->src.val =0 ; ctxt->src.bytes =1 ; -fastop (ctxt, em_or); +em_or (ctxt);return X86EMUL_CONTINUE ; }
@@ -3113,7 +3122,7 @@ static int em_aad(struct x86_emulate_ctxt *ctxt) ctxt->src.type =OP_IMM ; ctxt->src.val =0 ; ctxt->src.bytes =1 ; -fastop (ctxt, em_or); +em_or (ctxt);return X86EMUL_CONTINUE ; }
@@ -3998,9 +4007,9 @@ static int check_perm_out(struct x86_emulate_ctxt *ctxt)#define I2bvIP (_f, _e, _i, _p) \ IIP((_f) | ByteOp, _e, _i, _p), IIP(_f, _e, _i, _p) -#define F6ALU (_f, _e) F2bv((_f) | DstMem | SrcReg | ModRM, _e), \ - F2bv(((_f) | DstReg | SrcMem | ModRM) & ~Lock, _e), \ - F2bv(((_f) & ~Lock) | DstAcc | SrcImm, _e) +#define I6ALU (_f, _e) I2bv((_f) | DstMem | SrcReg | ModRM, _e), \ + I2bv(((_f) | DstReg | SrcMem | ModRM) & ~Lock, _e), \ + I2bv(((_f) & ~Lock) | DstAcc | SrcImm, _e)static const struct opcode group7_rm0[] = {N ,
@@ -4038,14 +4047,14 @@ static const struct opcode group7_rm7[] = { };static const struct opcode group1[] = { -F (Lock, em_add), -F (Lock | PageTable, em_or), -F (Lock, em_adc), -F (Lock, em_sbb), -F (Lock | PageTable, em_and), -F (Lock, em_sub), -F (Lock, em_xor), -F (NoWrite, em_cmp), +I (Lock, em_add), +I (Lock | PageTable, em_or), +I (Lock, em_adc), +I (Lock, em_sbb), +I (Lock | PageTable, em_and), +I (Lock, em_sub), +I (Lock, em_xor), +I (NoWrite, em_cmp), };static const struct opcode group1A[] = {
@@ -4064,8 +4073,8 @@ static const struct opcode group2[] = { };static const struct opcode group3[] = { -F (DstMem | SrcImm | NoWrite, em_test), -F (DstMem | SrcImm | NoWrite, em_test), +I (DstMem | SrcImm | NoWrite, em_test), +I (DstMem | SrcImm | NoWrite, em_test),I (DstMem | SrcNone | Lock, em_not),I (DstMem | SrcNone | Lock, em_neg),F (DstXacc | Src2Mem, em_mul_ex),
@@ -4258,29 +4267,29 @@ static const struct instr_dual instr_dual_8d = {static const struct opcode opcode_table[256 ] = {/* 0x00 - 0x07 */ -F6ALU (Lock, em_add), +I6ALU (Lock, em_add),I (ImplicitOps | Stack | No64 | Src2ES, em_push_sreg),I (ImplicitOps | Stack | No64 | Src2ES, em_pop_sreg),/* 0x08 - 0x0F */ -F6ALU (Lock | PageTable, em_or), +I6ALU (Lock | PageTable, em_or),I (ImplicitOps | Stack | No64 | Src2CS, em_push_sreg),N ,/* 0x10 - 0x17 */ -F6ALU (Lock, em_adc), +I6ALU (Lock, em_adc),I (ImplicitOps | Stack | No64 | Src2SS, em_push_sreg),I (ImplicitOps | Stack | No64 | Src2SS, em_pop_sreg),/* 0x18 - 0x1F */ -F6ALU (Lock, em_sbb), +I6ALU (Lock, em_sbb),I (ImplicitOps | Stack | No64 | Src2DS, em_push_sreg),I (ImplicitOps | Stack | No64 | Src2DS, em_pop_sreg),/* 0x20 - 0x27 */ -F6ALU (Lock | PageTable, em_and),N ,N , +I6ALU (Lock | PageTable, em_and),N ,N ,/* 0x28 - 0x2F */ -F6ALU (Lock, em_sub),N ,I (ByteOp | DstAcc | No64, em_das), +I6ALU (Lock, em_sub),N ,I (ByteOp | DstAcc | No64, em_das),/* 0x30 - 0x37 */ -F6ALU (Lock, em_xor),N ,N , +I6ALU (Lock, em_xor),N ,N ,/* 0x38 - 0x3F */ -F6ALU (NoWrite, em_cmp),N ,N , +I6ALU (NoWrite, em_cmp),N ,N ,/* 0x40 - 0x4F */ X8 (I (DstReg, em_inc)),X8 (I (DstReg, em_dec)),/* 0x50 - 0x57 */
@@ -4306,7 +4315,7 @@ static const struct opcode opcode_table[256] = {G (DstMem | SrcImm, group1),G (ByteOp | DstMem | SrcImm | No64, group1),G (DstMem | SrcImmByte, group1), -F2bv (DstMem | SrcReg | ModRM | NoWrite, em_test), +I2bv (DstMem | SrcReg | ModRM | NoWrite, em_test),I2bv (DstMem | SrcReg | ModRM | Lock | PageTable, em_xchg),/* 0x88 - 0x8F */ I2bv(DstMem | SrcReg | ModRM | Mov | PageTable, em_mov),
@@ -4329,7 +4338,7 @@ static const struct opcode opcode_table[256] = {I2bv (SrcSI | DstDI | Mov | String | TwoMemOp, em_mov),F2bv (SrcSI | DstDI | String | NoWrite | TwoMemOp, em_cmp_r),/* 0xA8 - 0xAF */ -F2bv (DstAcc | SrcImm | NoWrite, em_test), +I2bv (DstAcc | SrcImm | NoWrite, em_test),I2bv (SrcAcc | DstDI | Mov | String, em_mov),I2bv (SrcSI | DstAcc | Mov | String, em_mov), F2bv(SrcAcc | DstDI | String | NoWrite, em_cmp_r),
@@ -4467,7 +4476,7 @@ static const struct opcode twobyte_table[256] = {I (DstReg | SrcMem | ModRM, em_bsr_c),D (DstReg | SrcMem8 | ModRM | Mov),D (DstReg | SrcMem16 | ModRM | Mov),/* 0xC0 - 0xC7 */ -F2bv (DstMem | SrcReg | ModRM | SrcWrite | Lock, em_xadd), +I2bv (DstMem | SrcReg | ModRM | SrcWrite | Lock, em_xadd),N ,ID (0 , &instr_dual_0f_c3), N, N, N, GD(0, &group9),/* 0xC8 - 0xCF */
[tip: x86/core] KVM: x86: Introduce EM_ASM_1
In reply to Peter ZijlstraThe following commit has been merged into the x86/core branch of tip:
The following commit has been merged into the x86/core branch of tip:
Commit-ID: 6204aea36b74cd2001a142e92e492e301597eafb Gitweb: https://git.kernel.org/tip/6204aea36b74cd2001a142e92e492e301597eafb Author: Peter Zijlstra <[email protected]> AuthorDate: Sat, 26 Apr 2025 11:51:49 +02:00 Committer: Peter Zijlstra <[email protected]> CommitterDate: Mon, 18 Aug 2025 14:23:05 +02:00
KVM: x86: Introduce EM_ASM_1
Replace fastops with C based stubs. There are a bunch of problems with the current fastop infrastructure, most all related to their special calling convention, which bypasses the normal C-ABI.
There are two immediate problems with this at present:
- it relies on RET preserving EFLAGS; whereas C-ABI does not.
- it circumvents compiler based control-flow-integrity checking because its all asm magic.
The first is a problem for some mitigations where the x86_indirect_return_thunk needs to include non-trivial work that clobbers EFLAGS (eg. the Skylake call depth tracking thing).
The second is a problem because it presents a 'naked' indirect call on kCFI builds, making it a prime target for control flow hijacking.
Additionally, given that a large chunk of virtual machine performance relies on absolutely avoiding vmexit these days, this emulation stuff just isn't that critical for performance anymore.
As such, replace the fastop calls with normal C functions using the 'execute' member.
As noted by Paolo: this code was performance critical for pre-Westmere (2010) and only when running big real mode code.
--- arch/x86/kvm/emulate.c | 71 +++++++++++++++++++++++++++++++++-------- 1 file changed, 58 insertions(+), 13 deletions(-)
arch/x86/kvm/emulate.c +58 -13
@@ -267,11 +267,56 @@ static void invalidate_registers(struct x86_emulate_ctxt *ctxt) X86_EFLAGS_PF|X86_EFLAGS_CF)#ifdef CONFIG_X86_64 -#define ON64 (x) x +#define ON64(x...) x#else -#define ON64 (x) +#define ON64(x...)#endif +#define EM_ASM_START (op) \ +static int em_##op(struct x86_emulate_ctxt *ctxt) \ +{ \ + unsigned long flags = (ctxt->eflags & EFLAGS_MASK) | X86_EFLAGS_IF; \ + int bytes = 1, ok = 1; \ + if (!(ctxt->d & ByteOp)) \ + bytes = ctxt->dst.bytes; \ + switch (bytes) { + +#define __EM_ASM (str) \ + asm("push %[flags]; popf \n\t" \ + "10: " str \ + "pushf; pop %[flags] \n\t" \ + "11: \n\t" \ + : "+a" (ctxt->dst.val), \ + "+d" (ctxt->src.val), \ + [flags] "+D" (flags), \ + "+S" (ok) \ + : "c" (ctxt->src2.val)) + +#define __EM_ASM_1 (op, dst) \ + __EM_ASM(#op " %%" #dst " \n\t") + +#define __EM_ASM_1_EX (op, dst) \ + __EM_ASM(#op " %%" #dst " \n\t" \ + _ASM_EXTABLE_TYPE_REG(10b, 11f, EX_TYPE_ZERO_REG, %%esi)) + +#define __EM_ASM_2 (op, dst, src) \ + __EM_ASM(#op " %%" #src ", %%" #dst " \n\t") + +#define EM_ASM_END \ + } \ + ctxt->eflags = (ctxt->eflags & ~EFLAGS_MASK) | (flags & EFLAGS_MASK); \ + return !ok ? emulate_de(ctxt) : X86EMUL_CONTINUE; \ +} + +/* 1-operand, using "a" (dst) */ +#define EM_ASM_1 (op) \ + EM_ASM_START(op) \ + case 1: __EM_ASM_1(op##b, al); break; \ + case 2: __EM_ASM_1(op##w, ax); break; \ + case 4: __EM_ASM_1(op##l, eax); break; \ + ON64(case 8: __EM_ASM_1(op##q, rax); break;) \ + EM_ASM_END +/* * fastop functions have a special calling convention: *
@@ -1002,10 +1047,10 @@ FASTOP3WCL(shrd);FASTOP2W (imul); -FASTOP1 (not); -FASTOP1 (neg); -FASTOP1 (inc); -FASTOP1 (dec); +EM_ASM_1 (not); +EM_ASM_1 (neg); +EM_ASM_1 (inc); +EM_ASM_1 (dec);FASTOP2CL (rol);FASTOP2CL (ror);
@@ -4021,8 +4066,8 @@ static const struct opcode group2[] = {static const struct opcode group3[] = {F (DstMem | SrcImm | NoWrite, em_test),F (DstMem | SrcImm | NoWrite, em_test), -F (DstMem | SrcNone | Lock, em_not), -F (DstMem | SrcNone | Lock, em_neg), +I (DstMem | SrcNone | Lock, em_not), +I (DstMem | SrcNone | Lock, em_neg),F (DstXacc | Src2Mem, em_mul_ex),F (DstXacc | Src2Mem, em_imul_ex),F (DstXacc | Src2Mem, em_div_ex),
@@ -4030,14 +4075,14 @@ static const struct opcode group3[] = { };static const struct opcode group4[] = { -F (ByteOp | DstMem | SrcNone | Lock, em_inc), -F (ByteOp | DstMem | SrcNone | Lock, em_dec), +I (ByteOp | DstMem | SrcNone | Lock, em_inc), +I (ByteOp | DstMem | SrcNone | Lock, em_dec),N ,N ,N ,N ,N ,N , };static const struct opcode group5[] = { -F (DstMem | SrcNone | Lock, em_inc), -F (DstMem | SrcNone | Lock, em_dec), +I (DstMem | SrcNone | Lock, em_inc), +I (DstMem | SrcNone | Lock, em_dec),I (SrcMem | NearBranch | IsBranch, em_call_near_abs),I (SrcMemFAddr | ImplicitOps | IsBranch, em_call_far),I (SrcMem | NearBranch | IsBranch, em_jmp_abs),
@@ -4237,7 +4282,7 @@ static const struct opcode opcode_table[256] = {/* 0x38 - 0x3F */ F6ALU (NoWrite, em_cmp),N ,N ,/* 0x40 - 0x4F */ -X8 (F (DstReg, em_inc)),X8 (F (DstReg, em_dec)), +X8 (I (DstReg, em_inc)),X8 (I (DstReg, em_dec)),/* 0x50 - 0x57 */ X8(I(SrcReg | Stack, em_push)),/* 0x58 - 0x5F */
[tip: x86/core] KVM: x86: Implement test_cc() in C
In reply to Peter ZijlstraThe following commit has been merged into the x86/core branch of tip:
The following commit has been merged into the x86/core branch of tip:
Commit-ID: 0cb6f1e436accba7882bb3115408d1474c1e14af Gitweb: https://git.kernel.org/tip/0cb6f1e436accba7882bb3115408d1474c1e14af Author: Peter Zijlstra <[email protected]> AuthorDate: Fri, 01 Dec 2023 18:53:40 +01:00 Committer: Peter Zijlstra <[email protected]> CommitterDate: Mon, 18 Aug 2025 14:23:04 +02:00
KVM: x86: Implement test_cc() in C
Current test_cc() uses the fastop infrastructure to test flags using SETcc instructions. However, int3_emulate_jcc() already fully implements the flags->CC mapping, use that.
Removes a pile of gnarly asm.
--- arch/x86/include/asm/text-patching.h | 20 ++++++++++------ arch/x86/kvm/emulate.c | 34 +-------------------------- 2 files changed, 15 insertions(+), 39 deletions(-)
arch/x86/include/asm/text-patching.h +13 -7
@@ -178,9 +178,9 @@ void int3_emulate_ret(struct pt_regs *regs) }static __always_inline -void int3_emulate_jcc(struct pt_regs *regs, u8 cc, unsigned long ip, unsigned long disp) +bool __emulate_cc(unsigned long flags, u8 cc) { -static const unsigned long jcc_mask[6 ] = { +static const unsigned long cc_mask[6 ] = { [0 ] =X86_EFLAGS_OF , [1 ] =X86_EFLAGS_CF , [2 ] =X86_EFLAGS_ZF ,
@@ -193,15 +193,21 @@ void int3_emulate_jcc(struct pt_regs *regs, u8 cc, unsigned long ip, unsigned lobool match;if (cc <0xc ) { - match = regs->flags & jcc_mask[cc >>1 ]; + match = flags & cc_mask[cc >>1 ]; }else { - match = ((regs->flags &X86_EFLAGS_SF ) >>X86_EFLAGS_SF_BIT ) ^ - ((regs->flags &X86_EFLAGS_OF ) >>X86_EFLAGS_OF_BIT ); + match = ((flags &X86_EFLAGS_SF ) >>X86_EFLAGS_SF_BIT ) ^ + ((flags &X86_EFLAGS_OF ) >>X86_EFLAGS_OF_BIT );if (cc >=0xe ) - match = match || (regs->flags &X86_EFLAGS_ZF ); + match = match || (flags &X86_EFLAGS_ZF ); } -if ((match && !invert) || (!match && invert)) +return (match && !invert) || (!match && invert); +} + +static __always_inline +void int3_emulate_jcc(struct pt_regs *regs, u8 cc, unsigned long ip, unsigned long disp) +{ +if (__emulate_cc (regs->flags, cc)) ip += disp;int3_emulate_jmp (regs, ip);
arch/x86/kvm/emulate.c +2 -32
@@ -26,6 +26,7 @@#include <asm/debugreg.h> #include <asm/nospec-branch.h> #include <asm/ibt.h> +#include <asm/text-patching.h> #include "x86.h" #include "tss.h"
@@ -416,31 +417,6 @@ static int fastop(struct x86_emulate_ctxt *ctxt, fastop_t fop); ON64(FOP3E(op##q, rax, rdx, cl)) \ FOP_END -/* Special case for SETcc - 1 instruction per cc */ -#define FOP_SETCC (op) \ - FOP_FUNC(op) \ - #op " %al \n\t" \ - FOP_RET(op) - -FOP_START(setcc) -FOP_SETCC(seto) -FOP_SETCC(setno) -FOP_SETCC(setc) -FOP_SETCC(setnc) -FOP_SETCC(setz) -FOP_SETCC(setnz) -FOP_SETCC(setbe) -FOP_SETCC(setnbe) -FOP_SETCC(sets) -FOP_SETCC(setns) -FOP_SETCC(setp) -FOP_SETCC(setnp) -FOP_SETCC(setl) -FOP_SETCC(setnl) -FOP_SETCC(setle) -FOP_SETCC(setnle) -FOP_END; - FOP_START(salc) FOP_FUNC(salc)"pushf; sbb %al, %al; popf \n\t"
@@ -1068,13 +1044,7 @@ static int em_bsr_c(struct x86_emulate_ctxt *ctxt) static __always_inline u8 test_cc(unsigned int condition, unsigned long flags) { -u8 rc; -void (*fop)(void ) = (void *)em_setcc +FASTOP_SIZE * (condition &0xf ); - - flags = (flags &EFLAGS_MASK ) |X86_EFLAGS_IF ; - asm("push %[flags]; popf; " CALL_NOSPEC - : "=a"(rc), ASM_CALL_CONSTRAINT : [thunk_target]"r"(fop), [flags]"r"(flags)); -return rc; +return __emulate_cc (flags, condition &0xf ); }static void fetch_register_operand (struct operand *op)