x86: re-inline indirect CALL/JMP
PATCH 1/1 x86: re-inline indirect CALL/JMP
Using the same thunk for heterogeneous calls (i.e. ones to functions with entirely different parameter types) is a problem: Speculation may e.g. result in use as pointers for arguments which are are actually integers. This is a result of target prediction information being tied to the address of the RET instruction in each thunk (of which we've got only 15, and of which in turn a fair share of the call sites use the single one where %rax is holding the target address).
Except when actually using the full retpoline, arrange for alternatives patching to put the JMP and LFENCE forms of the thunk back inline.
---
I'm sure I didn't get the reasoning right / complete; I'd appreciate clear enough indication of what needs adding/changing.
Some of the changes to may not strictly be necessary:
- INDIRECT_BRANCH: We don't have any uses left in assembly code.
- GEN_INDIRECT_THUNK: Continuing to patch the thunks when they're not used is merely pointless. The change, however, is more in anticipation that X86_FEATURE_IND_THUNK_{LFENCE,JMP} may not be fully suitable names anymore when the code gets inlined (at least I probably wouldn't call such "thunk" anymore).
While perhaps not a big problem, I'd like to point out that this more than doubles the size of the .altinstr_replacement section (with an accordingly large increase of .altinstructions), unless we were to make use of "x86/alternatives: allow replacement code snippets to be merged" (which of course affects only .altinstr_replacement).
xen/Makefile +1 -1
@@ -195,7 +195,7 @@ t2 = $(call as-insn,$(CC) -I$(BASEDIR)/a # https://bugs.llvm.org/show_bug.cgi?id=36110 t3 = $(call as-insn,$(CC),".macro FOO;.endm"$(close); asm volatile $(open)".macro FOO;.endm",-no-integrated-as) -CLANG_FLAGS += $(call or,$(t1),$(t2),$(t3)) +CLANG_FLAGS += $(call or,$(t1),$(t2),$(t3),-DCLANG_INTEGRATED_AS) endif CLANG_FLAGS += -Werror=unknown-warning-option
xen/arch/x86/Makefile +1 -1
@@ -240,7 +240,7 @@ $(obj)/efi/buildid.o $(obj)/efi/relocs-d .PHONY: include include: $(BASEDIR)/arch/x86/include/asm/asm-macros.h -$(obj)/asm-macros.i: CFLAGS-y += -D__ASSEMBLY__ -P +$(obj)/asm-macros.i: CFLAGS-y += -D__ASSEMBLY__ -D__ASM_MACROS__ -P $(BASEDIR)/arch/x86/include/asm/asm-macros.h: $(obj)/asm-macros.i $(src)/Makefile $(call filechk,asm-macros.h)
xen/arch/x86/alternative.c +17 -4
@@ -244,11 +244,19 @@ static void init_or_livepatch _apply_alt a->priv =1 ; -/* Nothing useful to do? */ -if ( toolchain_nops_are_ideal || a->pad_len <=1 ) +/* No padding at all? */ +if ( !a->pad_len )continue ; -add_nops (buf, a->pad_len); +/* For a JMP to an indirect thunk, replace NOP by INT3. */ +if (IS_ENABLED (CONFIG_SPECULATIVE_HARDEN_BRANCH ) && + a->cpuid ==X86_FEATURE_IND_THUNK_JMP && orig[0 ] ==0xe9 ) +memset (buf,0xcc , a->pad_len); +/* Nothing useful to do? */ +else if ( toolchain_nops_are_ideal || a->pad_len <=1 ) +continue ; +else +add_nops (buf, a->pad_len);text_poke (orig + a->orig_len, buf, a->pad_len);continue ; }
@@ -330,7 +338,12 @@ static void init_or_livepatch _apply_alt a->priv =1 ; -add_nops (buf + a->repl_len, total_len - a->repl_len); +/* When re-inlining an indirect JMP, pad it by INT3, not NOPs. */ +if (IS_ENABLED (CONFIG_SPECULATIVE_HARDEN_BRANCH ) && + a->cpuid ==X86_FEATURE_IND_THUNK_JMP && orig[0 ] ==0xe9 ) +memset (buf + a->repl_len,0xcc , total_len - a->repl_len); +else +add_nops (buf + a->repl_len, total_len - a->repl_len);text_poke (orig, buf, total_len); }
xen/arch/x86/asm-macros.c +1 -0
@@ -1,2 +1,3 @@#include <asm/asm-defns.h> #include <asm/alternative-asm.h> +#include <asm/spec_ctrl_asm.h>
xen/arch/x86/extable.c +35 -5
@@ -98,12 +98,42 @@ search_exception_table(const struct cpu_ regs->rsp > (unsigned long)regs && regs->rsp < (unsigned long)get_cpu_info() ) { -unsigned long retptr = *(unsigned long *)regs->rsp; +unsigned long retaddr = *(unsigned long *)regs->rsp; +unsigned long retptr =0 ; +unsigned int pad =0 ; - region =find_text_region (retptr); - retptr = region && region->ex - ?search_one_extable (region->ex, region->ex_end -1 , retptr) - :0 ; + region =find_text_region (retaddr); +while ( region && region->ex ) + { + retptr =search_one_extable (region->ex, region->ex_end -1 , + retaddr + pad); +if ( !retptr ) + { +/* +* Indirect call thunk patching can lead to NOP padding between +* the CALL and the recovery entry recorded in .fixup. Skip +* past such NOPs. See asm/nops.h for the forms possible and +* note that no more than 3 bytes of padding will be present. +*/ +const uint8_t *ptr = (void *)retaddr + pad; + +switch ( ptr[0 ] ) + { +case 0x66 : +case 0x90 : +if ( ++pad >3 ) +break ; +continue ; + +case 0x0f : +if ( pad || ptr[1 ] !=0x1f || ptr[2 ] !=0x00 ) +break ; + pad =3 ; +continue ; + } + } +break ; + }if ( retptr ) {/*
xen/arch/x86/include/asm/asm-defns.h +6 -0
@@ -33,7 +33,13 @@ $done =0 .irp reg, ax, cx, dx, bx, bp, si, di, 8, 9, 10, 11, 12, 13, 14, 15 .ifeqs "\arg", "%r\reg" +#ifdef __ASM_MACROS__ \insn __x86_indirect_thunk_r\reg +#else + ALTERNATIVE_2 "\insn __x86_indirect_thunk_r\reg", \ + "lfence; \insn *\arg", X86_FEATURE_IND_THUNK_LFENCE, \ + "\insn *\arg", X86_FEATURE_IND_THUNK_JMP +#endif $done =1 .exitm .endif
xen/arch/x86/include/asm/spec_ctrl_asm.h +45 -2
59 lines not on this page · raw message
xen/arch/x86/indirect-thunk.S +4 -0
@@ -38,9 +38,13 @@ .section .text.__x86_indirect_thunk_\reg, "ax", @progbits ENTRY(__x86_indirect_thunk_\reg) +#ifdef CLANG_INTEGRATED_AS ALTERNATIVE_2 __stringify(IND_THUNK_RETPOLINE \reg), \ __stringify(IND_THUNK_LFENCE \reg), X86_FEATURE_IND_THUNK_LFENCE, \ __stringify(IND_THUNK_JMP \reg), X86_FEATURE_IND_THUNK_JMP +#else + IND_THUNK_RETPOLINE \reg +#endif int3 /* Halt straight-line speculation */