Skip to contentSkip to conversation

x86/bugs: BHI fixes / improvements - round 2

PATCH Applied Josh Poimboeuf 6 participants 16 messages Apr 12, 2024 – Apr 15, 2024

Subsystems (6)
Documentation x86 BPF x86 entry code +2 more subsystems
lore
Share & tools
lore

Josh Poimboeuf

author maintainer
#0 lore raw

cover x86/bugs: BHI fixes / improvements - round 2

BHI fixes round 2:

- An updated version of "Only harden syscalls when needed" with review comments addressed

- A BHI retpoline check fix

- Remove the obsolete LFENCE "retpolines"

Josh Poimboeuf (3):
x86/bugs: Only harden syscalls when needed x86/bugs: Fix BHI retpoline check
x86/bugs: Remove support for Spectre v2 LFENCE "retpolines"

 arch/x86/Makefile                             |  1 -
 arch/x86/entry/common.c                       | 15 +++-
 arch/x86/entry/syscall_32.c                   | 11 +--
 arch/x86/entry/syscall_64.c                   |  6 --
 arch/x86/entry/syscall_x32.c                  |  7 +-
 arch/x86/include/asm/cpufeatures.h            |  2 +-
 arch/x86/include/asm/disabled-features.h      |  3 +-
 arch/x86/include/asm/nospec-branch.h          | 18 +---
 arch/x86/include/asm/syscall.h                |  8 +-
 arch/x86/kernel/alternative.c                 | 17 +---
 arch/x86/kernel/cpu/bugs.c                    | 88 +++++++------------
 arch/x86/kernel/cpu/cpu.h                     |  3 +-
 arch/x86/lib/retpoline.S                      |  5 +-
 arch/x86/net/bpf_jit_comp.c                   |  5 +-
 tools/arch/x86/include/asm/cpufeatures.h      |  1 -
 .../arch/x86/include/asm/disabled-features.h  |  3 +-
 16 files changed, 69 insertions(+), 124 deletions(-)
Signature · 1 line
2.44.0
author maintainer
#1 lore raw

PATCH v2 1/3 x86/bugs: Only harden syscalls when needed

Syscall hardening (i.e., converting the syscall indirect branch to a series of direct branches) may cause performance regressions in certain scenarios. Only use the syscall hardening when indirect branches are considered unsafe.

Fixes 1e3ad78334a6 ("x86/syscall: Don't force use of indirect calls for system calls")
---
 arch/x86/entry/common.c            | 15 ++++++++++++---
 arch/x86/entry/syscall_32.c        | 11 +----------
 arch/x86/entry/syscall_64.c        |  6 ------
 arch/x86/entry/syscall_x32.c       |  7 ++++++-
 arch/x86/include/asm/cpufeatures.h |  1 +
 arch/x86/include/asm/syscall.h     |  8 +++++++-
 arch/x86/kernel/cpu/bugs.c         | 31 +++++++++++++++++++++++++++++-
 7 files changed, 57 insertions(+), 22 deletions(-)
arch/x86/entry/common.c +12 -3
index 6de50b80702e..c0f8116291e4 100644
@@ -49,7 +49,10 @@ static __always_inline bool do_syscall_x64(struct pt_regs *regs, int nr)
 
 	if (likely(unr < NR_syscalls)) {
 		unr = array_index_nospec(unr, NR_syscalls);
-		regs->ax = x64_sys_call(regs, unr);
+		if (likely(cpu_feature_enabled(X86_FEATURE_INDIRECT_SAFE)))
+			regs->ax = sys_call_table[unr](regs);
+		else
+			regs->ax = x64_sys_call(regs, unr);
 		return true;
 	}
 	return false;
@@ -66,7 +69,10 @@ static __always_inline bool do_syscall_x32(struct pt_regs *regs, int nr)
 
 	if (IS_ENABLED(CONFIG_X86_X32_ABI) && likely(xnr < X32_NR_syscalls)) {
 		xnr = array_index_nospec(xnr, X32_NR_syscalls);
-		regs->ax = x32_sys_call(regs, xnr);
+		if (likely(cpu_feature_enabled(X86_FEATURE_INDIRECT_SAFE)))
+			regs->ax = x32_sys_call_table[xnr](regs);
+		else
+			regs->ax = x32_sys_call(regs, xnr);
 		return true;
 	}
 	return false;
@@ -162,7 +168,10 @@ static __always_inline void do_syscall_32_irqs_on(struct pt_regs *regs, int nr)
 
 	if (likely(unr < IA32_NR_syscalls)) {
 		unr = array_index_nospec(unr, IA32_NR_syscalls);
-		regs->ax = ia32_sys_call(regs, unr);
+		if (likely(cpu_feature_enabled(X86_FEATURE_INDIRECT_SAFE)))
+			regs->ax = ia32_sys_call_table[unr](regs);
+		else
+			regs->ax = ia32_sys_call(regs, unr);
 	} else if (nr != -1) {
 		regs->ax = __ia32_sys_ni_syscall(regs);
 	}
arch/x86/entry/syscall_32.c +1 -10
index c2235bae17ef..aab31760b4e3 100644
@@ -14,25 +14,16 @@
 #endif
 
 #define __SYSCALL(nr, sym) extern long __ia32_##sym(const struct pt_regs *);
-
 #include <asm/syscalls_32.h>
 #undef __SYSCALL
 
-/*
- * The sys_call_table[] is no longer used for system calls, but
- * kernel/trace/trace_syscalls.c still wants to know the system
- * call address.
- */
-#ifdef CONFIG_X86_32
 #define __SYSCALL(nr, sym) __ia32_##sym,
-const sys_call_ptr_t sys_call_table[] = {
+const sys_call_ptr_t ia32_sys_call_table[] = {
 #include <asm/syscalls_32.h>
 };
 #undef __SYSCALL
-#endif
 
 #define __SYSCALL(nr, sym) case nr: return __ia32_##sym(regs);
-
 long ia32_sys_call(const struct pt_regs *regs, unsigned int nr)
 {
 	switch (nr) {
arch/x86/entry/syscall_64.c +0 -6
index 33b3f09e6f15..96ea1f8a1d3f 100644
@@ -11,11 +11,6 @@
 #include <asm/syscalls_64.h>
 #undef __SYSCALL
 
-/*
- * The sys_call_table[] is no longer used for system calls, but
- * kernel/trace/trace_syscalls.c still wants to know the system
- * call address.
- */
 #define __SYSCALL(nr, sym) __x64_##sym,
 const sys_call_ptr_t sys_call_table[] = {
 #include <asm/syscalls_64.h>
@@ -23,7 +18,6 @@ const sys_call_ptr_t sys_call_table[] = {
 #undef __SYSCALL
 
 #define __SYSCALL(nr, sym) case nr: return __x64_##sym(regs);
-
 long x64_sys_call(const struct pt_regs *regs, unsigned int nr)
 {
 	switch (nr) {
arch/x86/entry/syscall_x32.c +6 -1
index 03de4a932131..5aef4230faca 100644
@@ -11,8 +11,13 @@
 #include <asm/syscalls_x32.h>
 #undef __SYSCALL
 
-#define __SYSCALL(nr, sym) case nr: return __x64_##sym(regs);
+#define __SYSCALL(nr, sym) __x64_##sym,
+const sys_call_ptr_t x32_sys_call_table[] = {
+#include <asm/syscalls_x32.h>
+};
+#undef __SYSCALL
 
+#define __SYSCALL(nr, sym) case nr: return __x64_##sym(regs);
 long x32_sys_call(const struct pt_regs *regs, unsigned int nr)
 {
 	switch (nr) {
arch/x86/include/asm/cpufeatures.h +1 -0
index 3c7434329661..7c87fe80c696 100644
@@ -470,6 +470,7 @@
 #define X86_FEATURE_BHI_CTRL		(21*32+ 2) /* "" BHI_DIS_S HW control available */
 #define X86_FEATURE_CLEAR_BHB_HW	(21*32+ 3) /* "" BHI_DIS_S HW control enabled */
 #define X86_FEATURE_CLEAR_BHB_LOOP_ON_VMEXIT (21*32+ 4) /* "" Clear branch history at vmexit using SW loop */
+#define X86_FEATURE_INDIRECT_SAFE	(21*32+ 4) /* "" Indirect branches aren't vulnerable to Spectre v2 */
 
 /*
  * BUG word(s)
arch/x86/include/asm/syscall.h +7 -1
index 2fc7bc3863ff..dfb59521244c 100644
@@ -16,14 +16,20 @@
 #include <asm/thread_info.h>	/* for TS_COMPAT */
 #include <asm/unistd.h>
 
-/* This is used purely for kernel/trace/trace_syscalls.c */
 typedef long (*sys_call_ptr_t)(const struct pt_regs *);
 extern const sys_call_ptr_t sys_call_table[];
 
+#if defined(CONFIG_X86_32)
+#define ia32_sys_call_table sys_call_table
+#else
 /*
  * These may not exist, but still put the prototypes in so we
  * can use IS_ENABLED().
  */
+extern const sys_call_ptr_t ia32_sys_call_table[];
+extern const sys_call_ptr_t x32_sys_call_table[];
+#endif
+
 extern long ia32_sys_call(const struct pt_regs *, unsigned int nr);
 extern long x32_sys_call(const struct pt_regs *, unsigned int nr);
 extern long x64_sys_call(const struct pt_regs *, unsigned int nr);
arch/x86/kernel/cpu/bugs.c +30 -1
index ca295b0c1eee..dcb97cc2758f 100644
@@ -1664,6 +1664,15 @@ static void __init bhi_select_mitigation(void)
 	if (!IS_ENABLED(CONFIG_X86_64))
 		return;
 
+	/*
+	 * There's no hardware mitigation in place, so mark indirect branches
+	 * as unsafe.
+	 *
+	 * One could argue the SW loop makes indirect branches safe again, but
+	 * Linus prefers it this way.
+	 */
+	setup_clear_cpu_cap(X86_FEATURE_INDIRECT_SAFE);
+
 	/* Mitigate KVM by default */
 	setup_force_cpu_cap(X86_FEATURE_CLEAR_BHB_LOOP_ON_VMEXIT);
 	pr_info("Spectre BHI mitigation: SW BHB clearing on vm exit\n");
@@ -1678,6 +1687,21 @@ static void __init spectre_v2_select_mitigation(void)
 	enum spectre_v2_mitigation_cmd cmd = spectre_v2_parse_cmdline();
 	enum spectre_v2_mitigation mode = SPECTRE_V2_NONE;
 
+	/*
+	 * X86_FEATURE_INDIRECT_SAFE indicates whether indirect calls can be
+	 * considered safe.  That means either:
+	 *
+	 *   - the CPU isn't vulnerable to Spectre v2 or its variants;
+	 *
+	 *   - a hardware mitigation is in place (e.g., IBRS, BHI_DIS_S); or
+	 *
+	 *   - the user turned off mitigations altogether.
+	 *
+	 * Assume innocence until proven guilty: set the cap bit now, then
+	 * clear it later if/when needed.
+	 */
+	setup_force_cpu_cap(X86_FEATURE_INDIRECT_SAFE);
+
 	/*
 	 * If the CPU is not affected and the command line mode is NONE or AUTO
 	 * then nothing to do.
@@ -1764,11 +1788,16 @@ static void __init spectre_v2_select_mitigation(void)
 		break;
 
 	case SPECTRE_V2_LFENCE:
+		setup_clear_cpu_cap(X86_FEATURE_INDIRECT_SAFE);
+		fallthrough;
 	case SPECTRE_V2_EIBRS_LFENCE:
 		setup_force_cpu_cap(X86_FEATURE_RETPOLINE_LFENCE);
-		fallthrough;
+		setup_force_cpu_cap(X86_FEATURE_RETPOLINE);
+		break;
 
 	case SPECTRE_V2_RETPOLINE:
+		setup_clear_cpu_cap(X86_FEATURE_INDIRECT_SAFE);
+		fallthrough;
 	case SPECTRE_V2_EIBRS_RETPOLINE:
 		setup_force_cpu_cap(X86_FEATURE_RETPOLINE);
 		break;
Signature · 1 line
2.44.0
author maintainer
#8 lore raw

PATCH 2/3 x86/bugs: Fix BHI retpoline check

In reply to Josh Poimboeuf

Confusingly, X86_FEATURE_RETPOLINE doesn't mean retpolines are enabled, as it also includes the original "AMD retpoline" which isn't a retpoline at all.

Also replace cpu_feature_enabled() with boot_cpu_has() because this is before alternatives are patched and cpu_feature_enabled()'s fallback path is slower than plain old boot_cpu_has().

Fixes ec9404e40e8f ("x86/bhi: Add BHI mitigation knob")
---
 arch/x86/kernel/cpu/bugs.c | 11 +++++++----
 1 file changed, 7 insertions(+), 4 deletions(-)
arch/x86/kernel/cpu/bugs.c +7 -4
index dcb97cc2758f..e827613c9e39 100644
@@ -1652,7 +1652,8 @@ static void __init bhi_select_mitigation(void)
 		return;
 
 	/* Retpoline mitigates against BHI unless the CPU has RRSBA behavior */
-	if (cpu_feature_enabled(X86_FEATURE_RETPOLINE)) {
+	if (boot_cpu_has(X86_FEATURE_RETPOLINE) &&
+	    !boot_cpu_has(X86_FEATURE_RETPOLINE_LFENCE)) {
 		spec_ctrl_disable_kernel_rrsba();
 		if (rrsba_disabled)
 			return;
@@ -2833,11 +2834,13 @@ static const char *spectre_bhi_state(void)
 {
 	if (!boot_cpu_has_bug(X86_BUG_BHI))
 		return "; BHI: Not affected";
-	else if  (boot_cpu_has(X86_FEATURE_CLEAR_BHB_HW))
+	else if (boot_cpu_has(X86_FEATURE_CLEAR_BHB_HW))
 		return "; BHI: BHI_DIS_S";
-	else if  (boot_cpu_has(X86_FEATURE_CLEAR_BHB_LOOP))
+	else if (boot_cpu_has(X86_FEATURE_CLEAR_BHB_LOOP))
 		return "; BHI: SW loop, KVM: SW loop";
-	else if (boot_cpu_has(X86_FEATURE_RETPOLINE) && rrsba_disabled)
+	else if (boot_cpu_has(X86_FEATURE_RETPOLINE) &&
+		 !boot_cpu_has(X86_FEATURE_RETPOLINE_LFENCE) &&
+		 rrsba_disabled)
 		return "; BHI: Retpoline";
 	else if (boot_cpu_has(X86_FEATURE_CLEAR_BHB_LOOP_ON_VMEXIT))
 		return "; BHI: Vulnerable, KVM: SW loop";
Signature · 1 line
2.44.0
author maintainer
#11 lore raw

PATCH 3/3 x86/bugs: Remove support for Spectre v2 LFENCE "retpolines"

In reply to Josh Poimboeuf

I found several bugs where code assumes that X86_FEATURE_RETPOLINE actually means retpolines (imagine that!). In fact that feature also includes the original AMD LFENCE "retpolines", which aren't in fact retpolines.

Really, those "retpolines" should just be removed. They're already considered vulnerable due to the fact that the speculative window after the indirect branch can still be long enough to do multiple dependent loads. And recent tooling makes such gadgets easier to find.

Also, EIBRS_LFENCE tests worse in real-world benchmarks than the actual BHI mitigations, so it's both slower and less secure.

Specifically this removes support for the following cmdline options:

- spectre_v2=retpoline,amd
- spectre_v2=retpoline,lfence
- spectre_v2=eibrs,lfence

Now when any of those options are used, it will print an error and fall back to the defaults (spectre_v2=auto spectre_bhi=on).

---
 arch/x86/Makefile                             |  1 -
 arch/x86/include/asm/cpufeatures.h            |  1 -
 arch/x86/include/asm/disabled-features.h      |  3 +-
 arch/x86/include/asm/nospec-branch.h          | 18 ++---
 arch/x86/kernel/alternative.c                 | 17 +----
 arch/x86/kernel/cpu/bugs.c                    | 66 +------------------
 arch/x86/kernel/cpu/cpu.h                     |  3 +-
 arch/x86/lib/retpoline.S                      |  5 +-
 arch/x86/net/bpf_jit_comp.c                   |  5 +-
 tools/arch/x86/include/asm/cpufeatures.h      |  1 -
 .../arch/x86/include/asm/disabled-features.h  |  3 +-
 11 files changed, 15 insertions(+), 108 deletions(-)
arch/x86/Makefile +0 -1
index 5ab93fcdd691..f0f91810a79e 100644
@@ -20,7 +20,6 @@ ifdef CONFIG_CC_IS_CLANG
 RETPOLINE_CFLAGS	:= -mretpoline-external-thunk
 RETPOLINE_VDSO_CFLAGS	:= -mretpoline
 endif
-RETPOLINE_CFLAGS	+= $(call cc-option,-mindirect-branch-cs-prefix)
 
 ifdef CONFIG_MITIGATION_RETHUNK
 RETHUNK_CFLAGS		:= -mfunction-return=thunk-extern
arch/x86/include/asm/cpufeatures.h +0 -1
index 7c87fe80c696..fccc838ac8ff 100644
@@ -296,7 +296,6 @@
 #define X86_FEATURE_ENTRY_IBPB		(11*32+10) /* "" Issue an IBPB on kernel entry */
 #define X86_FEATURE_RRSBA_CTRL		(11*32+11) /* "" RET prediction control */
 #define X86_FEATURE_RETPOLINE		(11*32+12) /* "" Generic Retpoline mitigation for Spectre variant 2 */
-#define X86_FEATURE_RETPOLINE_LFENCE	(11*32+13) /* "" Use LFENCE for Spectre variant 2 */
 #define X86_FEATURE_RETHUNK		(11*32+14) /* "" Use REturn THUNK */
 #define X86_FEATURE_UNRET		(11*32+15) /* "" AMD BTB untrain return */
 #define X86_FEATURE_USE_IBPB_FW		(11*32+16) /* "" Use IBPB during runtime firmware calls */
arch/x86/include/asm/disabled-features.h +1 -2
index c492bdc97b05..e1c421282a78 100644
@@ -53,8 +53,7 @@
 #ifdef CONFIG_MITIGATION_RETPOLINE
 # define DISABLE_RETPOLINE	0
 #else
-# define DISABLE_RETPOLINE	((1 << (X86_FEATURE_RETPOLINE & 31)) | \
-				 (1 << (X86_FEATURE_RETPOLINE_LFENCE & 31)))
+# define DISABLE_RETPOLINE	(1 << (X86_FEATURE_RETPOLINE & 31)
 #endif
 
 #ifdef CONFIG_MITIGATION_RETHUNK
arch/x86/include/asm/nospec-branch.h +4 -14
index ff5f1ecc7d1e..b98fb18928f6 100644
@@ -434,15 +434,11 @@ static inline void call_depth_return_thunk(void) {}
  * which is ensured when CONFIG_MITIGATION_RETPOLINE is defined.
  */
 # define CALL_NOSPEC						\
-	ALTERNATIVE_2(						\
+	ALTERNATIVE(						\
 	ANNOTATE_RETPOLINE_SAFE					\
 	"call *%[thunk_target]\n",				\
 	"call __x86_indirect_thunk_%V[thunk_target]\n",		\
-	X86_FEATURE_RETPOLINE,					\
-	"lfence;\n"						\
-	ANNOTATE_RETPOLINE_SAFE					\
-	"call *%[thunk_target]\n",				\
-	X86_FEATURE_RETPOLINE_LFENCE)
+	X86_FEATURE_RETPOLINE)
 
 # define THUNK_TARGET(addr) [thunk_target] "r" (addr)
 
@@ -453,7 +449,7 @@ static inline void call_depth_return_thunk(void) {}
  * here, anyway.
  */
 # define CALL_NOSPEC						\
-	ALTERNATIVE_2(						\
+	ALTERNATIVE(						\
 	ANNOTATE_RETPOLINE_SAFE					\
 	"call *%[thunk_target]\n",				\
 	"       jmp    904f;\n"					\
@@ -468,11 +464,7 @@ static inline void call_depth_return_thunk(void) {}
 	"       ret;\n"						\
 	"       .align 16\n"					\
 	"904:	call   901b;\n",				\
-	X86_FEATURE_RETPOLINE,					\
-	"lfence;\n"						\
-	ANNOTATE_RETPOLINE_SAFE					\
-	"call *%[thunk_target]\n",				\
-	X86_FEATURE_RETPOLINE_LFENCE)
+	X86_FEATURE_RETPOLINE)
 
 # define THUNK_TARGET(addr) [thunk_target] "rm" (addr)
 #endif
@@ -485,10 +477,8 @@ static inline void call_depth_return_thunk(void) {}
 enum spectre_v2_mitigation {
 	SPECTRE_V2_NONE,
 	SPECTRE_V2_RETPOLINE,
-	SPECTRE_V2_LFENCE,
 	SPECTRE_V2_EIBRS,
 	SPECTRE_V2_EIBRS_RETPOLINE,
-	SPECTRE_V2_EIBRS_LFENCE,
 	SPECTRE_V2_IBRS,
 };
 
arch/x86/kernel/alternative.c +2 -15
index 45a280f2161c..a7eb8203a8cd 100644
@@ -639,8 +639,6 @@ static int emit_call_track_retpoline(void *addr, struct insn *insn, int reg, u8
  * into:
  *
  *   CALL *%\reg
- *
- * It also tries to inline spectre_v2=retpoline,lfence when size permits.
  */
 static int patch_retpoline(void *addr, struct insn *insn, u8 *bytes)
 {
@@ -657,8 +655,7 @@ static int patch_retpoline(void *addr, struct insn *insn, u8 *bytes)
 	/* If anyone ever does: CALL/JMP *%rsp, we're in deep trouble. */
 	BUG_ON(reg == 4);
 
-	if (cpu_feature_enabled(X86_FEATURE_RETPOLINE) &&
-	    !cpu_feature_enabled(X86_FEATURE_RETPOLINE_LFENCE)) {
+	if (cpu_feature_enabled(X86_FEATURE_RETPOLINE)) {
 		if (cpu_feature_enabled(X86_FEATURE_CALL_DEPTH))
 			return emit_call_track_retpoline(addr, insn, reg, bytes);
 
@@ -675,9 +672,8 @@ static int patch_retpoline(void *addr, struct insn *insn, u8 *bytes)
 	 * into:
 	 *
 	 *   Jncc.d8 1f
-	 *   [ LFENCE ]
 	 *   JMP *%\reg
-	 *   [ NOP ]
+	 *   NOP
 	 * 1:
 	 */
 	if (is_jcc32(insn)) {
@@ -691,15 +687,6 @@ static int patch_retpoline(void *addr, struct insn *insn, u8 *bytes)
 		op = JMP32_INSN_OPCODE;
 	}
 
-	/*
-	 * For RETPOLINE_LFENCE: prepend the indirect CALL/JMP with an LFENCE.
-	 */
-	if (cpu_feature_enabled(X86_FEATURE_RETPOLINE_LFENCE)) {
-		bytes[i++] = 0x0f;
-		bytes[i++] = 0xae;
-		bytes[i++] = 0xe8; /* LFENCE */
-	}
-
 	ret = emit_indirect(op, reg, bytes + i);
 	if (ret < 0)
 		return ret;
arch/x86/kernel/cpu/bugs.c +3 -63

190 lines not on this page · raw message

arch/x86/kernel/cpu/cpu.h +1 -2
index ea9e07d57c8d..602318b87bd9 100644
@@ -93,8 +93,7 @@ extern enum spectre_v2_mitigation spectre_v2_enabled;
 static inline bool spectre_v2_in_eibrs_mode(enum spectre_v2_mitigation mode)
 {
 	return mode == SPECTRE_V2_EIBRS ||
-	       mode == SPECTRE_V2_EIBRS_RETPOLINE ||
-	       mode == SPECTRE_V2_EIBRS_LFENCE;
+	       mode == SPECTRE_V2_EIBRS_RETPOLINE;
 }
 
 #endif /* ARCH_X86_CPU_H */
arch/x86/lib/retpoline.S +2 -3
index e674ccf720b9..e32199f40c48 100644
@@ -37,9 +37,8 @@ SYM_INNER_LABEL(__x86_indirect_thunk_\reg, SYM_L_GLOBAL)
 	UNWIND_HINT_UNDEFINED
 	ANNOTATE_NOENDBR
 
-	ALTERNATIVE_2 __stringify(RETPOLINE \reg), \
-		      __stringify(lfence; ANNOTATE_RETPOLINE_SAFE; jmp *%\reg; int3), X86_FEATURE_RETPOLINE_LFENCE, \
-		      __stringify(ANNOTATE_RETPOLINE_SAFE; jmp *%\reg), ALT_NOT(X86_FEATURE_RETPOLINE)
+	ALTERNATIVE __stringify(RETPOLINE \reg), \
+		    __stringify(ANNOTATE_RETPOLINE_SAFE; jmp *%\reg), ALT_NOT(X86_FEATURE_RETPOLINE)
 
 .endm
 
arch/x86/net/bpf_jit_comp.c +1 -4
index df5fac428408..39ed0107fb59 100644
@@ -556,10 +556,7 @@ static void emit_indirect_jump(u8 **pprog, int reg, u8 *ip)
 {
 	u8 *prog = *pprog;
 
-	if (cpu_feature_enabled(X86_FEATURE_RETPOLINE_LFENCE)) {
-		EMIT_LFENCE();
-		EMIT2(0xFF, 0xE0 + reg);
-	} else if (cpu_feature_enabled(X86_FEATURE_RETPOLINE)) {
+	if (cpu_feature_enabled(X86_FEATURE_RETPOLINE)) {
 		OPTIMIZER_HIDE_VAR(reg);
 		if (cpu_feature_enabled(X86_FEATURE_CALL_DEPTH))
 			emit_jump(&prog, &__x86_indirect_jump_thunk_array[reg], ip);
tools/arch/x86/include/asm/cpufeatures.h +0 -1
index 25160d26764b..6627e0d25425 100644
@@ -298,7 +298,6 @@
 #define X86_FEATURE_ENTRY_IBPB		(11*32+10) /* "" Issue an IBPB on kernel entry */
 #define X86_FEATURE_RRSBA_CTRL		(11*32+11) /* "" RET prediction control */
 #define X86_FEATURE_RETPOLINE		(11*32+12) /* "" Generic Retpoline mitigation for Spectre variant 2 */
-#define X86_FEATURE_RETPOLINE_LFENCE	(11*32+13) /* "" Use LFENCE for Spectre variant 2 */
 #define X86_FEATURE_RETHUNK		(11*32+14) /* "" Use REturn THUNK */
 #define X86_FEATURE_UNRET		(11*32+15) /* "" AMD BTB untrain return */
 #define X86_FEATURE_USE_IBPB_FW		(11*32+16) /* "" Use IBPB during runtime firmware calls */
tools/arch/x86/include/asm/disabled-features.h +1 -2
index 1f23960d2b06..74538147edc8 100644
@@ -53,8 +53,7 @@
 #ifdef CONFIG_MITIGATION_RETPOLINE
 # define DISABLE_RETPOLINE	0
 #else
-# define DISABLE_RETPOLINE	((1 << (X86_FEATURE_RETPOLINE & 31)) | \
-				 (1 << (X86_FEATURE_RETPOLINE_LFENCE & 31)))
+# define DISABLE_RETPOLINE	(1 << (X86_FEATURE_RETPOLINE & 31)
 #endif
 
 #ifdef CONFIG_MITIGATION_RETHUNK
Signature · 1 line
2.44.0
author maintainer
#12 lore raw

Re: [PATCH 3/3] x86/bugs: Remove support for Spectre v2 LFENCE "retpolines"

Quoted 4 lines · Josh Poimboeuf
---
 arch/x86/Makefile                             |  1 -
 arch/x86/include/asm/cpufeatures.h            |  1 -
 arch/x86/include/asm/disabled-features.h      |  3 +-
Quoted 8 lines
 arch/x86/include/asm/nospec-branch.h          | 18 ++---
 arch/x86/kernel/alternative.c                 | 17 +----
 arch/x86/kernel/cpu/bugs.c                    | 66 +------------------
 arch/x86/kernel/cpu/cpu.h                     |  3 +-
 arch/x86/lib/retpoline.S                      |  5 +-
 arch/x86/net/bpf_jit_comp.c                   |  5 +-
 tools/arch/x86/include/asm/cpufeatures.h      |  1 -
 .../arch/x86/include/asm/disabled-features.h  |  3 +-

Forgot the documentation updates:

Documentation/admin-guide/hw-vuln/spectre.rst +1 -6
index 25a04cda4c2c..de780db82cd8 100644
@@ -380,10 +380,8 @@ The possible values in this file are:
   'Not affected'                            The processor is not vulnerable
   'Mitigation: None'                        Vulnerable, no mitigation
   'Mitigation: Retpolines'                  Use Retpoline thunks
-  'Mitigation: LFENCE'                      Use LFENCE instructions
   'Mitigation: Enhanced IBRS'               Hardware-focused mitigation
   'Mitigation: Enhanced IBRS + Retpolines'  Hardware-focused + Retpolines
-  'Mitigation: Enhanced IBRS + LFENCE'      Hardware-focused + LFENCE
   ========================================  =================================
 
   - Firmware status: Show if Indirect Branch Restricted Speculation (IBRS) is
@@ -640,13 +638,10 @@ kernel command line.
 
 		Specific mitigations can also be selected manually:
 
-                retpoline               auto pick between generic,lfence
+                retpoline               Retpolines
                 retpoline,generic       Retpolines
-                retpoline,lfence        LFENCE; indirect branch
-                retpoline,amd           alias for retpoline,lfence
                 eibrs                   Enhanced/Auto IBRS
                 eibrs,retpoline         Enhanced/Auto IBRS + Retpolines
-                eibrs,lfence            Enhanced/Auto IBRS + LFENCE
                 ibrs                    use IBRS to protect kernel
 
 		Not specifying this option is equivalent to
Documentation/admin-guide/kernel-parameters.txt +1 -4
index 902ecd92a29f..edbfba7299e7 100644
@@ -6099,13 +6099,10 @@
 
 			Specific mitigations can also be selected manually:
 
-			retpoline	  - replace indirect branches
+			retpoline	  - Retpolines
 			retpoline,generic - Retpolines
-			retpoline,lfence  - LFENCE; indirect branch
-			retpoline,amd     - alias for retpoline,lfence
 			eibrs		  - Enhanced/Auto IBRS
 			eibrs,retpoline   - Enhanced/Auto IBRS + Retpolines
-			eibrs,lfence      - Enhanced/Auto IBRS + LFENCE
 			ibrs		  - use IBRS to protect kernel
 
 			Not specifying this option is equivalent to
#13 lore raw

Re: [PATCH 3/3] x86/bugs: Remove support for Spectre v2 LFENCE "retpolines"

In reply to Josh Poimboeuf
Quoted 2 lines · Josh Poimboeuf

I found several bugs where code assumes that X86_FEATURE_RETPOLINE actually means retpolines (imagine that!).

Yeah :(   One could also imagine a past where that was pointed out, or just read about it in the archives.

Quoted 8 lines

In fact that feature also
includes the original AMD LFENCE "retpolines", which aren't in fact retpolines.

Really, those "retpolines" should just be removed. They're already considered vulnerable due to the fact that the speculative window after the indirect branch can still be long enough to do multiple dependent loads. And recent tooling makes such gadgets easier to find.

There are two Atom CPUs which are not repotline safe, and for which Intel released a statement saying "use lfence/jmp" on these.

I'm still trying to find it...

~Andrew

reviewer Reviewed-by on v2 1/3
#2 lore raw

Re: [PATCH v2 1/3] x86/bugs: Only harden syscalls when needed

In reply to Josh Poimboeuf

arch/x86/include/asm/cpufeatures.h line 473

Quoted 9 lines · Josh Poimboeuf
arch/x86/include/asm/cpufeatures.h +1 -0
index 3c7434329661..7c87fe80c696 100644
@@ -470,6 +470,7 @@
 #define X86_FEATURE_BHI_CTRL		(21*32+ 2) /* "" BHI_DIS_S HW control available */
 #define X86_FEATURE_CLEAR_BHB_HW	(21*32+ 3) /* "" BHI_DIS_S HW control enabled */
 #define X86_FEATURE_CLEAR_BHB_LOOP_ON_VMEXIT (21*32+ 4) /* "" Clear branch history at vmexit using SW loop */
+#define X86_FEATURE_INDIRECT_SAFE	(21*32+ 4) /* "" Indirect branches aren't vulnerable to Spectre v2 */

This should be (21*32+ 5).

Other than that:

author maintainer
#3 lore raw

Re: [PATCH v2 1/3] x86/bugs: Only harden syscalls when needed

Quoted 12 lines · Pawan Gupta

arch/x86/include/asm/cpufeatures.h line 473

Quoted 9 lines · Josh Poimboeuf
arch/x86/include/asm/cpufeatures.h +1 -0
index 3c7434329661..7c87fe80c696 100644
@@ -470,6 +470,7 @@
 #define X86_FEATURE_BHI_CTRL		(21*32+ 2) /* "" BHI_DIS_S HW control available */
 #define X86_FEATURE_CLEAR_BHB_HW	(21*32+ 3) /* "" BHI_DIS_S HW control enabled */
 #define X86_FEATURE_CLEAR_BHB_LOOP_ON_VMEXIT (21*32+ 4) /* "" Clear branch history at vmexit using SW loop */
+#define X86_FEATURE_INDIRECT_SAFE	(21*32+ 4) /* "" Indirect branches aren't vulnerable to Spectre v2 */

This should be (21*32+ 5).

Argh :-/

Quoted 3 lines

Other than that:

Thanks!

Signature · 1 line
Josh
reviewer Reviewed-by on v1 2/3
#9 lore raw

Re: [PATCH 2/3] x86/bugs: Fix BHI retpoline check

In reply to Josh Poimboeuf
Quoted 10 lines · Josh Poimboeuf

Confusingly, X86_FEATURE_RETPOLINE doesn't mean retpolines are enabled, as it also includes the original "AMD retpoline" which isn't a retpoline at all.

Also replace cpu_feature_enabled() with boot_cpu_has() because this is before alternatives are patched and cpu_feature_enabled()'s fallback path is slower than plain old boot_cpu_has().

Fixes ec9404e40e8f ("x86/bhi: Add BHI mitigation knob")
author maintainer
#15 lore raw

Re: [PATCH 3/3] x86/bugs: Remove support for Spectre v2 LFENCE "retpolines"

In reply to Josh Poimboeuf
Quoted 15 lines · Josh Poimboeuf

I found several bugs where code assumes that X86_FEATURE_RETPOLINE actually means retpolines (imagine that!). In fact that feature also includes the original AMD LFENCE "retpolines", which aren't in fact retpolines.

Really, those "retpolines" should just be removed. They're already considered vulnerable due to the fact that the speculative window after the indirect branch can still be long enough to do multiple dependent loads. And recent tooling makes such gadgets easier to find.

Also, EIBRS_LFENCE tests worse in real-world benchmarks than the actual BHI mitigations, so it's both slower and less secure.

Specifically this removes support for the following cmdline options:

Quoted 8 lines

- spectre_v2=retpoline,amd
- spectre_v2=retpoline,lfence
- spectre_v2=eibrs,lfence

Now when any of those options are used, it will print an error and fall back to the defaults (spectre_v2=auto spectre_bhi=on).

Compile fix:

arch/x86/include/asm/disabled-features.h +1 -1
index e1c421282a78..3a1349c0f225 100644
@@ -53,7 +53,7 @@
 #ifdef CONFIG_MITIGATION_RETPOLINE
 # define DISABLE_RETPOLINE	0
 #else
-# define DISABLE_RETPOLINE	(1 << (X86_FEATURE_RETPOLINE & 31)
+# define DISABLE_RETPOLINE	(1 << (X86_FEATURE_RETPOLINE & 31))
 #endif
 
 #ifdef CONFIG_MITIGATION_RETHUNK
merge bot Applied
#10 lore raw

[tip: x86/urgent] x86/bugs: Fix BHI retpoline check

In reply to Josh Poimboeuf
The following commit has been merged into the x86/urgent branch of tip:

The following commit has been merged into the x86/urgent branch of tip:

Commit-ID:     69129794d94c544810e68b2b4eaa7e44063f9bf2
Gitweb:        https://git.kernel.org/tip/69129794d94c544810e68b2b4eaa7e44063f9bf2
Author:        Josh Poimboeuf <[email protected]>
AuthorDate:    Fri, 12 Apr 2024 11:10:33 -07:00
Committer:     Ingo Molnar <[email protected]>
CommitterDate: Sun, 14 Apr 2024 11:10:05 +02:00

x86/bugs: Fix BHI retpoline check

Confusingly, X86_FEATURE_RETPOLINE doesn't mean retpolines are enabled, as it also includes the original "AMD retpoline" which isn't a retpoline at all.

Also replace cpu_feature_enabled() with boot_cpu_has() because this is before alternatives are patched and cpu_feature_enabled()'s fallback path is slower than plain old boot_cpu_has().

---
 arch/x86/kernel/cpu/bugs.c | 11 +++++++----
 1 file changed, 7 insertions(+), 4 deletions(-)
arch/x86/kernel/cpu/bugs.c +7 -4
index ca295b0..ab18185 100644
@@ -1652,7 +1652,8 @@ static void __init bhi_select_mitigation(void)
 		return;
 
 	/* Retpoline mitigates against BHI unless the CPU has RRSBA behavior */
-	if (cpu_feature_enabled(X86_FEATURE_RETPOLINE)) {
+	if (boot_cpu_has(X86_FEATURE_RETPOLINE) &&
+	    !boot_cpu_has(X86_FEATURE_RETPOLINE_LFENCE)) {
 		spec_ctrl_disable_kernel_rrsba();
 		if (rrsba_disabled)
 			return;
@@ -2804,11 +2805,13 @@ static const char *spectre_bhi_state(void)
 {
 	if (!boot_cpu_has_bug(X86_BUG_BHI))
 		return "; BHI: Not affected";
-	else if  (boot_cpu_has(X86_FEATURE_CLEAR_BHB_HW))
+	else if (boot_cpu_has(X86_FEATURE_CLEAR_BHB_HW))
 		return "; BHI: BHI_DIS_S";
-	else if  (boot_cpu_has(X86_FEATURE_CLEAR_BHB_LOOP))
+	else if (boot_cpu_has(X86_FEATURE_CLEAR_BHB_LOOP))
 		return "; BHI: SW loop, KVM: SW loop";
-	else if (boot_cpu_has(X86_FEATURE_RETPOLINE) && rrsba_disabled)
+	else if (boot_cpu_has(X86_FEATURE_RETPOLINE) &&
+		 !boot_cpu_has(X86_FEATURE_RETPOLINE_LFENCE) &&
+		 rrsba_disabled)
 		return "; BHI: Retpoline";
 	else if (boot_cpu_has(X86_FEATURE_CLEAR_BHB_LOOP_ON_VMEXIT))
 		return "; BHI: Vulnerable, KVM: SW loop";
#4 lore raw

Re: [PATCH v2 1/3] x86/bugs: Only harden syscalls when needed

In reply to Josh Poimboeuf
Quoted 7 lines · Josh Poimboeuf

Syscall hardening (i.e., converting the syscall indirect branch to a series of direct branches) may cause performance regressions in certain scenarios. Only use the syscall hardening when indirect branches are considered unsafe.

Fixes 1e3ad78334a6 ("x86/syscall: Don't force use of indirect calls for system calls")
Quoted 10 lines
---
  arch/x86/entry/common.c            | 15 ++++++++++++---
  arch/x86/entry/syscall_32.c        | 11 +----------
  arch/x86/entry/syscall_64.c        |  6 ------
  arch/x86/entry/syscall_x32.c       |  7 ++++++-
  arch/x86/include/asm/cpufeatures.h |  1 +
  arch/x86/include/asm/syscall.h     |  8 +++++++-
  arch/x86/kernel/cpu/bugs.c         | 31 +++++++++++++++++++++++++++++-
  7 files changed, 57 insertions(+), 22 deletions(-)

To ask again, what do we gain by having this syscall hardening at the same time as the always on BHB scrubbing sequence?

#5 lore raw

Re: [PATCH v2 1/3] x86/bugs: Only harden syscalls when needed

Quoted 3 lines · Nikolay Borisov

To ask again, what do we gain by having this syscall hardening at the same time as the always on BHB scrubbing sequence?

What happens the next time some indirect call problem comes up?

If we had had *one* hardware bug in this area, that would be one thing. But this has been going on for a decade now.

Linus

#6 lore raw

Re: [PATCH v2 1/3] x86/bugs: Only harden syscalls when needed

Quoted 6 lines · Linus Torvalds
Quoted 3 lines · Nikolay Borisov

To ask again, what do we gain by having this syscall hardening at the same time as the always on BHB scrubbing sequence?

What happens the next time some indirect call problem comes up?

Same as with every issue - assess the problem and develop fixes. Let's be honest, the indirect branches in the syscall handler aren't the biggest problem, it's the stacked LSMs. And even if those get fixes chances are the security people will likely find some other avenue of attack, I think even now the attack is somewhat hard to pull off.

So in any case this could have been a completely independent patch of the BHI series.

Quoted 9 lines

If we had had *one* hardware bug in this area, that would be one thing. But this has been going on for a decade now.

Linus

#7 lore raw

Re: [PATCH v2 1/3] x86/bugs: Only harden syscalls when needed

Quoted 2 lines · Nikolay Borisov

Same as with every issue - assess the problem and develop fixes.

No. Let's have at least all the infrastructure in place to be a bit proactive.

Quoted 2 lines

Let's be honest, the indirect branches in the syscall handler aren't the biggest problem

Oh, they have been.

Quoted 1 line

it's the stacked LSMs.

Hopefully those will get fixed too.

There's a few other fairly reachable ones (the timer indirection ones are much too close, and VFS file ops aren't entirely out of reach).

But maybe some day we'll be in a situation where it's actually fairly hard to reach indirect kernel calls from untrusted user space.

The system call ones are pretty much always the first ones, though.

Quoted 3 lines

And even if those get fixes
chances are the security people will likely find some other avenue of attack, I think even now the attack is somewhat hard to pull off.

No disagreement about that. I think outright sw bugs are still the 99.9% thing. But let's learn from history instead of "assess the problem" every time anew.

Linus

author maintainer
#14 lore raw

Re: [PATCH 3/3] x86/bugs: Remove support for Spectre v2 LFENCE "retpolines"

In reply to Andrew Cooper
Quoted 16 lines · Andrew Cooper
Quoted 2 lines · Josh Poimboeuf

I found several bugs where code assumes that X86_FEATURE_RETPOLINE actually means retpolines (imagine that!).

Yeah :(   One could also imagine a past where that was pointed out, or just read about it in the archives.

Quoted 8 lines

In fact that feature also
includes the original AMD LFENCE "retpolines", which aren't in fact retpolines.

Really, those "retpolines" should just be removed. They're already considered vulnerable due to the fact that the speculative window after the indirect branch can still be long enough to do multiple dependent loads. And recent tooling makes such gadgets easier to find.

Quoted 4 lines

There are two Atom CPUs which are not repotline safe, and for which Intel released a statement saying "use lfence/jmp" on these.

I'm still trying to find it...

Any luck finding it? The only thing I found [1] isn't exactly a ringing endorsement of LFENCE;JMP over retpolines.

[1] https://www.intel.com/content/www/us/en/developer/articles/technical/software-security-guidance/technical-documentation/branch-history-injection.html

Signature · 1 line
Josh