kprobes/x86: retpoline: Fix kprobes for retpoline

Hi,

This is the 2nd version of the series to fix kprobes issues on the kernel with CONFIG_RETPOLINE=y.

- [1/3]: This introduces __x86_indirect_thunk_* boundary symbols so that kprobes easily identify those functions.
- [2/3]: Mark __x86_indirect_thunk_* as blacklisted function for kprobes, since it can be called from other blacklisted functions.
- [3/3]: Check jmp instructions in the probe target function whether it jumps into the __x86_indirect_thunk_*, because it is equal to an indirect jump instruction.

In this version, just [1/3] has been changed according to David Woodhouse's comment.

[1/3]: Consolidate .text.__x86.indirect_thunk.* sections
to .text.__x86.indirect_thunk section.

Thank you,

---

Masami Hiramatsu (3):
retpoline: Introduce start/end markers of indirect thunk kprobes/x86: Blacklist indirect thunk functions for kprobes kprobes/x86: Disable optimizing on the function jumps to indirect thunk

 arch/x86/include/asm/nospec-branch.h |    3 +++
 arch/x86/kernel/kprobes/opt.c        |   23 ++++++++++++++++++++++-
 arch/x86/kernel/vmlinux.lds.S        |    6 ++++++
 arch/x86/lib/retpoline.S             |    5 +++--
 4 files changed, 34 insertions(+), 3 deletions(-)

--
Masami Hiramatsu (Linaro) <[email protected]>

Introduce start/end markers of __x86_indirect_thunk_* functions. To make it easy, consolidate .text.__x86.indirect_thunk.* sections to one .text.__x86.indirect_thunk section and put it in the end of kernel text section and adds __indirect_thunk_start/end so that other subsystem (e.g. kprobes) can identify it.

---
Changes in v2:
- Consolidate .text.__x86.indirect_thunk.* sections to one .text.__x86.indirect_thunk section. (Thanks David!)

---
 arch/x86/include/asm/nospec-branch.h |    3 +++
 arch/x86/kernel/vmlinux.lds.S        |    6 ++++++
 arch/x86/lib/retpoline.S             |    2 +-
 3 files changed, 10 insertions(+), 1 deletion(-)

index 7b45d8424150..19ba5ad19c65 100644
@@ -194,6 +194,9 @@ enum spectre_v2_mitigation {
 	SPECTRE_V2_IBRS,
 };
 
+extern char __indirect_thunk_start[];
+extern char __indirect_thunk_end[];
+
 /*
  * On VMEXIT we must ensure that no RSB predictions learned in the guest
  * can be followed in the host, by overwriting the RSB completely. Both

index 1e413a9326aa..9b138a06c1a4 100644
@@ -124,6 +124,12 @@ SECTIONS
 		ASSERT(. - _entry_trampoline == PAGE_SIZE, "entry trampoline is too big");
 #endif
 
+#ifdef CONFIG_RETPOLINE
+		__indirect_thunk_start = .;
+		*(.text.__x86.indirect_thunk)
+		__indirect_thunk_end = .;
+#endif
+
 		/* End of text section */
 		_etext = .;
 	} :text = 0x9090

index cb45c6cb465f..d3415dc30f82 100644
@@ -9,7 +9,7 @@
 #include <asm/nospec-branch.h>
 
 .macro THUNK reg
-	.section .text.__x86.indirect_thunk.\reg
+	.section .text.__x86.indirect_thunk
 
 ENTRY(__x86_indirect_thunk_\reg)
 	CFI_STARTPROC

Mark __x86_indirect_thunk_* functions as blacklist for kprobes because those functions can be called from anywhere in the kernel including blacklist functions of kprobes.

---
 arch/x86/lib/retpoline.S |    3 ++-
 1 file changed, 2 insertions(+), 1 deletion(-)

index d3415dc30f82..dfb2ba91b670 100644
@@ -25,7 +25,8 @@ ENDPROC(__x86_indirect_thunk_\reg)
  * than one per register with the correct names. So we do it
  * the simple and nasty way...
  */
-#define EXPORT_THUNK(reg) EXPORT_SYMBOL(__x86_indirect_thunk_ ## reg)
+#define __EXPORT_THUNK(sym) _ASM_NOKPROBE(sym); EXPORT_SYMBOL(sym)
+#define EXPORT_THUNK(reg) __EXPORT_THUNK(__x86_indirect_thunk_ ## reg)
 #define GENERATE_THUNK(reg) THUNK reg ; EXPORT_THUNK(reg)
 
 GENERATE_THUNK(_ASM_AX)

Since indirect jump instructions will be replaced by jump to __x86_indirect_thunk_*, those jmp instruction must be treated as an indirect jump. Since optprobe prohibits to optimize probes in the function which uses an indirect jump, it also needs to find out the function which jump to __x86_indirect_thunk_* and disable optimization.

This adds a check that the jump target address is between the __indirect_thunk_start/end when optimizing kprobe.

---
 arch/x86/kernel/kprobes/opt.c |   23 ++++++++++++++++++++++-
 1 file changed, 22 insertions(+), 1 deletion(-)

index e941136e24d8..203d398802a3 100644
@@ -40,6 +40,7 @@
 #include <asm/debugreg.h>
 #include <asm/set_memory.h>
 #include <asm/sections.h>
+#include <asm/nospec-branch.h>
 
 #include "common.h"
 

@@ -203,7 +204,7 @@ static int copy_optimized_instructions(u8 *dest, u8 *src, u8 *real)
 }
 
 /* Check whether insn is indirect jump */
-static int insn_is_indirect_jump(struct insn *insn)
+static int __insn_is_indirect_jump(struct insn *insn)
 {
 	return ((insn->opcode.bytes[0] == 0xff &&
 		(X86_MODRM_REG(insn->modrm.value) & 6) == 4) || /* Jump */

@@ -237,6 +238,26 @@ static int insn_jump_into_range(struct insn *insn, unsigned long start, int len)
 	return (start <= target && target <= start + len);
 }
 
+static int insn_is_indirect_jump(struct insn *insn)
+{
+	int ret = __insn_is_indirect_jump(insn);
+
+#ifdef CONFIG_RETPOLINE
+	/*
+	 * Jump to x86_indirect_thunk_* is treated as an indirect jump.
+	 * Note that even with CONFIG_RETPOLINE=y, the kernel compiled with
+	 * older gcc may use indirect jump. So we add this check instead of
+	 * replace indirect-jump check.
+	 */
+	if (!ret)
+		ret = insn_jump_into_range(insn,
+				(unsigned long)__indirect_thunk_start,
+				(unsigned long)__indirect_thunk_end -
+				(unsigned long)__indirect_thunk_start);
+#endif
+	return ret;
+}
+
 /* Decode whole function to ensure any instructions don't jump into target */
 static int can_optimize(unsigned long paddr)
 {

For all three:

Thank you.