Skip to contentSkip to conversation

Rewrite x86/ftrace to use text_poke()

PATCH In review Peter Zijlstra 1 participant 7 messages Aug 26, 2019

Subsystems (3)
lore
Share & tools
lore

Peter Zijlstra

author maintainer
#0 lore raw

cover Rewrite x86/ftrace to use text_poke()

Ftrace was one of the last W^X violators; these patches move it over to the generic text_poke() interface and thereby get rid of this oddity.

Very lightly tested...

author maintainer
#1 lore raw

PATCH 1/3 x86/alternatives: Teach text_poke_bp() to emulate instructions

In preparation for static_call and variable size jump_label support, teach text_poke_bp() to emulate instructions, namely:

JMP32, JMP8, CALL, NOP2, NOP_ATOMIC5

The current text_poke_bp() takes a @handler argument which is used as a jump target when the temporary INT3 is hit by a different CPU.

When patching CALL instructions, this doesn't work because we'd miss the PUSH of the return address. Instead, teach poke_int3_handler() to emulate an instruction, typically the instruction we're patching in.

This fits almost all text_poke_bp() users, except arch_unoptimize_kprobe() which restores random text, and for that site we have to build an explicit emulate instruction.

Cc Masami Hiramatsu <[email protected]>
Cc Steven Rostedt <[email protected]>
Cc Daniel Bristot de Oliveira <[email protected]>
---
 arch/x86/include/asm/text-patching.h |   24 ++++++--
 arch/x86/kernel/alternative.c        |   98 ++++++++++++++++++++++++++---------
 arch/x86/kernel/jump_label.c         |    9 +--
 arch/x86/kernel/kprobes/opt.c        |   11 ++-
 4 files changed, 103 insertions(+), 39 deletions(-)
arch/x86/include/asm/text-patching.h +18 -6
@@ -26,10 +26,11 @@ static inline void apply_paravirt(struct
 #define POKE_MAX_OPCODE_SIZE	5
 
 struct text_poke_loc {
-	void *detour;
 	void *addr;
-	size_t len;
-	const char opcode[POKE_MAX_OPCODE_SIZE];
+	int len;
+	s32 rel32;
+	u8 opcode;
+	const char text[POKE_MAX_OPCODE_SIZE];
 };
 
 extern void text_poke_early(void *addr, const void *opcode, size_t len);
@@ -51,8 +52,10 @@ extern void text_poke_early(void *addr,
 extern void *text_poke(void *addr, const void *opcode, size_t len);
 extern void *text_poke_kgdb(void *addr, const void *opcode, size_t len);
 extern int poke_int3_handler(struct pt_regs *regs);
-extern void text_poke_bp(void *addr, const void *opcode, size_t len, void *handler);
+extern void text_poke_bp(void *addr, const void *opcode, size_t len, const void *emulate);
 extern void text_poke_bp_batch(struct text_poke_loc *tp, unsigned int nr_entries);
+extern void text_poke_loc_init(struct text_poke_loc *tp, void *addr,
+			       const void *opcode, size_t len, const void *emulate);
 extern int after_bootmem;
 extern __ro_after_init struct mm_struct *poking_mm;
 extern __ro_after_init unsigned long poking_addr;
@@ -63,8 +66,17 @@ static inline void int3_emulate_jmp(stru
 	regs->ip = ip;
 }
 
-#define INT3_INSN_SIZE 1
-#define CALL_INSN_SIZE 5
+#define INT3_INSN_SIZE		1
+#define INT3_INSN_OPCODE	0xCC
+
+#define CALL_INSN_SIZE		5
+#define CALL_INSN_OPCODE	0xE8
+
+#define JMP32_INSN_SIZE		5
+#define JMP32_INSN_OPCODE	0xE9
+
+#define JMP8_INSN_SIZE		2
+#define JMP8_INSN_OPCODE	0xEB
 
 static inline void int3_emulate_push(struct pt_regs *regs, unsigned long val)
 {
arch/x86/kernel/alternative.c +74 -24

155 lines not on this page · raw message

arch/x86/kernel/jump_label.c +3 -6
@@ -89,8 +89,7 @@ static void __ref __jump_label_transform
 		return;
 	}
 
-	text_poke_bp((void *)jump_entry_code(entry), &code, JUMP_LABEL_NOP_SIZE,
-		     (void *)jump_entry_code(entry) + JUMP_LABEL_NOP_SIZE);
+	text_poke_bp((void *)jump_entry_code(entry), &code, JUMP_LABEL_NOP_SIZE, NULL);
 }
 
 void arch_jump_label_transform(struct jump_entry *entry,
@@ -147,11 +146,9 @@ bool arch_jump_label_transform_queue(str
 	}
 
 	__jump_label_set_jump_code(entry, type,
-				   (union jump_code_union *) &tp->opcode, 0);
+				   (union jump_code_union *)&tp->text, 0);
 
-	tp->addr = entry_code;
-	tp->detour = entry_code + JUMP_LABEL_NOP_SIZE;
-	tp->len = JUMP_LABEL_NOP_SIZE;
+	text_poke_loc_init(tp, entry_code, NULL, JUMP_LABEL_NOP_SIZE, NULL);
 
 	tp_vec_nr++;
 
arch/x86/kernel/kprobes/opt.c +8 -3
@@ -437,8 +437,7 @@ void arch_optimize_kprobes(struct list_h
 		insn_buff[0] = RELATIVEJUMP_OPCODE;
 		*(s32 *)(&insn_buff[1]) = rel;
 
-		text_poke_bp(op->kp.addr, insn_buff, RELATIVEJUMP_SIZE,
-			     op->optinsn.insn);
+		text_poke_bp(op->kp.addr, insn_buff, RELATIVEJUMP_SIZE, NULL);
 
 		list_del_init(&op->list);
 	}
@@ -448,12 +447,18 @@ void arch_optimize_kprobes(struct list_h
 void arch_unoptimize_kprobe(struct optimized_kprobe *op)
 {
 	u8 insn_buff[RELATIVEJUMP_SIZE];
+	u8 emulate_buff[RELATIVEJUMP_SIZE];
 
 	/* Set int3 to first byte for kprobes */
 	insn_buff[0] = BREAKPOINT_INSTRUCTION;
 	memcpy(insn_buff + 1, op->optinsn.copied_insn, RELATIVE_ADDR_SIZE);
+
+	emulate_buff[0] = RELATIVEJUMP_OPCODE;
+	*(s32 *)(&emulate_buff[1]) = (s32)((long)op->optinsn.insn -
+			((long)op->kp.addr + RELATIVEJUMP_SIZE));
+
 	text_poke_bp(op->kp.addr, insn_buff, RELATIVEJUMP_SIZE,
-		     op->optinsn.insn);
+		     emulate_buff);
 }
 
 /*
author maintainer
#2 lore raw

PATCH 2/3 x86/alternatives: Move tp_vec

In reply to Peter Zijlstra

In order to allow other users to make use of the tp_vec; move it near the text_poke_bp_batch() code, instead of keeping it in the jump_label code.

Cc Daniel Bristot de Oliveira <[email protected]>
---
 arch/x86/include/asm/text-patching.h |    4 ++++
 arch/x86/kernel/alternative.c        |    3 +++
 arch/x86/kernel/jump_label.c         |   25 ++++++++++++-------------
 3 files changed, 19 insertions(+), 13 deletions(-)
arch/x86/include/asm/text-patching.h +4 -0
@@ -60,6 +60,10 @@ extern int after_bootmem;
 extern __ro_after_init struct mm_struct *poking_mm;
 extern __ro_after_init unsigned long poking_addr;
 
+#define TP_VEC_MAX (PAGE_SIZE / sizeof(struct text_poke_loc))
+extern struct text_poke_loc tp_vec[TP_VEC_MAX];
+extern int tp_vec_nr;
+
 #ifndef CONFIG_UML_X86
 static inline void int3_emulate_jmp(struct pt_regs *regs, unsigned long ip)
 {
arch/x86/kernel/alternative.c +3 -0
@@ -1017,6 +1017,9 @@ int poke_int3_handler(struct pt_regs *re
 }
 NOKPROBE_SYMBOL(poke_int3_handler);
 
+struct text_poke_loc tp_vec[TP_VEC_MAX];
+int tp_vec_nr;
+
 /**
  * text_poke_bp_batch() -- update instructions on live kernel on SMP
  * @tp:			vector of instructions to patch
arch/x86/kernel/jump_label.c +12 -13
@@ -100,15 +100,12 @@ void arch_jump_label_transform(struct ju
 	mutex_unlock(&text_mutex);
 }
 
-#define TP_VEC_MAX (PAGE_SIZE / sizeof(struct text_poke_loc))
-static struct text_poke_loc tp_vec[TP_VEC_MAX];
-static int tp_vec_nr;
-
 bool arch_jump_label_transform_queue(struct jump_entry *entry,
 				     enum jump_label_type type)
 {
 	struct text_poke_loc *tp;
 	void *entry_code;
+	bool ret = true;
 
 	if (system_state == SYSTEM_BOOTING) {
 		/*
@@ -118,12 +115,15 @@ bool arch_jump_label_transform_queue(str
 		return true;
 	}
 
+	mutex_lock(&text_mutex);
 	/*
 	 * No more space in the vector, tell upper layer to apply
 	 * the queue before continuing.
 	 */
-	if (tp_vec_nr == TP_VEC_MAX)
-		return false;
+	if (tp_vec_nr == TP_VEC_MAX) {
+		ret = false;
+		goto unlock;
+	}
 
 	tp = &tp_vec[tp_vec_nr];
 
@@ -151,20 +151,19 @@ bool arch_jump_label_transform_queue(str
 	text_poke_loc_init(tp, entry_code, NULL, JUMP_LABEL_NOP_SIZE, NULL);
 
 	tp_vec_nr++;
+unlock:
+	mutex_unlock(&text_mutex);
 
-	return true;
+	return ret;
 }
 
 void arch_jump_label_transform_apply(void)
 {
-	if (!tp_vec_nr)
-		return;
-
 	mutex_lock(&text_mutex);
-	text_poke_bp_batch(tp_vec, tp_vec_nr);
-	mutex_unlock(&text_mutex);
-
+	if (tp_vec_nr)
+		text_poke_bp_batch(tp_vec, tp_vec_nr);
 	tp_vec_nr = 0;
+	mutex_unlock(&text_mutex);
 }
 
 static enum {
author maintainer
#3 lore raw

Re: [PATCH 2/3] x86/alternatives: Move tp_vec

arch/x86/include/asm/text-patching.h extern int after_bootmem; · line 65

Quoted 3 lines · Peter Zijlstra
arch/x86/include/asm/text-patching.h +3 -0
+#define TP_VEC_MAX (PAGE_SIZE / sizeof(struct text_poke_loc))
+extern struct text_poke_loc tp_vec[TP_VEC_MAX];
+extern int tp_vec_nr;

FWIW, that currently results in a batch size of 128, but I've not noticed any delay in flipping ftrace on and off on the commandline.

Growing that buffer really shouldn't be a problem, but I'm thinking we'd want solid performance numbers to justify it.

author maintainer
#4 lore raw

RFC PATCH 3/3 x86/ftrace: Use text_poke()

In reply to Peter Zijlstra

Move ftrace over to using the generic x86 text_poke functions; this avoids having a second/different copy of that code around.

Cc Daniel Bristot de Oliveira <[email protected]>
Cc Steven Rostedt <[email protected]>
---
 arch/x86/include/asm/ftrace.h |    2 
 arch/x86/kernel/ftrace.c      |  571 +++---------------------------------------
 arch/x86/kernel/traps.c       |    9 
 3 files changed, 51 insertions(+), 531 deletions(-)
arch/x86/include/asm/ftrace.h +0 -2
@@ -35,8 +35,6 @@ struct dyn_arch_ftrace {
 	/* No extra data needed for x86 */
 };
 
-int ftrace_int3_handler(struct pt_regs *regs);
-
 #define FTRACE_GRAPH_TRAMP_ADDR FTRACE_GRAPH_ADDR
 
 #endif /*  CONFIG_DYNAMIC_FTRACE */
arch/x86/kernel/ftrace.c +51 -520

680 lines not on this page · raw message

arch/x86/kernel/traps.c +0 -9
@@ -568,15 +568,6 @@ NOKPROBE_SYMBOL(do_general_protection);
 
 dotraplinkage void notrace do_int3(struct pt_regs *regs, long error_code)
 {
-#ifdef CONFIG_DYNAMIC_FTRACE
-	/*
-	 * ftrace must be first, everything else may cause a recursive crash.
-	 * See note by declaration of modifying_ftrace_code in ftrace.c
-	 */
-	if (unlikely(atomic_read(&modifying_ftrace_code)) &&
-	    ftrace_int3_handler(regs))
-		return;
-#endif
 	if (poke_int3_handler(regs))
 		return;
 
author maintainer
#5 lore raw

Re: [RFC][PATCH 3/3] x86/ftrace: Use text_poke()

Quoted 6 lines · Peter Zijlstra

Move ftrace over to using the generic x86 text_poke functions; this avoids having a second/different copy of that code around.

Cc Daniel Bristot de Oliveira <[email protected]>
Cc Steven Rostedt <[email protected]>

*sigh*.. one of those days again. I seem to have tested without this patch applied.

Let me go fix this.

author maintainer
#6 lore raw
In reply to Peter Zijlstra
Quoted 4 lines · Peter Zijlstra

Ftrace was one of the last W^X violators; these patches move it over to the generic text_poke() interface and thereby get rid of this oddity.

Very lightly tested...

I'm thinking there's more cleanup to be had; set_kernel_text_*() can go away now, and I'm thinking the whole batching thing can be further simplified/shared between this and jump_label.