Rewrite x86/ftrace to use text_poke()
Subsystems (3)
Peter Zijlstra
maintainercover Rewrite x86/ftrace to use text_poke()
Ftrace was one of the last W^X violators; these patches move it over to the generic text_poke() interface and thereby get rid of this oddity.
Very lightly tested...
Peter Zijlstra
maintainerPATCH 1/3 x86/alternatives: Teach text_poke_bp() to emulate instructions
In preparation for static_call and variable size jump_label support, teach text_poke_bp() to emulate instructions, namely:
JMP32, JMP8, CALL, NOP2, NOP_ATOMIC5
The current text_poke_bp() takes a @handler argument which is used as a jump target when the temporary INT3 is hit by a different CPU.
When patching CALL instructions, this doesn't work because we'd miss the PUSH of the return address. Instead, teach poke_int3_handler() to emulate an instruction, typically the instruction we're patching in.
This fits almost all text_poke_bp() users, except arch_unoptimize_kprobe() which restores random text, and for that site we have to build an explicit emulate instruction.
--- arch/x86/include/asm/text-patching.h | 24 ++++++-- arch/x86/kernel/alternative.c | 98 ++++++++++++++++++++++++++--------- arch/x86/kernel/jump_label.c | 9 +-- arch/x86/kernel/kprobes/opt.c | 11 ++- 4 files changed, 103 insertions(+), 39 deletions(-)
arch/x86/include/asm/text-patching.h +18 -6
@@ -26,10 +26,11 @@ static inline void apply_paravirt(struct#define POKE_MAX_OPCODE_SIZE 5struct text_poke_loc { -void *detour;void *addr; -size_t len; -const char opcode[POKE_MAX_OPCODE_SIZE ]; +int len; +s32 rel32; +u8 opcode; +const char text[POKE_MAX_OPCODE_SIZE ]; };extern void text_poke_early (void *addr,const void *opcode,size_t len);
@@ -51,8 +52,10 @@ extern void text_poke_early(void *addr,extern void *text_poke (void *addr,const void *opcode,size_t len);extern void *text_poke_kgdb (void *addr,const void *opcode,size_t len);extern int poke_int3_handler (struct pt_regs *regs); -extern void text_poke_bp (void *addr,const void *opcode,size_t len,void *handler); +extern void text_poke_bp (void *addr,const void *opcode,size_t len,const void *emulate);extern void text_poke_bp_batch (struct text_poke_loc *tp,unsigned int nr_entries); +extern void text_poke_loc_init (struct text_poke_loc *tp,void *addr, +const void *opcode,size_t len,const void *emulate);extern int after_bootmem;extern __ro_after_init struct mm_struct *poking_mm;extern __ro_after_init unsigned long poking_addr;
@@ -63,8 +66,17 @@ static inline void int3_emulate_jmp(stru regs->ip = ip; } -#define INT3_INSN_SIZE 1 -#define CALL_INSN_SIZE 5 +#define INT3_INSN_SIZE 1 +#define INT3_INSN_OPCODE 0xCC + +#define CALL_INSN_SIZE 5 +#define CALL_INSN_OPCODE 0xE8 + +#define JMP32_INSN_SIZE 5 +#define JMP32_INSN_OPCODE 0xE9 + +#define JMP8_INSN_SIZE 2 +#define JMP8_INSN_OPCODE 0xEBstatic inline void int3_emulate_push (struct pt_regs *regs,unsigned long val) {
arch/x86/kernel/alternative.c +74 -24
155 lines not on this page · raw message
arch/x86/kernel/jump_label.c +3 -6
@@ -89,8 +89,7 @@ static void __ref __jump_label_transformreturn ; } -text_poke_bp ((void *)jump_entry_code (entry), &code,JUMP_LABEL_NOP_SIZE , - (void *)jump_entry_code (entry) +JUMP_LABEL_NOP_SIZE ); +text_poke_bp ((void *)jump_entry_code (entry), &code,JUMP_LABEL_NOP_SIZE ,NULL ); } void arch_jump_label_transform(struct jump_entry *entry,
@@ -147,11 +146,9 @@ bool arch_jump_label_transform_queue(str }__jump_label_set_jump_code (entry, type, - (union jump_code_union *) &tp->opcode,0 ); + (union jump_code_union *)&tp->text,0 ); - tp->addr = entry_code; - tp->detour = entry_code +JUMP_LABEL_NOP_SIZE ; - tp->len =JUMP_LABEL_NOP_SIZE ; +text_poke_loc_init (tp, entry_code,NULL ,JUMP_LABEL_NOP_SIZE ,NULL ); tp_vec_nr++;
arch/x86/kernel/kprobes/opt.c +8 -3
@@ -437,8 +437,7 @@ void arch_optimize_kprobes(struct list_h insn_buff[0 ] =RELATIVEJUMP_OPCODE ; *(s32 *)(&insn_buff[1 ]) = rel; -text_poke_bp (op->kp.addr, insn_buff,RELATIVEJUMP_SIZE , - op->optinsn.insn); +text_poke_bp (op->kp.addr, insn_buff,RELATIVEJUMP_SIZE ,NULL );list_del_init (&op->list); }
@@ -448,12 +447,18 @@ void arch_optimize_kprobes(struct list_hvoid arch_unoptimize_kprobe (struct optimized_kprobe *op) {u8 insn_buff[RELATIVEJUMP_SIZE ]; +u8 emulate_buff[RELATIVEJUMP_SIZE ];/* Set int3 to first byte for kprobes */ insn_buff[0 ] =BREAKPOINT_INSTRUCTION ;memcpy (insn_buff +1 , op->optinsn.copied_insn,RELATIVE_ADDR_SIZE ); + + emulate_buff[0 ] =RELATIVEJUMP_OPCODE ; + *(s32 *)(&emulate_buff[1 ]) = (s32)((long )op->optinsn.insn - + ((long )op->kp.addr +RELATIVEJUMP_SIZE )); +text_poke_bp (op->kp.addr, insn_buff,RELATIVEJUMP_SIZE , - op->optinsn.insn); + emulate_buff); }/*
Peter Zijlstra
maintainerPATCH 2/3 x86/alternatives: Move tp_vec
In reply to Peter ZijlstraIn order to allow other users to make use of the tp_vec; move it near the text_poke_bp_batch() code, instead of keeping it in the jump_label code.
--- arch/x86/include/asm/text-patching.h | 4 ++++ arch/x86/kernel/alternative.c | 3 +++ arch/x86/kernel/jump_label.c | 25 ++++++++++++------------- 3 files changed, 19 insertions(+), 13 deletions(-)
arch/x86/include/asm/text-patching.h +4 -0
@@ -60,6 +60,10 @@ extern int after_bootmem;extern __ro_after_init struct mm_struct *poking_mm;extern __ro_after_init unsigned long poking_addr; +#define TP_VEC_MAX (PAGE_SIZE / sizeof(struct text_poke_loc)) +extern struct text_poke_loc tp_vec[TP_VEC_MAX ]; +extern int tp_vec_nr; +#ifndef CONFIG_UML_X86 static inline void int3_emulate_jmp (struct pt_regs *regs,unsigned long ip) {
arch/x86/kernel/alternative.c +3 -0
@@ -1017,6 +1017,9 @@ int poke_int3_handler(struct pt_regs *re }NOKPROBE_SYMBOL (poke_int3_handler); +struct text_poke_loc tp_vec[TP_VEC_MAX ]; +int tp_vec_nr; +/** * text_poke_bp_batch() -- update instructions on live kernel on SMP * @tp: vector of instructions to patch
arch/x86/kernel/jump_label.c +12 -13
@@ -100,15 +100,12 @@ void arch_jump_label_transform(struct jumutex_unlock (&text_mutex); } -#define TP_VEC_MAX (PAGE_SIZE / sizeof(struct text_poke_loc)) -static struct text_poke_loc tp_vec[TP_VEC_MAX ]; -static int tp_vec_nr; -bool arch_jump_label_transform_queue (struct jump_entry *entry,enum jump_label_type type) {struct text_poke_loc *tp;void *entry_code; +bool ret = true;if (system_state ==SYSTEM_BOOTING ) {/*
@@ -118,12 +115,15 @@ bool arch_jump_label_transform_queue(strreturn true; } +mutex_lock (&text_mutex);/* * No more space in the vector, tell upper layer to apply * the queue before continuing. */ -if (tp_vec_nr ==TP_VEC_MAX ) -return false; +if (tp_vec_nr ==TP_VEC_MAX ) { + ret = false; + goto unlock; + } tp = &tp_vec[tp_vec_nr];
@@ -151,20 +151,19 @@ bool arch_jump_label_transform_queue(strtext_poke_loc_init (tp, entry_code,NULL ,JUMP_LABEL_NOP_SIZE ,NULL ); tp_vec_nr++; +unlock: +mutex_unlock (&text_mutex); -return true; +return ret; }void arch_jump_label_transform_apply (void ) { -if (!tp_vec_nr) -return ; -mutex_lock (&text_mutex); -text_poke_bp_batch (tp_vec, tp_vec_nr); -mutex_unlock (&text_mutex); - +if (tp_vec_nr) +text_poke_bp_batch (tp_vec, tp_vec_nr); tp_vec_nr =0 ; +mutex_unlock (&text_mutex); } static enum {
Peter Zijlstra
maintainerRe: [PATCH 2/3] x86/alternatives: Move tp_vec
arch/x86/include/asm/text-patching.h extern int after_bootmem; · line 65
Quoted 3 lines · Peter Zijlstra
arch/x86/include/asm/text-patching.h +3
-0+#define TP_VEC_MAX (PAGE_SIZE / sizeof(struct text_poke_loc)) +extern struct text_poke_loc tp_vec[TP_VEC_MAX ]; +extern int tp_vec_nr;
FWIW, that currently results in a batch size of 128, but I've not noticed any delay in flipping ftrace on and off on the commandline.
Growing that buffer really shouldn't be a problem, but I'm thinking we'd want solid performance numbers to justify it.
Peter Zijlstra
maintainerRFC PATCH 3/3 x86/ftrace: Use text_poke()
In reply to Peter ZijlstraMove ftrace over to using the generic x86 text_poke functions; this avoids having a second/different copy of that code around.
--- arch/x86/include/asm/ftrace.h | 2 arch/x86/kernel/ftrace.c | 571 +++--------------------------------------- arch/x86/kernel/traps.c | 9 3 files changed, 51 insertions(+), 531 deletions(-)
arch/x86/include/asm/ftrace.h +0 -2
@@ -35,8 +35,6 @@ struct dyn_arch_ftrace { /* No extra data needed for x86 */ }; -int ftrace_int3_handler (struct pt_regs *regs); -#define FTRACE_GRAPH_TRAMP_ADDR FTRACE_GRAPH_ADDR#endif /* CONFIG_DYNAMIC_FTRACE */
arch/x86/kernel/ftrace.c +51 -520
680 lines not on this page · raw message
arch/x86/kernel/traps.c +0 -9
@@ -568,15 +568,6 @@ NOKPROBE_SYMBOL(do_general_protection); dotraplinkage void notrace do_int3(struct pt_regs *regs, long error_code) { -#ifdef CONFIG_DYNAMIC_FTRACE -/* -* ftrace must be first, everything else may cause a recursive crash. -* See note by declaration of modifying_ftrace_code in ftrace.c -*/ -if (unlikely (atomic_read (&modifying_ftrace_code)) && -ftrace_int3_handler (regs)) -return ; -#endif if (poke_int3_handler (regs))return ;
Peter Zijlstra
maintainerRe: [RFC][PATCH 3/3] x86/ftrace: Use text_poke()
Quoted 6 lines · Peter Zijlstra
Move ftrace over to using the generic x86
text_pokefunctions; this avoids having a second/different copy of that code around.Cc Daniel Bristot de Oliveira <[email protected]>Cc Steven Rostedt <[email protected]>Signed-off-by Peter Zijlstra (Intel) <[email protected]>
*sigh*.. one of those days again. I seem to have tested without this patch applied.
Let me go fix this.
Peter Zijlstra
maintainerQuoted 4 lines · Peter Zijlstra
Ftrace was one of the last W^X violators; these patches move it over to the generic
text_poke()interface and thereby get rid of this oddity.Very lightly tested...
I'm thinking there's more cleanup to be had; set_kernel_text_*() can go away now, and I'm thinking the whole batching thing can be further simplified/shared between this and jump_label.