Skip to contentSkip to conversation

jump_label, x86: Support variable sized JMP instructions

RFC PATCH New Peter Zijlstra 1 participant 9 messages Jun 28, 2019

Subsystems (3)
lore
Share & tools
lore

Peter Zijlstra

author maintainer
#0 lore raw

cover jump_label, x86: Support variable sized JMP instructions

After discussing jump_labels with Eugeny and Vineet the other day, I started playing with adding support for our 2 byte JMP/8 instruction. Much thanks to Josh for (re)discovering the .skip trick.

These patches boot an x86_64 defconfig in kvm, so it must be perfect.

Using the 2 byte jumps saves about ~6k text for an x86_64-defconfig build.

author maintainer
#1 lore raw

RFC PATCH 1/8 x86/alternatives: Teach text_poke_bp() to emulate instructions

In preparation for static_call and variable size jump_label support, teach text_poke_bp() to emulate instructions, namely:

JMP32, JMP8, CALL, NOP2, NOP_ATOMIC5

The current text_poke_bp() takes a @handler argument which is used as a jump target when the temporary INT3 is hit by a different CPU.

When patching CALL instructions, this doesn't work because we'd miss the PUSH of the return address. Instead, teach poke_int3_handler() to emulate an instruction, typically the instruction we're patching in.

This fits almost all text_poke_bp() users, except arch_unoptimize_kprobe() which restores random text, and for that site we have to build an explicit emulate instruction.

---
 arch/x86/include/asm/text-patching.h |   24 ++++++--
 arch/x86/kernel/alternative.c        |   98 ++++++++++++++++++++++++++---------
 arch/x86/kernel/jump_label.c         |    9 +--
 arch/x86/kernel/kprobes/opt.c        |   11 ++-
 4 files changed, 103 insertions(+), 39 deletions(-)
arch/x86/include/asm/text-patching.h +18 -6
@@ -26,10 +26,11 @@ static inline void apply_paravirt(struct
 #define POKE_MAX_OPCODE_SIZE	5
 
 struct text_poke_loc {
-	void *detour;
 	void *addr;
-	size_t len;
-	const char opcode[POKE_MAX_OPCODE_SIZE];
+	int len;
+	s32 rel32;
+	u8 opcode;
+	const char text[POKE_MAX_OPCODE_SIZE];
 };
 
 extern void text_poke_early(void *addr, const void *opcode, size_t len);
@@ -51,8 +52,10 @@ extern void text_poke_early(void *addr,
 extern void *text_poke(void *addr, const void *opcode, size_t len);
 extern void *text_poke_kgdb(void *addr, const void *opcode, size_t len);
 extern int poke_int3_handler(struct pt_regs *regs);
-extern void text_poke_bp(void *addr, const void *opcode, size_t len, void *handler);
+extern void text_poke_bp(void *addr, const void *opcode, size_t len, const void *emulate);
 extern void text_poke_bp_batch(struct text_poke_loc *tp, unsigned int nr_entries);
+extern void text_poke_loc_init(struct text_poke_loc *tp, void *addr,
+			       const void *opcode, size_t len, const void *emulate);
 extern int after_bootmem;
 extern __ro_after_init struct mm_struct *poking_mm;
 extern __ro_after_init unsigned long poking_addr;
@@ -63,8 +66,17 @@ static inline void int3_emulate_jmp(stru
 	regs->ip = ip;
 }
 
-#define INT3_INSN_SIZE 1
-#define CALL_INSN_SIZE 5
+#define INT3_INSN_SIZE		1
+#define INT3_INSN_OPCODE	0xCC
+
+#define CALL_INSN_SIZE		5
+#define CALL_INSN_OPCODE	0xE8
+
+#define JMP32_INSN_SIZE		5
+#define JMP32_INSN_OPCODE	0xE9
+
+#define JMP8_INSN_SIZE		2
+#define JMP8_INSN_OPCODE	0xEB
 
 static inline void int3_emulate_push(struct pt_regs *regs, unsigned long val)
 {
arch/x86/kernel/alternative.c +74 -24

155 lines not on this page · raw message

arch/x86/kernel/jump_label.c +3 -6
@@ -89,8 +89,7 @@ static void __ref __jump_label_transform
 		return;
 	}
 
-	text_poke_bp((void *)jump_entry_code(entry), &code, JUMP_LABEL_NOP_SIZE,
-		     (void *)jump_entry_code(entry) + JUMP_LABEL_NOP_SIZE);
+	text_poke_bp((void *)jump_entry_code(entry), &code, JUMP_LABEL_NOP_SIZE, NULL);
 }
 
 void arch_jump_label_transform(struct jump_entry *entry,
@@ -147,11 +146,9 @@ bool arch_jump_label_transform_queue(str
 	}
 
 	__jump_label_set_jump_code(entry, type,
-				   (union jump_code_union *) &tp->opcode, 0);
+				   (union jump_code_union *)&tp->text, 0);
 
-	tp->addr = entry_code;
-	tp->detour = entry_code + JUMP_LABEL_NOP_SIZE;
-	tp->len = JUMP_LABEL_NOP_SIZE;
+	text_poke_loc_init(tp, entry_code, NULL, JUMP_LABEL_NOP_SIZE, NULL);
 
 	tp_vec_nr++;
 
arch/x86/kernel/kprobes/opt.c +8 -3
@@ -437,8 +437,7 @@ void arch_optimize_kprobes(struct list_h
 		insn_buff[0] = RELATIVEJUMP_OPCODE;
 		*(s32 *)(&insn_buff[1]) = rel;
 
-		text_poke_bp(op->kp.addr, insn_buff, RELATIVEJUMP_SIZE,
-			     op->optinsn.insn);
+		text_poke_bp(op->kp.addr, insn_buff, RELATIVEJUMP_SIZE, NULL);
 
 		list_del_init(&op->list);
 	}
@@ -448,12 +447,18 @@ void arch_optimize_kprobes(struct list_h
 void arch_unoptimize_kprobe(struct optimized_kprobe *op)
 {
 	u8 insn_buff[RELATIVEJUMP_SIZE];
+	u8 emulate_buff[RELATIVEJUMP_SIZE];
 
 	/* Set int3 to first byte for kprobes */
 	insn_buff[0] = BREAKPOINT_INSTRUCTION;
 	memcpy(insn_buff + 1, op->optinsn.copied_insn, RELATIVE_ADDR_SIZE);
+
+	emulate_buff[0] = RELATIVEJUMP_OPCODE;
+	*(s32 *)(&emulate_buff[1]) = (s32)((long)op->optinsn.insn -
+			((long)op->kp.addr + RELATIVEJUMP_SIZE));
+
 	text_poke_bp(op->kp.addr, insn_buff, RELATIVEJUMP_SIZE,
-		     op->optinsn.insn);
+		     emulate_buff);
 }
 
 /*
author maintainer
#2 lore raw

RFC PATCH 2/8 jump_label, x86: Strip ASM jump_label support

In reply to Peter Zijlstra

In prepration for variable size jump_label support; remove all ASM bits that are not used.

---
 arch/x86/entry/calling.h          |    2 +-
 arch/x86/include/asm/jump_label.h |   28 ++++------------------------
 2 files changed, 5 insertions(+), 25 deletions(-)
arch/x86/entry/calling.h +1 -1
@@ -377,7 +377,7 @@ For 32-bit we have the following convent
 .macro CALL_enter_from_user_mode
 #ifdef CONFIG_CONTEXT_TRACKING
 #ifdef CONFIG_JUMP_LABEL
-	STATIC_JUMP_IF_FALSE .Lafter_call_\@, context_tracking_enabled, def=0
+	STATIC_BRANCH_FALSE_LIKELY .Lafter_call_\@, context_tracking_enabled
 #endif
 	call enter_from_user_mode
 .Lafter_call_\@:
arch/x86/include/asm/jump_label.h +4 -24
@@ -55,36 +55,16 @@ static __always_inline bool arch_static_
 
 #else	/* __ASSEMBLY__ */
 
-.macro STATIC_JUMP_IF_TRUE target, key, def
+.macro STATIC_BRANCH_FALSE_LIKELY target, key
 .Lstatic_jump_\@:
-	.if \def
 	/* Equivalent to "jmp.d32 \target" */
 	.byte		0xe9
-	.long		\target - .Lstatic_jump_after_\@
-.Lstatic_jump_after_\@:
-	.else
-	.byte		STATIC_KEY_INIT_NOP
-	.endif
-	.pushsection __jump_table, "aw"
-	_ASM_ALIGN
-	.long		.Lstatic_jump_\@ - ., \target - .
-	_ASM_PTR	\key - .
-	.popsection
-.endm
+	.long		\target - (. + 4)
 
-.macro STATIC_JUMP_IF_FALSE target, key, def
-.Lstatic_jump_\@:
-	.if \def
-	.byte		STATIC_KEY_INIT_NOP
-	.else
-	/* Equivalent to "jmp.d32 \target" */
-	.byte		0xe9
-	.long		\target - .Lstatic_jump_after_\@
-.Lstatic_jump_after_\@:
-	.endif
 	.pushsection __jump_table, "aw"
 	_ASM_ALIGN
-	.long		.Lstatic_jump_\@ - ., \target - .
+	.long		.Lstatic_jump_\@ - .
+	.long		\target - .
 	_ASM_PTR	\key + 1 - .
 	.popsection
 .endm
author maintainer
#3 lore raw

RFC PATCH 3/8 jump_label, x86: Factor out the __jump_table generation

In reply to Peter Zijlstra

Both arch_static_branch() and arch_static_branch_jump() have the same blurb to generate the __jump_table entry, share it.

---
 arch/x86/include/asm/jump_label.h |   24 ++++++++++++------------
 1 file changed, 12 insertions(+), 12 deletions(-)
arch/x86/include/asm/jump_label.h +12 -12
@@ -20,15 +20,19 @@
 #include <linux/stringify.h>
 #include <linux/types.h>
 
+#define JUMP_TABLE_ENTRY				\
+	".pushsection __jump_table,  \"aw\" \n\t"	\
+	_ASM_ALIGN "\n\t"				\
+	".long 1b - . \n\t"				\
+	".long %l[l_yes] - . \n\t"			\
+	_ASM_PTR "%c0 + %c1 - .\n\t"			\
+	".popsection \n\t"
+
 static __always_inline bool arch_static_branch(struct static_key *key, bool branch)
 {
 	asm_volatile_goto("1:"
 		".byte " __stringify(STATIC_KEY_INIT_NOP) "\n\t"
-		".pushsection __jump_table,  \"aw\" \n\t"
-		_ASM_ALIGN "\n\t"
-		".long 1b - ., %l[l_yes] - . \n\t"
-		_ASM_PTR "%c0 + %c1 - .\n\t"
-		".popsection \n\t"
+		JUMP_TABLE_ENTRY
 		: :  "i" (key), "i" (branch) : : l_yes);
 
 	return false;
@@ -39,13 +43,9 @@ static __always_inline bool arch_static_
 static __always_inline bool arch_static_branch_jump(struct static_key *key, bool branch)
 {
 	asm_volatile_goto("1:"
-		".byte 0xe9\n\t .long %l[l_yes] - 2f\n\t"
-		"2:\n\t"
-		".pushsection __jump_table,  \"aw\" \n\t"
-		_ASM_ALIGN "\n\t"
-		".long 1b - ., %l[l_yes] - . \n\t"
-		_ASM_PTR "%c0 + %c1 - .\n\t"
-		".popsection \n\t"
+		".byte 0xe9 \n\t"
+		".long %l[l_yes] - (. + 4) \n\t"
+		JUMP_TABLE_ENTRY
 		: :  "i" (key), "i" (branch) : : l_yes);
 
 	return false;
author maintainer
#4 lore raw

RFC PATCH 4/8 jump_label, x86: Remove init NOP optimization

In reply to Peter Zijlstra

Instead of checking if the emitted (default) NOP is the ideal NOP, and conditionally rewrite the NOP, just rewrite the NOP.

This shouldn't be a problem because init / module_load uses text_poke_early() which is cheap and this saves us from having to go figure out which NOP to compare against when we go variable size.

---
 arch/x86/kernel/jump_label.c |   19 ++++---------------
 1 file changed, 4 insertions(+), 15 deletions(-)
arch/x86/kernel/jump_label.c +4 -15
@@ -177,21 +177,10 @@ __init_or_module void arch_jump_label_tr
 				      enum jump_label_type type)
 {
 	/*
-	 * This function is called at boot up and when modules are
-	 * first loaded. Check if the default nop, the one that is
-	 * inserted at compile time, is the ideal nop. If it is, then
-	 * we do not need to update the nop, and we can leave it as is.
-	 * If it is not, then we need to update the nop to the ideal nop.
+	 * Rewrite the NOP on init / module-load to ensure we got the ideal
+	 * nop.  Don't bother with trying to figure out what size and what nop
+	 * it should be for now, simply do an unconditional rewrite.
 	 */
-	if (jlstate == JL_STATE_START) {
-		const unsigned char default_nop[] = { STATIC_KEY_INIT_NOP };
-		const unsigned char *ideal_nop = ideal_nops[NOP_ATOMIC5];
-
-		if (memcmp(ideal_nop, default_nop, 5) != 0)
-			jlstate = JL_STATE_UPDATE;
-		else
-			jlstate = JL_STATE_NO_UPDATE;
-	}
-	if (jlstate == JL_STATE_UPDATE)
+	if (jlstate == JL_STATE_UPDATE || jlstate == JL_STATE_START)
 		__jump_label_transform(entry, type, 1);
 }
author maintainer
#5 lore raw

RFC PATCH 5/8 jump_label, x86: Improve error when we fail expected text

In reply to Peter Zijlstra

There is only a single usage site left, remove the function and extend the print to include more information, like the expected text and the patch type.

---
 arch/x86/kernel/jump_label.c |   24 +++++++++++-------------
 1 file changed, 11 insertions(+), 13 deletions(-)
arch/x86/kernel/jump_label.c +11 -13
@@ -24,17 +24,6 @@ union jump_code_union {
 	} __attribute__((packed));
 };
 
-static void bug_at(unsigned char *ip, int line)
-{
-	/*
-	 * The location is not an op that we were expecting.
-	 * Something went wrong. Crash the box, as something could be
-	 * corrupting the kernel.
-	 */
-	pr_crit("jump_label: Fatal kernel bug, unexpected op at %pS [%p] (%5ph) %d\n", ip, ip, ip, line);
-	BUG();
-}
-
 static void __jump_label_set_jump_code(struct jump_entry *entry,
 				       enum jump_label_type type,
 				       union jump_code_union *code,
@@ -42,6 +31,7 @@ static void __jump_label_set_jump_code(s
 {
 	const unsigned char default_nop[] = { STATIC_KEY_INIT_NOP };
 	const unsigned char *ideal_nop = ideal_nops[NOP_ATOMIC5];
+	unsigned char *ip = (void *)jump_entry_code(entry);
 	const void *expect;
 	int line;
 
@@ -57,8 +47,16 @@ static void __jump_label_set_jump_code(s
 		expect = code->code; line = __LINE__;
 	}
 
-	if (memcmp((void *)jump_entry_code(entry), expect, JUMP_LABEL_NOP_SIZE))
-		bug_at((void *)jump_entry_code(entry), line);
+	if (memcmp(ip, expect, JUMP_LABEL_NOP_SIZE)) {
+		/*
+		 * The location is not an op that we were expecting.
+		 * Something went wrong. Crash the box, as something could be
+		 * corrupting the kernel.
+		 */
+		pr_crit("jump_label: Fatal kernel bug, unexpected op at %pS [%p] (%5ph != %5ph)) line:%d init:%d type:%d\n",
+				ip, ip, ip, expect, line, init, type);
+		BUG();
+	}
 
 	if (type == JUMP_LABEL_NOP)
 		memcpy(code, ideal_nop, JUMP_LABEL_NOP_SIZE);
author maintainer
#6 lore raw

RFC PATCH 6/8 jump_label, x86: Add variable length patching support

In reply to Peter Zijlstra

This allows the patching to to emit 2 byte JMP/NOP instruction in addition to the 5 byte JMP/NOP we already did. This allows for more compact code.

This code is not yet used, as we don't emit shorter code at compile time yet.

---
 arch/x86/include/asm/jump_label.h |    6 +-
 arch/x86/include/asm/nops.h       |    1 
 arch/x86/kernel/jump_label.c      |   77 +++++++++++++++++++++++---------------
 3 files changed, 53 insertions(+), 31 deletions(-)
arch/x86/include/asm/jump_label.h +4 -2
@@ -7,9 +7,11 @@
 #define JUMP_LABEL_NOP_SIZE 5
 
 #ifdef CONFIG_X86_64
-# define STATIC_KEY_INIT_NOP P6_NOP5_ATOMIC
+# define STATIC_KEY_NOP2 P6_NOP2
+# define STATIC_KEY_NOP5 P6_NOP5_ATOMIC
 #else
-# define STATIC_KEY_INIT_NOP GENERIC_NOP5_ATOMIC
+# define STATIC_KEY_NOP2 GENERIC_NOP2
+# define STATIC_KEY_NOP5 GENERIC_NOP5_ATOMIC
 #endif
 
 #include <asm/asm.h>
arch/x86/include/asm/nops.h +1 -0
@@ -5,6 +5,7 @@
 /*
  * Define nops for use with alternative() and for tracing.
  *
+ * *_NOP2 must be a single instruction
  * *_NOP5_ATOMIC must be a single instruction.
  */
 
arch/x86/kernel/jump_label.c +48 -29

133 lines not on this page · raw message

author maintainer
#7 lore raw

RFC PATCH 7/8 jump_label, x86: Introduce jump_entry_size()

In reply to Peter Zijlstra

This allows architectures to have variable sized jumps.

---
 arch/x86/include/asm/jump_label.h |    4 ++--
 arch/x86/kernel/jump_label.c      |    8 ++++++++
 include/linux/jump_label.h        |    9 +++++++++
 kernel/jump_label.c               |    2 +-
 4 files changed, 20 insertions(+), 3 deletions(-)
arch/x86/include/asm/jump_label.h +2 -2
@@ -4,8 +4,6 @@
 
 #define HAVE_JUMP_LABEL_BATCH
 
-#define JUMP_LABEL_NOP_SIZE 5
-
 #ifdef CONFIG_X86_64
 # define STATIC_KEY_NOP2 P6_NOP2
 # define STATIC_KEY_NOP5 P6_NOP5_ATOMIC
@@ -55,6 +53,8 @@ static __always_inline bool arch_static_
 	return true;
 }
 
+extern int arch_jump_entry_size(struct jump_entry *entry);
+
 #else	/* __ASSEMBLY__ */
 
 .macro STATIC_BRANCH_FALSE_LIKELY target, key
arch/x86/kernel/jump_label.c +8 -0
@@ -32,6 +32,14 @@ static inline bool __jump_disp_is_byte(s
 	return false;
 }
 
+int arch_jump_entry_size(struct jump_entry *entry)
+{
+	s32 disp = jump_entry_target(entry) - jump_entry_code(entry);
+	if (__jump_disp_is_byte(disp))
+		return JMP8_INSN_SIZE;
+	return JMP32_INSN_SIZE;
+}
+
 static int __jump_label_set_jump_code(struct jump_entry *entry,
 				      enum jump_label_type type,
 				      union jump_code_union *code,
include/linux/jump_label.h +9 -0
@@ -176,6 +176,15 @@ static inline void jump_entry_set_init(s
 	entry->key |= 2;
 }
 
+static inline int jump_entry_size(struct jump_entry *entry)
+{
+#ifdef JUMP_LABEL_NOP_SIZE
+	return JUMP_LABEL_NOP_SIZE;
+#else
+	return arch_jump_entry_size(entry);
+#endif
+}
+
 #endif
 #endif
 
kernel/jump_label.c +1 -1
@@ -309,7 +309,7 @@ EXPORT_SYMBOL_GPL(jump_label_rate_limit)
 static int addr_conflict(struct jump_entry *entry, void *start, void *end)
 {
 	if (jump_entry_code(entry) <= (unsigned long)end &&
-	    jump_entry_code(entry) + JUMP_LABEL_NOP_SIZE > (unsigned long)start)
+	    jump_entry_code(entry) + jump_entry_size(entry) > (unsigned long)start)
 		return 1;
 
 	return 0;
author maintainer
#8 lore raw

RFC PATCH 8/8 jump_label, x86: Enable JMP8/NOP2 support

In reply to Peter Zijlstra

Enable and emit short JMP/NOP jump_label entries.

Much thanks to Josh for (re)discovering the .skip trick to conditionally emit variable length text.

Due to how early we enable jump_labels on x86, if any of this comes apart, the machine is completely dead. Qemu+GDB saved the day this time.

---
 arch/x86/include/asm/jump_label.h |   37 +++++++++++++++++++++++++++++++------
 arch/x86/kernel/jump_label.c      |    5 ++++-
 2 files changed, 35 insertions(+), 7 deletions(-)
arch/x86/include/asm/jump_label.h +31 -6
@@ -31,7 +31,35 @@
 static __always_inline bool arch_static_branch(struct static_key *key, bool branch)
 {
 	asm_volatile_goto("1:"
-		".byte " __stringify(STATIC_KEY_INIT_NOP) "\n\t"
+
+		".set disp, %l[l_yes] - (1b + 2) \n\t"
+		".set sign, disp >> 31 \n\t"
+		".set res, (disp >> 7) ^ sign \n\t"
+		".set is_byte, -(res == 0) \n\t"
+		".set is_long, -(res != 0) \n\t"
+
+#ifdef CONFIG_X86_64
+		".skip is_byte, 0x66 \n\t"
+		".skip is_byte, 0x90 \n\t"
+#else
+		".skip is_byte, 0x89 \n\t"
+		".skip is_byte, 0xf6 \n\t"
+#endif
+
+#ifdef CONFIG_X86_64
+		".skip is_long, 0x0f \n\t"
+		".skip is_long, 0x1f \n\t"
+		".skip is_long, 0x44 \n\t"
+		".skip is_long, 0x00 \n\t"
+		".skip is_long, 0x00 \n\t"
+#else
+		".skip is_long, 0x3e \n\t"
+		".skip is_long, 0x8d \n\t"
+		".skip is_long, 0x74 \n\t"
+		".skip is_long, 0x26 \n\t"
+		".skip is_long, 0x00 \n\t"
+#endif
+
 		JUMP_TABLE_ENTRY
 		: :  "i" (key), "i" (branch) : : l_yes);
 
@@ -43,8 +71,7 @@ static __always_inline bool arch_static_
 static __always_inline bool arch_static_branch_jump(struct static_key *key, bool branch)
 {
 	asm_volatile_goto("1:"
-		".byte 0xe9 \n\t"
-		".long %l[l_yes] - (. + 4) \n\t"
+		"jmp %l[l_yes] \n\t"
 		JUMP_TABLE_ENTRY
 		: :  "i" (key), "i" (branch) : : l_yes);
 
@@ -59,9 +86,7 @@ extern int arch_jump_entry_size(struct j
 
 .macro STATIC_BRANCH_FALSE_LIKELY target, key
 .Lstatic_jump_\@:
-	/* Equivalent to "jmp.d32 \target" */
-	.byte		0xe9
-	.long		\target - (. + 4)
+	jmp \target
 
 	.pushsection __jump_table, "aw"
 	_ASM_ALIGN
arch/x86/kernel/jump_label.c +4 -1
@@ -29,7 +29,10 @@ union jump_code_union {
 
 static inline bool __jump_disp_is_byte(s32 disp)
 {
-	return false;
+	s32 sign;
+	disp -= JMP8_INSN_SIZE;
+	sign = disp >> 31;
+	return ((disp >> 7) ^ sign) == 0;
 }
 
 int arch_jump_entry_size(struct jump_entry *entry)