Skip to contentSkip to conversation

x86 cleanups and static_call()

PATCH In review Peter Zijlstra 10 participants 87 messages Jun 5, 2019 – Oct 2, 2019

Subsystems (12)
lore
Share & tools
lore

Peter Zijlstra

author maintainer
#0 lore raw

cover x86 cleanups and static_call()

Hi!

Now that all the x86_64 int3_emulate_call() stuff is upstream, here are the i386 patches and all the rest of the cleanups that resulted from that discussion.

And I figured I should have a go at making that static_call() thing working now that we have the prerequisites sorted. This is mostly a combination of the v2 and v3 static_call() patches done on top of an 'enhanced' text_poke_bp().

I wrote a little self-test and added Steve's ftrace convertion on top for testing, and suspicously I've not had it explode, so I'm sure this is going to set all your computers on fire.

Enjoy!

author maintainer
#1 lore raw

PATCH 1/15 x86/entry/32: Clean up return from interrupt preemption path

The code flow around the return from interrupt preemption point seems needlesly complicated.

There is only one site jumping to resume_kernel, and none (outside of resume_kernel) jumping to restore_all_kernel. Inline resume_kernel in restore_all_kernel and avoid the CONFIG_PREEMPT dependent label.

---
 arch/x86/entry/entry_32.S |   24 ++++++++++--------------
 1 file changed, 10 insertions(+), 14 deletions(-)
arch/x86/entry/entry_32.S +10 -14
@@ -67,7 +67,6 @@
 # define preempt_stop(clobbers)	DISABLE_INTERRUPTS(clobbers); TRACE_IRQS_OFF
 #else
 # define preempt_stop(clobbers)
-# define resume_kernel		restore_all_kernel
 #endif
 
 .macro TRACE_IRQS_IRET
@@ -755,7 +754,7 @@ END(ret_from_fork)
 	andl	$SEGMENT_RPL_MASK, %eax
 #endif
 	cmpl	$USER_RPL, %eax
-	jb	resume_kernel			# not returning to v8086 or userspace
+	jb	restore_all_kernel		# not returning to v8086 or userspace
 
 ENTRY(resume_userspace)
 	DISABLE_INTERRUPTS(CLBR_ANY)
@@ -765,18 +764,6 @@ ENTRY(resume_userspace)
 	jmp	restore_all
 END(ret_from_exception)
 
-#ifdef CONFIG_PREEMPT
-ENTRY(resume_kernel)
-	DISABLE_INTERRUPTS(CLBR_ANY)
-	cmpl	$0, PER_CPU_VAR(__preempt_count)
-	jnz	restore_all_kernel
-	testl	$X86_EFLAGS_IF, PT_EFLAGS(%esp)	# interrupts off (exception path) ?
-	jz	restore_all_kernel
-	call	preempt_schedule_irq
-	jmp	restore_all_kernel
-END(resume_kernel)
-#endif
-
 GLOBAL(__begin_SYSENTER_singlestep_region)
 /*
  * All code from here through __end_SYSENTER_singlestep_region is subject
@@ -1027,6 +1014,15 @@ ENTRY(entry_INT80_32)
 	INTERRUPT_RETURN
 
 restore_all_kernel:
+#ifdef CONFIG_PREEMPT
+	DISABLE_INTERRUPTS(CLBR_ANY)
+	cmpl	$0, PER_CPU_VAR(__preempt_count)
+	jnz	.Lno_preempt
+	testl	$X86_EFLAGS_IF, PT_EFLAGS(%esp)	# interrupts off (exception path) ?
+	jz	.Lno_preempt
+	call	preempt_schedule_irq
+.Lno_preempt:
+#endif
 	TRACE_IRQS_IRET
 	PARANOID_EXIT_TO_KERNEL_MODE
 	BUG_IF_WRONG_CR3
Reviewed-by on v1 1/15
#2 lore raw

Re: [PATCH 01/15] x86/entry/32: Clean up return from interrupt preemption path

Quoted 2 lines · Peter Zijlstra

The code flow around the return from interrupt preemption point seems needlesly complicated.

"needlessly"

Quoted 6 lines

There is only one site jumping to resume_kernel, and none (outside of resume_kernel) jumping to restore_all_kernel. Inline resume_kernel in restore_all_kernel and avoid the CONFIG_PREEMPT dependent label.

Signature · 1 line
Josh
author maintainer
#3 lore raw

PATCH 2/15 x86: Move ENCODE_FRAME_POINTER to asm/frame.h

In reply to Peter Zijlstra

In preparation for wider use, move the ENCODE_FRAME_POINTER macros to a common header and provide inline asm versions.

These macros are used to encode a pt_regs frame for the unwinder; see unwind_frame.c:decode_frame_pointer().

---
 arch/x86/entry/calling.h     |   15 -------------
 arch/x86/entry/entry_32.S    |   16 --------------
 arch/x86/include/asm/frame.h |   49 +++++++++++++++++++++++++++++++++++++++++++
 3 files changed, 49 insertions(+), 31 deletions(-)
arch/x86/entry/calling.h +0 -15
@@ -172,21 +172,6 @@ For 32-bit we have the following convent
 	.endif
 .endm
 
-/*
- * This is a sneaky trick to help the unwinder find pt_regs on the stack.  The
- * frame pointer is replaced with an encoded pointer to pt_regs.  The encoding
- * is just setting the LSB, which makes it an invalid stack address and is also
- * a signal to the unwinder that it's a pt_regs pointer in disguise.
- *
- * NOTE: This macro must be used *after* PUSH_AND_CLEAR_REGS because it corrupts
- * the original rbp.
- */
-.macro ENCODE_FRAME_POINTER ptregs_offset=0
-#ifdef CONFIG_FRAME_POINTER
-	leaq 1+\ptregs_offset(%rsp), %rbp
-#endif
-.endm
-
 #ifdef CONFIG_PAGE_TABLE_ISOLATION
 
 /*
arch/x86/entry/entry_32.S +0 -16
@@ -246,22 +246,6 @@
 .Lend_\@:
 .endm
 
-/*
- * This is a sneaky trick to help the unwinder find pt_regs on the stack.  The
- * frame pointer is replaced with an encoded pointer to pt_regs.  The encoding
- * is just clearing the MSB, which makes it an invalid stack address and is also
- * a signal to the unwinder that it's a pt_regs pointer in disguise.
- *
- * NOTE: This macro must be used *after* SAVE_ALL because it corrupts the
- * original rbp.
- */
-.macro ENCODE_FRAME_POINTER
-#ifdef CONFIG_FRAME_POINTER
-	mov %esp, %ebp
-	andl $0x7fffffff, %ebp
-#endif
-.endm
-
 .macro RESTORE_INT_REGS
 	popl	%ebx
 	popl	%ecx
arch/x86/include/asm/frame.h +49 -0

67 lines not on this page · raw message

Reviewed-by on v1 2/15
#4 lore raw

Re: [PATCH 02/15] x86: Move ENCODE_FRAME_POINTER to asm/frame.h

Quoted 7 lines · Peter Zijlstra

In preparation for wider use, move the ENCODE_FRAME_POINTER macros to a common header and provide inline asm versions.

These macros are used to encode a pt_regs frame for the unwinder; see unwind_frame.c:decode_frame_pointer().

Signature · 1 line
Josh
author maintainer
#5 lore raw

PATCH 3/15 x86/kprobes: Fix frame pointer annotations

In reply to Peter Zijlstra

The kprobe trampolines have a FRAME_POINTER annotation that makes no sense. It marks the frame in the middle of pt_regs, at the place of saving BP.

Change it to mark the pt_regs frame as per the ENCODE_FRAME_POINTER from the respective entry_*.S.

---
 arch/x86/kernel/kprobes/common.h |   24 ++++++++++++------------
 1 file changed, 12 insertions(+), 12 deletions(-)
arch/x86/kernel/kprobes/common.h +12 -12
@@ -5,15 +5,10 @@
 /* Kprobes and Optprobes common header */
 
 #include <asm/asm.h>
-
-#ifdef CONFIG_FRAME_POINTER
-# define SAVE_RBP_STRING "	push %" _ASM_BP "\n" \
-			 "	mov  %" _ASM_SP ", %" _ASM_BP "\n"
-#else
-# define SAVE_RBP_STRING "	push %" _ASM_BP "\n"
-#endif
+#include <asm/frame.h>
 
 #ifdef CONFIG_X86_64
+
 #define SAVE_REGS_STRING			\
 	/* Skip cs, ip, orig_ax. */		\
 	"	subq $24, %rsp\n"		\
@@ -27,11 +22,13 @@
 	"	pushq %r10\n"			\
 	"	pushq %r11\n"			\
 	"	pushq %rbx\n"			\
-	SAVE_RBP_STRING				\
+	"	pushq %rbp\n"			\
 	"	pushq %r12\n"			\
 	"	pushq %r13\n"			\
 	"	pushq %r14\n"			\
-	"	pushq %r15\n"
+	"	pushq %r15\n"			\
+	ENCODE_FRAME_POINTER
+
 #define RESTORE_REGS_STRING			\
 	"	popq %r15\n"			\
 	"	popq %r14\n"			\
@@ -51,19 +48,22 @@
 	/* Skip orig_ax, ip, cs */		\
 	"	addq $24, %rsp\n"
 #else
+
 #define SAVE_REGS_STRING			\
 	/* Skip cs, ip, orig_ax and gs. */	\
-	"	subl $16, %esp\n"		\
+	"	subl $4*4, %esp\n"		\
 	"	pushl %fs\n"			\
 	"	pushl %es\n"			\
 	"	pushl %ds\n"			\
 	"	pushl %eax\n"			\
-	SAVE_RBP_STRING				\
+	"	pushl %ebp\n"			\
 	"	pushl %edi\n"			\
 	"	pushl %esi\n"			\
 	"	pushl %edx\n"			\
 	"	pushl %ecx\n"			\
-	"	pushl %ebx\n"
+	"	pushl %ebx\n"			\
+	ENCODE_FRAME_POINTER
+
 #define RESTORE_REGS_STRING			\
 	"	popl %ebx\n"			\
 	"	popl %ecx\n"			\
maintainer
#6 lore raw

Re: [PATCH 03/15] x86/kprobes: Fix frame pointer annotations

Quoted 3 lines · On Wed, 05 Jun 2019 15:07:56 +0200 Peter Zijlstra <[email protected]> wrote:

The kprobe trampolines have a FRAME_POINTER annotation that makes no sense. It marks the frame in the middle of pt_regs, at the place of saving BP.

commit ee213fc72fd67 introduced this code, and this is for unwinder which uses frame pointer. I think current code stores the address of previous (original context's) frame pointer into %rbp. So with that, if unwinder tries to decode frame pointer, it can get the original %rbp value, instead of &pt_regs from current %rbp.

Quoted 4 lines

Change it to mark the pt_regs frame as per the ENCODE_FRAME_POINTER from the respective entry_*.S.

With this change, I think stack unwinder can not get the original %rbp value. Peter, could you check the above commit?

Thank you,

Quoted 69 lines
---
 arch/x86/kernel/kprobes/common.h |   24 ++++++++++++------------
 1 file changed, 12 insertions(+), 12 deletions(-)
arch/x86/kernel/kprobes/common.h +12 -12
@@ -5,15 +5,10 @@
 /* Kprobes and Optprobes common header */
 
 #include <asm/asm.h>
-
-#ifdef CONFIG_FRAME_POINTER
-# define SAVE_RBP_STRING "	push %" _ASM_BP "\n" \
-			 "	mov  %" _ASM_SP ", %" _ASM_BP "\n"
-#else
-# define SAVE_RBP_STRING "	push %" _ASM_BP "\n"
-#endif
+#include <asm/frame.h>
 
 #ifdef CONFIG_X86_64
+
 #define SAVE_REGS_STRING			\
 	/* Skip cs, ip, orig_ax. */		\
 	"	subq $24, %rsp\n"		\
@@ -27,11 +22,13 @@
 	"	pushq %r10\n"			\
 	"	pushq %r11\n"			\
 	"	pushq %rbx\n"			\
-	SAVE_RBP_STRING				\
+	"	pushq %rbp\n"			\
 	"	pushq %r12\n"			\
 	"	pushq %r13\n"			\
 	"	pushq %r14\n"			\
-	"	pushq %r15\n"
+	"	pushq %r15\n"			\
+	ENCODE_FRAME_POINTER
+
 #define RESTORE_REGS_STRING			\
 	"	popq %r15\n"			\
 	"	popq %r14\n"			\
@@ -51,19 +48,22 @@
 	/* Skip orig_ax, ip, cs */		\
 	"	addq $24, %rsp\n"
 #else
+
 #define SAVE_REGS_STRING			\
 	/* Skip cs, ip, orig_ax and gs. */	\
-	"	subl $16, %esp\n"		\
+	"	subl $4*4, %esp\n"		\
 	"	pushl %fs\n"			\
 	"	pushl %es\n"			\
 	"	pushl %ds\n"			\
 	"	pushl %eax\n"			\
-	SAVE_RBP_STRING				\
+	"	pushl %ebp\n"			\
 	"	pushl %edi\n"			\
 	"	pushl %esi\n"			\
 	"	pushl %edx\n"			\
 	"	pushl %ecx\n"			\
-	"	pushl %ebx\n"
+	"	pushl %ebx\n"			\
+	ENCODE_FRAME_POINTER
+
 #define RESTORE_REGS_STRING			\
 	"	popl %ebx\n"			\
 	"	popl %ecx\n"			\
Signature · 1 line
Masami Hiramatsu <[email protected]>
Reviewed-by
#7 lore raw

Re: [PATCH 03/15] x86/kprobes: Fix frame pointer annotations

Quoted 14 lines · Masami Hiramatsu
Quoted 3 lines · On Wed, 05 Jun 2019 15:07:56 +0200 Peter Zijlstra <[email protected]> wrote:

The kprobe trampolines have a FRAME_POINTER annotation that makes no sense. It marks the frame in the middle of pt_regs, at the place of saving BP.

commit ee213fc72fd67 introduced this code, and this is for unwinder which uses frame pointer. I think current code stores the address of previous (original context's) frame pointer into %rbp. So with that, if unwinder tries to decode frame pointer, it can get the original %rbp value, instead of &pt_regs from current %rbp.

Quoted 1 line
Quoted 6 lines
Quoted 3 lines

Change it to mark the pt_regs frame as per the ENCODE_FRAME_POINTER from the respective entry_*.S.

With this change, I think stack unwinder can not get the original %rbp value. Peter, could you check the above commit?

The unwinder knows how to decode the encoded frame pointer. So it can find regs by decoding the new rbp value, and it also knows that regs->bp is the original rbp value.

Signature · 1 line
Josh
maintainer Acked-by
#8 lore raw

Re: [PATCH 03/15] x86/kprobes: Fix frame pointer annotations

Quoted 19 lines · On Fri, 7 Jun 2019 09:36:02 -0400 Josh Poimboeuf <[email protected]> wrote:
Quoted 18 lines · Masami Hiramatsu
Quoted 3 lines · On Wed, 05 Jun 2019 15:07:56 +0200 Peter Zijlstra <[email protected]> wrote:

The kprobe trampolines have a FRAME_POINTER annotation that makes no sense. It marks the frame in the middle of pt_regs, at the place of saving BP.

commit ee213fc72fd67 introduced this code, and this is for unwinder which uses frame pointer. I think current code stores the address of previous (original context's) frame pointer into %rbp. So with that, if unwinder tries to decode frame pointer, it can get the original %rbp value, instead of &pt_regs from current %rbp.

Quoted 4 lines

Change it to mark the pt_regs frame as per the ENCODE_FRAME_POINTER from the respective entry_*.S.

Quoted 9 lines
Quoted 2 lines

With this change, I think stack unwinder can not get the original %rbp value. Peter, could you check the above commit?

The unwinder knows how to decode the encoded frame pointer. So it can find regs by decoding the new rbp value, and it also knows that regs->bp is the original rbp value.

Ah, OK. My misunderstood. So this encode framepointer as same as other interrupt entry stack.
Then, it looks good to me too.

Thank you Josh!

Quoted 2 lines
Signature · 1 line
Josh
Signature · 1 line
Masami Hiramatsu <[email protected]>
author maintainer
#9 lore raw

Re: [PATCH 03/15] x86/kprobes: Fix frame pointer annotations

In reply to Josh Poimboeuf
Quoted 8 lines · Josh Poimboeuf
Quoted 7 lines · Masami Hiramatsu
Quoted 3 lines · On Wed, 05 Jun 2019 15:07:56 +0200 Peter Zijlstra <[email protected]> wrote:

The kprobe trampolines have a FRAME_POINTER annotation that makes no sense. It marks the frame in the middle of pt_regs, at the place of saving BP.

Quoted 5 lines
Quoted 5 lines

commit ee213fc72fd67 introduced this code, and this is for unwinder which uses frame pointer. I think current code stores the address of previous (original context's) frame pointer into %rbp. So with that, if unwinder tries to decode frame pointer, it can get the original %rbp value, instead of &pt_regs from current %rbp.

The way I read that code is that we'll put the value of SP into BP at the point where we've done 'PUSH BP', which is right in the middle of that PUSH sequence. So while it works for a FP based unwinder, it doesn't 'properly' identify the current frame.

Quoted 10 lines
Quoted 6 lines
Quoted 3 lines

Change it to mark the pt_regs frame as per the ENCODE_FRAME_POINTER from the respective entry_*.S.

With this change, I think stack unwinder can not get the original %rbp value. Peter, could you check the above commit?

The unwinder knows how to decode the encoded frame pointer. So it can find regs by decoding the new rbp value, and it also knows that regs->bp is the original rbp value.

Right, as Josh says the unwinder has a special case for this and it knows these 'odd' BP values (either MSB or LSB set) indicate a pt_regs set.

author maintainer
#10 lore raw

PATCH 4/15 x86/ftrace: Add pt_regs frame annotations

In reply to Peter Zijlstra

When CONFIG_FRAME_POINTER, we should mark pt_regs frames.

---
 arch/x86/kernel/ftrace_32.S |    3 +++
 arch/x86/kernel/ftrace_64.S |    3 +++
 2 files changed, 6 insertions(+)
arch/x86/kernel/ftrace_32.S +3 -0
@@ -9,6 +9,7 @@
 #include <asm/export.h>
 #include <asm/ftrace.h>
 #include <asm/nospec-branch.h>
+#include <asm/frame.h>
 
 # define function_hook	__fentry__
 EXPORT_SYMBOL(__fentry__)
@@ -116,6 +117,8 @@ ENTRY(ftrace_regs_caller)
 	pushl	%ecx
 	pushl	%ebx
 
+	ENCODE_FRAME_POINTER
+
 	movl	12*4(%esp), %eax		/* Load ip (1st parameter) */
 	subl	$MCOUNT_INSN_SIZE, %eax		/* Adjust ip */
 	movl	15*4(%esp), %edx		/* Load parent ip (2nd parameter) */
arch/x86/kernel/ftrace_64.S +3 -0
@@ -9,6 +9,7 @@
 #include <asm/export.h>
 #include <asm/nospec-branch.h>
 #include <asm/unwind_hints.h>
+#include <asm/frame.h>
 
 	.code64
 	.section .entry.text, "ax"
@@ -203,6 +204,8 @@ GLOBAL(ftrace_regs_caller_op_ptr)
 	leaq MCOUNT_REG_SIZE+8*2(%rsp), %rcx
 	movq %rcx, RSP(%rsp)
 
+	ENCODE_FRAME_POINTER
+
 	/* regs go into 4th parameter */
 	leaq (%rsp), %rcx
 
author maintainer
#12 lore raw

PATCH 5/15 x86_32: Provide consistent pt_regs

In reply to Peter Zijlstra

Currently pt_regs on x86_32 has an oddity in that kernel regs (!user_mode(regs)) are short two entries (esp/ss). This means that any code trying to use them (typically: regs->sp) needs to jump through some unfortunate hoops.

Change the entry code to fix this up and create a full pt_regs frame.

This then simplifies various trampolines in ftrace and kprobes, the stack unwinder, ptrace, kdump and kgdb.

---
 arch/x86/entry/entry_32.S         |  105 ++++++++++++++++++++++++++++++++++----
 arch/x86/include/asm/kexec.h      |   17 ------
 arch/x86/include/asm/ptrace.h     |   17 ------
 arch/x86/include/asm/stacktrace.h |    2 
 arch/x86/kernel/crash.c           |    8 --
 arch/x86/kernel/ftrace_32.S       |   77 +++++++++++++++------------
 arch/x86/kernel/kgdb.c            |    8 --
 arch/x86/kernel/kprobes/common.h  |    4 -
 arch/x86/kernel/kprobes/core.c    |   29 ++++------
 arch/x86/kernel/kprobes/opt.c     |   20 ++++---
 arch/x86/kernel/process_32.c      |   16 +----
 arch/x86/kernel/ptrace.c          |   29 ----------
 arch/x86/kernel/time.c            |    3 -
 arch/x86/kernel/unwind_frame.c    |   32 +----------
 arch/x86/kernel/unwind_orc.c      |    2 
 15 files changed, 178 insertions(+), 191 deletions(-)
arch/x86/entry/entry_32.S +95 -10
@@ -202,9 +202,102 @@
 .Lend_\@:
 .endm
 
+#define CS_FROM_ENTRY_STACK	(1 << 31)
+#define CS_FROM_USER_CR3	(1 << 30)
+#define CS_FROM_KERNEL		(1 << 29)
+
+.macro FIXUP_FRAME
+	/*
+	 * The high bits of the CS dword (__csh) are used for CS_FROM_*.
+	 * Clear them in case hardware didn't do this for us.
+	 */
+	andl	$0x0000ffff, 3*4(%esp)
+
+#ifdef CONFIG_VM86
+	testl	$X86_EFLAGS_VM, 4*4(%esp)
+	jnz	.Lfrom_usermode_no_fixup_\@
+#endif
+	testl	$SEGMENT_RPL_MASK, 3*4(%esp)
+	jnz	.Lfrom_usermode_no_fixup_\@
+
+	orl	$CS_FROM_KERNEL, 3*4(%esp)
+
+	/*
+	 * When we're here from kernel mode; the (exception) stack looks like:
+	 *
+	 *  5*4(%esp) - <previous context>
+	 *  4*4(%esp) - flags
+	 *  3*4(%esp) - cs
+	 *  2*4(%esp) - ip
+	 *  1*4(%esp) - orig_eax
+	 *  0*4(%esp) - gs / function
+	 *
+	 * Lets build a 5 entry IRET frame after that, such that struct pt_regs
+	 * is complete and in particular regs->sp is correct. This gives us
+	 * the original 5 enties as gap:
+	 *
+	 * 12*4(%esp) - <previous context>
+	 * 11*4(%esp) - gap / flags
+	 * 10*4(%esp) - gap / cs
+	 *  9*4(%esp) - gap / ip
+	 *  8*4(%esp) - gap / orig_eax
+	 *  7*4(%esp) - gap / gs / function
+	 *  6*4(%esp) - ss
+	 *  5*4(%esp) - sp
+	 *  4*4(%esp) - flags
+	 *  3*4(%esp) - cs
+	 *  2*4(%esp) - ip
+	 *  1*4(%esp) - orig_eax
+	 *  0*4(%esp) - gs / function
+	 */
+
+	pushl	%ss		# ss
+	pushl	%esp		# sp (points at ss)
+	addl	$6*4, (%esp)	# point sp back at the previous context
+	pushl	6*4(%esp)	# flags
+	pushl	6*4(%esp)	# cs
+	pushl	6*4(%esp)	# ip
+	pushl	6*4(%esp)	# orig_eax
+	pushl	6*4(%esp)	# gs / function
+.Lfrom_usermode_no_fixup_\@:
+.endm
+
+.macro IRET_FRAME
+	testl $CS_FROM_KERNEL, 1*4(%esp)
+	jz .Lfinished_frame_\@
+
+	/*
+	 * Reconstruct the 3 entry IRET frame right after the (modified)
+	 * regs->sp without lowering %esp in between, such that an NMI in the
+	 * middle doesn't scribble our stack.
+	 */
+	pushl	%eax
+	pushl	%ecx
+	movl	5*4(%esp), %eax		# (modified) regs->sp
+
+	movl	4*4(%esp), %ecx		# flags
+	movl	%ecx, -4(%eax)
+
+	movl	3*4(%esp), %ecx		# cs
+	andl	$0x0000ffff, %ecx
+	movl	%ecx, -8(%eax)
+
+	movl	2*4(%esp), %ecx		# ip
+	movl	%ecx, -12(%eax)
+
+	movl	1*4(%esp), %ecx		# eax
+	movl	%ecx, -16(%eax)
+
+	popl	%ecx
+	lea	-16(%eax), %esp
+	popl	%eax
+.Lfinished_frame_\@:
+.endm
+
 .macro SAVE_ALL pt_regs_ax=%eax switch_stacks=0
 	cld
 	PUSH_GS
+	FIXUP_FRAME
 	pushl	%fs
 	pushl	%es
 	pushl	%ds
@@ -358,9 +451,6 @@
  * switch to it before we do any copying.
  */
 
-#define CS_FROM_ENTRY_STACK	(1 << 31)
-#define CS_FROM_USER_CR3	(1 << 30)
-
 .macro SWITCH_TO_KERNEL_STACK
 
 	ALTERNATIVE     "", "jmp .Lend_\@", X86_FEATURE_XENPV
@@ -374,13 +464,6 @@
 	 * that register for the time this macro runs
 	 */
 
-	/*
-	 * The high bits of the CS dword (__csh) are used for
-	 * CS_FROM_ENTRY_STACK and CS_FROM_USER_CR3. Clear them in case
-	 * hardware didn't do this for us.
-	 */
-	andl	$(0x0000ffff), PT_CS(%esp)
-
 	/* Are we on the entry stack? Bail out if not! */
 	movl	PER_CPU_VAR(cpu_entry_area), %ecx
 	addl	$CPU_ENTRY_AREA_entry_stack + SIZEOF_entry_stack, %ecx
@@ -990,6 +1073,7 @@ ENTRY(entry_INT80_32)
 	/* Restore user state */
 	RESTORE_REGS pop=4			# skip orig_eax/error_code
 .Lirq_return:
+	IRET_FRAME
 	/*
 	 * ARCH_HAS_MEMBARRIER_SYNC_CORE rely on IRET core serialization
 	 * when returning from IPI handler and when returning from
@@ -1340,6 +1424,7 @@ END(page_fault)
 
 common_exception:
 	/* the function address is in %gs's slot on the stack */
+	FIXUP_FRAME
 	pushl	%fs
 	pushl	%es
 	pushl	%ds
arch/x86/include/asm/kexec.h +0 -17
@@ -71,22 +71,6 @@ struct kimage;
 #define KEXEC_BACKUP_SRC_END	(640 * 1024UL - 1)	/* 640K */
 
 /*
- * CPU does not save ss and sp on stack if execution is already
- * running in kernel mode at the time of NMI occurrence. This code
- * fixes it.
- */
-static inline void crash_fixup_ss_esp(struct pt_regs *newregs,
-				      struct pt_regs *oldregs)
-{
-#ifdef CONFIG_X86_32
-	newregs->sp = (unsigned long)&(oldregs->sp);
-	asm volatile("xorl %%eax, %%eax\n\t"
-		     "movw %%ss, %%ax\n\t"
-		     :"=a"(newregs->ss));
-#endif
-}
-
-/*
  * This function is responsible for capturing register states if coming
  * via panic otherwise just fix up the ss and sp if coming via kernel
  * mode exception.
@@ -96,7 +80,6 @@ static inline void crash_setup_regs(stru
 {
 	if (oldregs) {
 		memcpy(newregs, oldregs, sizeof(*newregs));
-		crash_fixup_ss_esp(newregs, oldregs);
 	} else {
 #ifdef CONFIG_X86_32
 		asm volatile("movl %%ebx,%0" : "=m"(newregs->bx));
arch/x86/include/asm/ptrace.h +2 -15
@@ -166,14 +166,10 @@ static inline bool user_64bit_mode(struc
 #define compat_user_stack_pointer()	current_pt_regs()->sp
 #endif
 
-#ifdef CONFIG_X86_32
-extern unsigned long kernel_stack_pointer(struct pt_regs *regs);
-#else
 static inline unsigned long kernel_stack_pointer(struct pt_regs *regs)
 {
 	return regs->sp;
 }
-#endif
 
 #define GET_IP(regs) ((regs)->ip)
 #define GET_FP(regs) ((regs)->bp)
@@ -201,14 +197,6 @@ static inline unsigned long regs_get_reg
 	if (unlikely(offset > MAX_REG_OFFSET))
 		return 0;
 #ifdef CONFIG_X86_32
-	/*
-	 * Traps from the kernel do not save sp and ss.
-	 * Use the helper function to retrieve sp.
-	 */
-	if (offset == offsetof(struct pt_regs, sp) &&
-	    regs->cs == __KERNEL_CS)
-		return kernel_stack_pointer(regs);
-
 	/* The selector fields are 16-bit. */
 	if (offset == offsetof(struct pt_regs, cs) ||
 	    offset == offsetof(struct pt_regs, ss) ||
@@ -234,8 +222,7 @@ static inline unsigned long regs_get_reg
 static inline int regs_within_kernel_stack(struct pt_regs *regs,
 					   unsigned long addr)
 {
-	return ((addr & ~(THREAD_SIZE - 1))  ==
-		(kernel_stack_pointer(regs) & ~(THREAD_SIZE - 1)));
+	return ((addr & ~(THREAD_SIZE - 1)) == (regs->sp & ~(THREAD_SIZE - 1)));
 }
 
 /**
@@ -249,7 +236,7 @@ static inline int regs_within_kernel_sta
  */
 static inline unsigned long *regs_get_kernel_stack_nth_addr(struct pt_regs *regs, unsigned int n)
 {
-	unsigned long *addr = (unsigned long *)kernel_stack_pointer(regs);
+	unsigned long *addr = (unsigned long *)regs->sp;
 
 	addr += n;
 	if (regs_within_kernel_stack(regs, (unsigned long)addr))
arch/x86/include/asm/stacktrace.h +1 -1
@@ -78,7 +78,7 @@ static inline unsigned long *
 get_stack_pointer(struct task_struct *task, struct pt_regs *regs)
 {
 	if (regs)
-		return (unsigned long *)kernel_stack_pointer(regs);
+		return (unsigned long *)regs->sp;
 
 	if (task == current)
 		return __builtin_frame_address(0);
arch/x86/kernel/crash.c +0 -8
@@ -72,14 +72,6 @@ static inline void cpu_crash_vmclear_loa
 
 static void kdump_nmi_callback(int cpu, struct pt_regs *regs)
 {
-#ifdef CONFIG_X86_32
-	struct pt_regs fixed_regs;
-
-	if (!user_mode(regs)) {
-		crash_fixup_ss_esp(&fixed_regs, regs);
-		regs = &fixed_regs;
-	}
-#endif
 	crash_save_cpu(regs, cpu);
 
 	/*
arch/x86/kernel/ftrace_32.S +43 -34

113 lines not on this page · raw message

arch/x86/kernel/kgdb.c +0 -8
@@ -127,14 +127,6 @@ char *dbg_get_reg(int regno, void *mem,
 
 #ifdef CONFIG_X86_32
 	switch (regno) {
-	case GDB_SS:
-		if (!user_mode(regs))
-			*(unsigned long *)mem = __KERNEL_DS;
-		break;
-	case GDB_SP:
-		if (!user_mode(regs))
-			*(unsigned long *)mem = kernel_stack_pointer(regs);
-		break;
 	case GDB_GS:
 	case GDB_FS:
 		*(unsigned long *)mem = 0xFFFF;
arch/x86/kernel/kprobes/common.h +2 -2
@@ -72,8 +72,8 @@
 	"	popl %edi\n"			\
 	"	popl %ebp\n"			\
 	"	popl %eax\n"			\
-	/* Skip ds, es, fs, gs, orig_ax, and ip. Note: don't pop cs here*/\
-	"	addl $24, %esp\n"
+	/* Skip ds, es, fs, gs, orig_ax, ip, and cs. */\
+	"	addl $7*4, %esp\n"
 #endif
 
 /* Ensure if the instruction can be boostable */
arch/x86/kernel/kprobes/core.c +12 -17
@@ -69,7 +69,7 @@
 DEFINE_PER_CPU(struct kprobe *, current_kprobe) = NULL;
 DEFINE_PER_CPU(struct kprobe_ctlblk, kprobe_ctlblk);
 
-#define stack_addr(regs) ((unsigned long *)kernel_stack_pointer(regs))
+#define stack_addr(regs) ((unsigned long *)regs->sp)
 
 #define W(row, b0, b1, b2, b3, b4, b5, b6, b7, b8, b9, ba, bb, bc, bd, be, bf)\
 	(((b0##UL << 0x0)|(b1##UL << 0x1)|(b2##UL << 0x2)|(b3##UL << 0x3) |   \
@@ -731,29 +731,27 @@ asm(
 	".global kretprobe_trampoline\n"
 	".type kretprobe_trampoline, @function\n"
 	"kretprobe_trampoline:\n"
-#ifdef CONFIG_X86_64
 	/* We don't bother saving the ss register */
+#ifdef CONFIG_X86_64
 	"	pushq %rsp\n"
 	"	pushfq\n"
 	SAVE_REGS_STRING
 	"	movq %rsp, %rdi\n"
 	"	call trampoline_handler\n"
 	/* Replace saved sp with true return address. */
-	"	movq %rax, 152(%rsp)\n"
+	"	movq %rax, 19*8(%rsp)\n"
 	RESTORE_REGS_STRING
 	"	popfq\n"
 #else
-	"	pushf\n"
+	"	pushl %esp\n"
+	"	pushfl\n"
 	SAVE_REGS_STRING
 	"	movl %esp, %eax\n"
 	"	call trampoline_handler\n"
-	/* Move flags to cs */
-	"	movl 56(%esp), %edx\n"
-	"	movl %edx, 52(%esp)\n"
-	/* Replace saved flags with true return address. */
-	"	movl %eax, 56(%esp)\n"
+	/* Replace saved sp with true return address. */
+	"	movl %eax, 15*4(%esp)\n"
 	RESTORE_REGS_STRING
-	"	popf\n"
+	"	popfl\n"
 #endif
 	"	ret\n"
 	".size kretprobe_trampoline, .-kretprobe_trampoline\n"
@@ -794,16 +792,13 @@ __used __visible void *trampoline_handle
 	INIT_HLIST_HEAD(&empty_rp);
 	kretprobe_hash_lock(current, &head, &flags);
 	/* fixup registers */
-#ifdef CONFIG_X86_64
 	regs->cs = __KERNEL_CS;
-	/* On x86-64, we use pt_regs->sp for return address holder. */
-	frame_pointer = &regs->sp;
-#else
-	regs->cs = __KERNEL_CS | get_kernel_rpl();
+#ifdef CONFIG_X86_32
+	regs->cs |= get_kernel_rpl();
 	regs->gs = 0;
-	/* On x86-32, we use pt_regs->flags for return address holder. */
-	frame_pointer = &regs->flags;
 #endif
+	/* We use pt_regs->sp for return address holder. */
+	frame_pointer = &regs->sp;
 	regs->ip = trampoline_address;
 	regs->orig_ax = ~0UL;
 
arch/x86/kernel/kprobes/opt.c +12 -8
@@ -115,14 +115,15 @@ asm (
 			"optprobe_template_call:\n"
 			ASM_NOP5
 			/* Move flags to rsp */
-			"	movq 144(%rsp), %rdx\n"
-			"	movq %rdx, 152(%rsp)\n"
+			"	movq 18*8(%rsp), %rdx\n"
+			"	movq %rdx, 19*8(%rsp)\n"
 			RESTORE_REGS_STRING
 			/* Skip flags entry */
 			"	addq $8, %rsp\n"
 			"	popfq\n"
 #else /* CONFIG_X86_32 */
-			"	pushf\n"
+			"	pushl %esp\n"
+			"	pushfl\n"
 			SAVE_REGS_STRING
 			"	movl %esp, %edx\n"
 			".global optprobe_template_val\n"
@@ -131,9 +132,13 @@ asm (
 			".global optprobe_template_call\n"
 			"optprobe_template_call:\n"
 			ASM_NOP5
+			/* Move flags into esp */
+			"	movl 14*4(%esp), %edx\n"
+			"	movl %edx, 15*4(%esp)\n"
 			RESTORE_REGS_STRING
-			"	addl $4, %esp\n"	/* skip cs */
-			"	popf\n"
+			/* Skip flags entry */
+			"	addl $4, %esp\n"
+			"	popfl\n"
 #endif
 			".global optprobe_template_end\n"
 			"optprobe_template_end:\n"
@@ -165,10 +170,9 @@ optimized_callback(struct optimized_kpro
 	} else {
 		struct kprobe_ctlblk *kcb = get_kprobe_ctlblk();
 		/* Save skipped registers */
-#ifdef CONFIG_X86_64
 		regs->cs = __KERNEL_CS;
-#else
-		regs->cs = __KERNEL_CS | get_kernel_rpl();
+#ifdef CONFIG_X86_32
+		regs->cs |= get_kernel_rpl();
 		regs->gs = 0;
 #endif
 		regs->ip = (unsigned long)op->kp.addr + INT3_SIZE;
arch/x86/kernel/process_32.c +5 -11
@@ -62,27 +62,21 @@ void __show_regs(struct pt_regs *regs, e
 {
 	unsigned long cr0 = 0L, cr2 = 0L, cr3 = 0L, cr4 = 0L;
 	unsigned long d0, d1, d2, d3, d6, d7;
-	unsigned long sp;
-	unsigned short ss, gs;
+	unsigned short gs;
 
-	if (user_mode(regs)) {
-		sp = regs->sp;
-		ss = regs->ss;
+	if (user_mode(regs))
 		gs = get_user_gs(regs);
-	} else {
-		sp = kernel_stack_pointer(regs);
-		savesegment(ss, ss);
+	else
 		savesegment(gs, gs);
-	}
 
 	show_ip(regs, KERN_DEFAULT);
 
 	printk(KERN_DEFAULT "EAX: %08lx EBX: %08lx ECX: %08lx EDX: %08lx\n",
 		regs->ax, regs->bx, regs->cx, regs->dx);
 	printk(KERN_DEFAULT "ESI: %08lx EDI: %08lx EBP: %08lx ESP: %08lx\n",
-		regs->si, regs->di, regs->bp, sp);
+		regs->si, regs->di, regs->bp, regs->sp);
 	printk(KERN_DEFAULT "DS: %04x ES: %04x FS: %04x GS: %04x SS: %04x EFLAGS: %08lx\n",
-	       (u16)regs->ds, (u16)regs->es, (u16)regs->fs, gs, ss, regs->flags);
+	       (u16)regs->ds, (u16)regs->es, (u16)regs->fs, gs, regs->ss, regs->flags);
 
 	if (mode != SHOW_REGS_ALL)
 		return;
arch/x86/kernel/ptrace.c +0 -29
@@ -153,35 +153,6 @@ static inline bool invalid_selector(u16
 
 #define FLAG_MASK		FLAG_MASK_32
 
-/*
- * X86_32 CPUs don't save ss and esp if the CPU is already in kernel mode
- * when it traps.  The previous stack will be directly underneath the saved
- * registers, and 'sp/ss' won't even have been saved. Thus the '&regs->sp'.
- *
- * Now, if the stack is empty, '&regs->sp' is out of range. In this
- * case we try to take the previous stack. To always return a non-null
- * stack pointer we fall back to regs as stack if no previous stack
- * exists.
- *
- * This is valid only for kernel mode traps.
- */
-unsigned long kernel_stack_pointer(struct pt_regs *regs)
-{
-	unsigned long context = (unsigned long)regs & ~(THREAD_SIZE - 1);
-	unsigned long sp = (unsigned long)&regs->sp;
-	u32 *prev_esp;
-
-	if (context == (sp & ~(THREAD_SIZE - 1)))
-		return sp;
-
-	prev_esp = (u32 *)(context);
-	if (*prev_esp)
-		return (unsigned long)*prev_esp;
-
-	return (unsigned long)regs;
-}
-EXPORT_SYMBOL_GPL(kernel_stack_pointer);
-
 static unsigned long *pt_regs_access(struct pt_regs *regs, unsigned long regno)
 {
 	BUILD_BUG_ON(offsetof(struct pt_regs, bx) != 0);
arch/x86/kernel/time.c +1 -2
@@ -37,8 +37,7 @@ unsigned long profile_pc(struct pt_regs
 #ifdef CONFIG_FRAME_POINTER
 		return *(unsigned long *)(regs->bp + sizeof(long));
 #else
-		unsigned long *sp =
-			(unsigned long *)kernel_stack_pointer(regs);
+		unsigned long *sp = (unsigned long *)regs->sp;
 		/*
 		 * Return address is either directly at stack pointer
 		 * or above a saved flags. Eflags has bits 22-31 zero,
arch/x86/kernel/unwind_frame.c +4 -28
@@ -69,15 +69,6 @@ static void unwind_dump(struct unwind_st
 	}
 }
 
-static size_t regs_size(struct pt_regs *regs)
-{
-	/* x86_32 regs from kernel mode are two words shorter: */
-	if (IS_ENABLED(CONFIG_X86_32) && !user_mode(regs))
-		return sizeof(*regs) - 2*sizeof(long);
-
-	return sizeof(*regs);
-}
-
 static bool in_entry_code(unsigned long ip)
 {
 	char *addr = (char *)ip;
@@ -197,12 +188,6 @@ static struct pt_regs *decode_frame_poin
 }
 #endif
 
-#ifdef CONFIG_X86_32
-#define KERNEL_REGS_SIZE (sizeof(struct pt_regs) - 2*sizeof(long))
-#else
-#define KERNEL_REGS_SIZE (sizeof(struct pt_regs))
-#endif
-
 static bool update_stack_state(struct unwind_state *state,
 			       unsigned long *next_bp)
 {
@@ -213,7 +198,7 @@ static bool update_stack_state(struct un
 	size_t len;
 
 	if (state->regs)
-		prev_frame_end = (void *)state->regs + regs_size(state->regs);
+		prev_frame_end = (void *)state->regs + sizeof(*state->regs);
 	else
 		prev_frame_end = (void *)state->bp + FRAME_HEADER_SIZE;
 
@@ -221,7 +206,7 @@ static bool update_stack_state(struct un
 	regs = decode_frame_pointer(next_bp);
 	if (regs) {
 		frame = (unsigned long *)regs;
-		len = KERNEL_REGS_SIZE;
+		len = sizeof(*regs);
 		state->got_irq = true;
 	} else {
 		frame = next_bp;
@@ -245,14 +230,6 @@ static bool update_stack_state(struct un
 	    frame < prev_frame_end)
 		return false;
 
-	/*
-	 * On 32-bit with user mode regs, make sure the last two regs are safe
-	 * to access:
-	 */
-	if (IS_ENABLED(CONFIG_X86_32) && regs && user_mode(regs) &&
-	    !on_stack(info, frame, len + 2*sizeof(long)))
-		return false;
-
 	/* Move state to the next frame: */
 	if (regs) {
 		state->regs = regs;
@@ -411,10 +388,9 @@ void __unwind_start(struct unwind_state
 	 * Pretend that the frame is complete and that BP points to it, but save
 	 * the real BP so that we can use it when looking for the next frame.
 	 */
-	if (regs && regs->ip == 0 &&
-	    (unsigned long *)kernel_stack_pointer(regs) >= first_frame) {
+	if (regs && regs->ip == 0 && (unsigned long *)regs->sp >= first_frame) {
 		state->next_bp = bp;
-		bp = ((unsigned long *)kernel_stack_pointer(regs)) - 1;
+		bp = ((unsigned long *)regs->sp) - 1;
 	}
 
 	/* Initialize stack info and make sure the frame data is accessible: */
arch/x86/kernel/unwind_orc.c +1 -1
@@ -579,7 +579,7 @@ void __unwind_start(struct unwind_state
 			goto done;
 
 		state->ip = regs->ip;
-		state->sp = kernel_stack_pointer(regs);
+		state->sp = regs->sp;
 		state->bp = regs->bp;
 		state->regs = regs;
 		state->full_regs = true;
maintainer Acked-by on v1 5/15
#13 lore raw

Re: [PATCH 05/15] x86_32: Provide consistent pt_regs

Quoted 9 lines · On Wed, 05 Jun 2019 15:07:58 +0200 Peter Zijlstra <[email protected]> wrote:

Currently pt_regs on x86_32 has an oddity in that kernel regs (!user_mode(regs)) are short two entries (esp/ss). This means that any code trying to use them (typically: regs->sp) needs to jump through some unfortunate hoops.

Change the entry code to fix this up and create a full pt_regs frame.

This then simplifies various trampolines in ftrace and kprobes, the stack unwinder, ptrace, kdump and kgdb.

The kprobes parts are looks good to me.

Thank you!

Quoted 720 lines
---
 arch/x86/entry/entry_32.S         |  105 ++++++++++++++++++++++++++++++++++----
 arch/x86/include/asm/kexec.h      |   17 ------
 arch/x86/include/asm/ptrace.h     |   17 ------
 arch/x86/include/asm/stacktrace.h |    2 
 arch/x86/kernel/crash.c           |    8 --
 arch/x86/kernel/ftrace_32.S       |   77 +++++++++++++++------------
 arch/x86/kernel/kgdb.c            |    8 --
 arch/x86/kernel/kprobes/common.h  |    4 -
 arch/x86/kernel/kprobes/core.c    |   29 ++++------
 arch/x86/kernel/kprobes/opt.c     |   20 ++++---
 arch/x86/kernel/process_32.c      |   16 +----
 arch/x86/kernel/ptrace.c          |   29 ----------
 arch/x86/kernel/time.c            |    3 -
 arch/x86/kernel/unwind_frame.c    |   32 +----------
 arch/x86/kernel/unwind_orc.c      |    2 
 15 files changed, 178 insertions(+), 191 deletions(-)
arch/x86/entry/entry_32.S +95 -10
@@ -202,9 +202,102 @@
 .Lend_\@:
 .endm
 
+#define CS_FROM_ENTRY_STACK	(1 << 31)
+#define CS_FROM_USER_CR3	(1 << 30)
+#define CS_FROM_KERNEL		(1 << 29)
+
+.macro FIXUP_FRAME
+	/*
+	 * The high bits of the CS dword (__csh) are used for CS_FROM_*.
+	 * Clear them in case hardware didn't do this for us.
+	 */
+	andl	$0x0000ffff, 3*4(%esp)
+
+#ifdef CONFIG_VM86
+	testl	$X86_EFLAGS_VM, 4*4(%esp)
+	jnz	.Lfrom_usermode_no_fixup_\@
+#endif
+	testl	$SEGMENT_RPL_MASK, 3*4(%esp)
+	jnz	.Lfrom_usermode_no_fixup_\@
+
+	orl	$CS_FROM_KERNEL, 3*4(%esp)
+
+	/*
+	 * When we're here from kernel mode; the (exception) stack looks like:
+	 *
+	 *  5*4(%esp) - <previous context>
+	 *  4*4(%esp) - flags
+	 *  3*4(%esp) - cs
+	 *  2*4(%esp) - ip
+	 *  1*4(%esp) - orig_eax
+	 *  0*4(%esp) - gs / function
+	 *
+	 * Lets build a 5 entry IRET frame after that, such that struct pt_regs
+	 * is complete and in particular regs->sp is correct. This gives us
+	 * the original 5 enties as gap:
+	 *
+	 * 12*4(%esp) - <previous context>
+	 * 11*4(%esp) - gap / flags
+	 * 10*4(%esp) - gap / cs
+	 *  9*4(%esp) - gap / ip
+	 *  8*4(%esp) - gap / orig_eax
+	 *  7*4(%esp) - gap / gs / function
+	 *  6*4(%esp) - ss
+	 *  5*4(%esp) - sp
+	 *  4*4(%esp) - flags
+	 *  3*4(%esp) - cs
+	 *  2*4(%esp) - ip
+	 *  1*4(%esp) - orig_eax
+	 *  0*4(%esp) - gs / function
+	 */
+
+	pushl	%ss		# ss
+	pushl	%esp		# sp (points at ss)
+	addl	$6*4, (%esp)	# point sp back at the previous context
+	pushl	6*4(%esp)	# flags
+	pushl	6*4(%esp)	# cs
+	pushl	6*4(%esp)	# ip
+	pushl	6*4(%esp)	# orig_eax
+	pushl	6*4(%esp)	# gs / function
+.Lfrom_usermode_no_fixup_\@:
+.endm
+
+.macro IRET_FRAME
+	testl $CS_FROM_KERNEL, 1*4(%esp)
+	jz .Lfinished_frame_\@
+
+	/*
+	 * Reconstruct the 3 entry IRET frame right after the (modified)
+	 * regs->sp without lowering %esp in between, such that an NMI in the
+	 * middle doesn't scribble our stack.
+	 */
+	pushl	%eax
+	pushl	%ecx
+	movl	5*4(%esp), %eax		# (modified) regs->sp
+
+	movl	4*4(%esp), %ecx		# flags
+	movl	%ecx, -4(%eax)
+
+	movl	3*4(%esp), %ecx		# cs
+	andl	$0x0000ffff, %ecx
+	movl	%ecx, -8(%eax)
+
+	movl	2*4(%esp), %ecx		# ip
+	movl	%ecx, -12(%eax)
+
+	movl	1*4(%esp), %ecx		# eax
+	movl	%ecx, -16(%eax)
+
+	popl	%ecx
+	lea	-16(%eax), %esp
+	popl	%eax
+.Lfinished_frame_\@:
+.endm
+
 .macro SAVE_ALL pt_regs_ax=%eax switch_stacks=0
 	cld
 	PUSH_GS
+	FIXUP_FRAME
 	pushl	%fs
 	pushl	%es
 	pushl	%ds
@@ -358,9 +451,6 @@
  * switch to it before we do any copying.
  */
 
-#define CS_FROM_ENTRY_STACK	(1 << 31)
-#define CS_FROM_USER_CR3	(1 << 30)
-
 .macro SWITCH_TO_KERNEL_STACK
 
 	ALTERNATIVE     "", "jmp .Lend_\@", X86_FEATURE_XENPV
@@ -374,13 +464,6 @@
 	 * that register for the time this macro runs
 	 */
 
-	/*
-	 * The high bits of the CS dword (__csh) are used for
-	 * CS_FROM_ENTRY_STACK and CS_FROM_USER_CR3. Clear them in case
-	 * hardware didn't do this for us.
-	 */
-	andl	$(0x0000ffff), PT_CS(%esp)
-
 	/* Are we on the entry stack? Bail out if not! */
 	movl	PER_CPU_VAR(cpu_entry_area), %ecx
 	addl	$CPU_ENTRY_AREA_entry_stack + SIZEOF_entry_stack, %ecx
@@ -990,6 +1073,7 @@ ENTRY(entry_INT80_32)
 	/* Restore user state */
 	RESTORE_REGS pop=4			# skip orig_eax/error_code
 .Lirq_return:
+	IRET_FRAME
 	/*
 	 * ARCH_HAS_MEMBARRIER_SYNC_CORE rely on IRET core serialization
 	 * when returning from IPI handler and when returning from
@@ -1340,6 +1424,7 @@ END(page_fault)
 
 common_exception:
 	/* the function address is in %gs's slot on the stack */
+	FIXUP_FRAME
 	pushl	%fs
 	pushl	%es
 	pushl	%ds
arch/x86/include/asm/kexec.h +0 -17
@@ -71,22 +71,6 @@ struct kimage;
 #define KEXEC_BACKUP_SRC_END	(640 * 1024UL - 1)	/* 640K */
 
 /*
- * CPU does not save ss and sp on stack if execution is already
- * running in kernel mode at the time of NMI occurrence. This code
- * fixes it.
- */
-static inline void crash_fixup_ss_esp(struct pt_regs *newregs,
-				      struct pt_regs *oldregs)
-{
-#ifdef CONFIG_X86_32
-	newregs->sp = (unsigned long)&(oldregs->sp);
-	asm volatile("xorl %%eax, %%eax\n\t"
-		     "movw %%ss, %%ax\n\t"
-		     :"=a"(newregs->ss));
-#endif
-}
-
-/*
  * This function is responsible for capturing register states if coming
  * via panic otherwise just fix up the ss and sp if coming via kernel
  * mode exception.
@@ -96,7 +80,6 @@ static inline void crash_setup_regs(stru
 {
 	if (oldregs) {
 		memcpy(newregs, oldregs, sizeof(*newregs));
-		crash_fixup_ss_esp(newregs, oldregs);
 	} else {
 #ifdef CONFIG_X86_32
 		asm volatile("movl %%ebx,%0" : "=m"(newregs->bx));
arch/x86/include/asm/ptrace.h +2 -15
@@ -166,14 +166,10 @@ static inline bool user_64bit_mode(struc
 #define compat_user_stack_pointer()	current_pt_regs()->sp
 #endif
 
-#ifdef CONFIG_X86_32
-extern unsigned long kernel_stack_pointer(struct pt_regs *regs);
-#else
 static inline unsigned long kernel_stack_pointer(struct pt_regs *regs)
 {
 	return regs->sp;
 }
-#endif
 
 #define GET_IP(regs) ((regs)->ip)
 #define GET_FP(regs) ((regs)->bp)
@@ -201,14 +197,6 @@ static inline unsigned long regs_get_reg
 	if (unlikely(offset > MAX_REG_OFFSET))
 		return 0;
 #ifdef CONFIG_X86_32
-	/*
-	 * Traps from the kernel do not save sp and ss.
-	 * Use the helper function to retrieve sp.
-	 */
-	if (offset == offsetof(struct pt_regs, sp) &&
-	    regs->cs == __KERNEL_CS)
-		return kernel_stack_pointer(regs);
-
 	/* The selector fields are 16-bit. */
 	if (offset == offsetof(struct pt_regs, cs) ||
 	    offset == offsetof(struct pt_regs, ss) ||
@@ -234,8 +222,7 @@ static inline unsigned long regs_get_reg
 static inline int regs_within_kernel_stack(struct pt_regs *regs,
 					   unsigned long addr)
 {
-	return ((addr & ~(THREAD_SIZE - 1))  ==
-		(kernel_stack_pointer(regs) & ~(THREAD_SIZE - 1)));
+	return ((addr & ~(THREAD_SIZE - 1)) == (regs->sp & ~(THREAD_SIZE - 1)));
 }
 
 /**
@@ -249,7 +236,7 @@ static inline int regs_within_kernel_sta
  */
 static inline unsigned long *regs_get_kernel_stack_nth_addr(struct pt_regs *regs, unsigned int n)
 {
-	unsigned long *addr = (unsigned long *)kernel_stack_pointer(regs);
+	unsigned long *addr = (unsigned long *)regs->sp;
 
 	addr += n;
 	if (regs_within_kernel_stack(regs, (unsigned long)addr))
arch/x86/include/asm/stacktrace.h +1 -1
@@ -78,7 +78,7 @@ static inline unsigned long *
 get_stack_pointer(struct task_struct *task, struct pt_regs *regs)
 {
 	if (regs)
-		return (unsigned long *)kernel_stack_pointer(regs);
+		return (unsigned long *)regs->sp;
 
 	if (task == current)
 		return __builtin_frame_address(0);
arch/x86/kernel/crash.c +0 -8
@@ -72,14 +72,6 @@ static inline void cpu_crash_vmclear_loa
 
 static void kdump_nmi_callback(int cpu, struct pt_regs *regs)
 {
-#ifdef CONFIG_X86_32
-	struct pt_regs fixed_regs;
-
-	if (!user_mode(regs)) {
-		crash_fixup_ss_esp(&fixed_regs, regs);
-		regs = &fixed_regs;
-	}
-#endif
 	crash_save_cpu(regs, cpu);
 
 	/*
arch/x86/kernel/ftrace_32.S +43 -34
@@ -10,6 +10,7 @@
 #include <asm/ftrace.h>
 #include <asm/nospec-branch.h>
 #include <asm/frame.h>
+#include <asm/asm-offsets.h>
 
 # define function_hook	__fentry__
 EXPORT_SYMBOL(__fentry__)
@@ -90,26 +91,38 @@ END(ftrace_caller)
 
 ENTRY(ftrace_regs_caller)
 	/*
-	 * i386 does not save SS and ESP when coming from kernel.
-	 * Instead, to get sp, &regs->sp is used (see ptrace.h).
-	 * Unfortunately, that means eflags must be at the same location
-	 * as the current return ip is. We move the return ip into the
-	 * regs->ip location, and move flags into the return ip location.
+	 * We're here from an mcount/fentry CALL, and the stack frame looks like:
+	 *
+	 *  <previous context>
+	 *  RET-IP
+	 *
+	 * The purpose of this function is to call out in an emulated INT3
+	 * environment with a stack frame like:
+	 *
+	 *  <previous context>
+	 *  gap / RET-IP
+	 *  gap
+	 *  gap
+	 *  gap
+	 *  pt_regs
+	 *
+	 * We do _NOT_ restore: ss, flags, cs, gs, fs, es, ds
 	 */
-	pushl	$__KERNEL_CS
-	pushl	4(%esp)				/* Save the return ip */
-	pushl	$0				/* Load 0 into orig_ax */
+	subl	$3*4, %esp	# RET-IP + 3 gaps
+	pushl	%ss		# ss
+	pushl	%esp		# points at ss
+	addl	$5*4, (%esp)	#   make it point at <previous context>
+	pushfl			# flags
+	pushl	$__KERNEL_CS	# cs
+	pushl	7*4(%esp)	# ip <- RET-IP
+	pushl	$0		# orig_eax
+
 	pushl	%gs
 	pushl	%fs
 	pushl	%es
 	pushl	%ds
-	pushl	%eax
-
-	/* Get flags and place them into the return ip slot */
-	pushf
-	popl	%eax
-	movl	%eax, 8*4(%esp)
 
+	pushl	%eax
 	pushl	%ebp
 	pushl	%edi
 	pushl	%esi
@@ -119,24 +132,25 @@ ENTRY(ftrace_regs_caller)
 
 	ENCODE_FRAME_POINTER
 
-	movl	12*4(%esp), %eax		/* Load ip (1st parameter) */
-	subl	$MCOUNT_INSN_SIZE, %eax		/* Adjust ip */
-	movl	15*4(%esp), %edx		/* Load parent ip (2nd parameter) */
-	movl	function_trace_op, %ecx		/* Save ftrace_pos in 3rd parameter */
-	pushl	%esp				/* Save pt_regs as 4th parameter */
+	movl	PT_EIP(%esp), %eax	# 1st argument: IP
+	subl	$MCOUNT_INSN_SIZE, %eax
+	movl	21*4(%esp), %edx	# 2nd argument: parent ip
+	movl	function_trace_op, %ecx	# 3rd argument: ftrace_pos
+	pushl	%esp			# 4th argument: pt_regs
 
 GLOBAL(ftrace_regs_call)
 	call	ftrace_stub
 
-	addl	$4, %esp			/* Skip pt_regs */
+	addl	$4, %esp		# skip 4th argument
 
-	/* restore flags */
-	push	14*4(%esp)
-	popf
-
-	/* Move return ip back to its original location */
-	movl	12*4(%esp), %eax
-	movl	%eax, 14*4(%esp)
+	/* place IP below the new SP */
+	movl	PT_OLDESP(%esp), %eax
+	movl	PT_EIP(%esp), %ecx
+	movl	%ecx, -4(%eax)
+
+	/* place EAX below that */
+	movl	PT_EAX(%esp), %ecx
+	movl	%ecx, -8(%eax)
 
 	popl	%ebx
 	popl	%ecx
@@ -144,14 +158,9 @@ GLOBAL(ftrace_regs_call)
 	popl	%esi
 	popl	%edi
 	popl	%ebp
-	popl	%eax
-	popl	%ds
-	popl	%es
-	popl	%fs
-	popl	%gs
 
-	/* use lea to not affect flags */
-	lea	3*4(%esp), %esp			/* Skip orig_ax, ip and cs */
+	lea	-8(%eax), %esp
+	popl	%eax
 
 	jmp	.Lftrace_ret
 
arch/x86/kernel/kgdb.c +0 -8
@@ -127,14 +127,6 @@ char *dbg_get_reg(int regno, void *mem,
 
 #ifdef CONFIG_X86_32
 	switch (regno) {
-	case GDB_SS:
-		if (!user_mode(regs))
-			*(unsigned long *)mem = __KERNEL_DS;
-		break;
-	case GDB_SP:
-		if (!user_mode(regs))
-			*(unsigned long *)mem = kernel_stack_pointer(regs);
-		break;
 	case GDB_GS:
 	case GDB_FS:
 		*(unsigned long *)mem = 0xFFFF;
arch/x86/kernel/kprobes/common.h +2 -2
@@ -72,8 +72,8 @@
 	"	popl %edi\n"			\
 	"	popl %ebp\n"			\
 	"	popl %eax\n"			\
-	/* Skip ds, es, fs, gs, orig_ax, and ip. Note: don't pop cs here*/\
-	"	addl $24, %esp\n"
+	/* Skip ds, es, fs, gs, orig_ax, ip, and cs. */\
+	"	addl $7*4, %esp\n"
 #endif
 
 /* Ensure if the instruction can be boostable */
arch/x86/kernel/kprobes/core.c +12 -17
@@ -69,7 +69,7 @@
 DEFINE_PER_CPU(struct kprobe *, current_kprobe) = NULL;
 DEFINE_PER_CPU(struct kprobe_ctlblk, kprobe_ctlblk);
 
-#define stack_addr(regs) ((unsigned long *)kernel_stack_pointer(regs))
+#define stack_addr(regs) ((unsigned long *)regs->sp)
 
 #define W(row, b0, b1, b2, b3, b4, b5, b6, b7, b8, b9, ba, bb, bc, bd, be, bf)\
 	(((b0##UL << 0x0)|(b1##UL << 0x1)|(b2##UL << 0x2)|(b3##UL << 0x3) |   \
@@ -731,29 +731,27 @@ asm(
 	".global kretprobe_trampoline\n"
 	".type kretprobe_trampoline, @function\n"
 	"kretprobe_trampoline:\n"
-#ifdef CONFIG_X86_64
 	/* We don't bother saving the ss register */
+#ifdef CONFIG_X86_64
 	"	pushq %rsp\n"
 	"	pushfq\n"
 	SAVE_REGS_STRING
 	"	movq %rsp, %rdi\n"
 	"	call trampoline_handler\n"
 	/* Replace saved sp with true return address. */
-	"	movq %rax, 152(%rsp)\n"
+	"	movq %rax, 19*8(%rsp)\n"
 	RESTORE_REGS_STRING
 	"	popfq\n"
 #else
-	"	pushf\n"
+	"	pushl %esp\n"
+	"	pushfl\n"
 	SAVE_REGS_STRING
 	"	movl %esp, %eax\n"
 	"	call trampoline_handler\n"
-	/* Move flags to cs */
-	"	movl 56(%esp), %edx\n"
-	"	movl %edx, 52(%esp)\n"
-	/* Replace saved flags with true return address. */
-	"	movl %eax, 56(%esp)\n"
+	/* Replace saved sp with true return address. */
+	"	movl %eax, 15*4(%esp)\n"
 	RESTORE_REGS_STRING
-	"	popf\n"
+	"	popfl\n"
 #endif
 	"	ret\n"
 	".size kretprobe_trampoline, .-kretprobe_trampoline\n"
@@ -794,16 +792,13 @@ __used __visible void *trampoline_handle
 	INIT_HLIST_HEAD(&empty_rp);
 	kretprobe_hash_lock(current, &head, &flags);
 	/* fixup registers */
-#ifdef CONFIG_X86_64
 	regs->cs = __KERNEL_CS;
-	/* On x86-64, we use pt_regs->sp for return address holder. */
-	frame_pointer = &regs->sp;
-#else
-	regs->cs = __KERNEL_CS | get_kernel_rpl();
+#ifdef CONFIG_X86_32
+	regs->cs |= get_kernel_rpl();
 	regs->gs = 0;
-	/* On x86-32, we use pt_regs->flags for return address holder. */
-	frame_pointer = &regs->flags;
 #endif
+	/* We use pt_regs->sp for return address holder. */
+	frame_pointer = &regs->sp;
 	regs->ip = trampoline_address;
 	regs->orig_ax = ~0UL;
 
arch/x86/kernel/kprobes/opt.c +12 -8
@@ -115,14 +115,15 @@ asm (
 			"optprobe_template_call:\n"
 			ASM_NOP5
 			/* Move flags to rsp */
-			"	movq 144(%rsp), %rdx\n"
-			"	movq %rdx, 152(%rsp)\n"
+			"	movq 18*8(%rsp), %rdx\n"
+			"	movq %rdx, 19*8(%rsp)\n"
 			RESTORE_REGS_STRING
 			/* Skip flags entry */
 			"	addq $8, %rsp\n"
 			"	popfq\n"
 #else /* CONFIG_X86_32 */
-			"	pushf\n"
+			"	pushl %esp\n"
+			"	pushfl\n"
 			SAVE_REGS_STRING
 			"	movl %esp, %edx\n"
 			".global optprobe_template_val\n"
@@ -131,9 +132,13 @@ asm (
 			".global optprobe_template_call\n"
 			"optprobe_template_call:\n"
 			ASM_NOP5
+			/* Move flags into esp */
+			"	movl 14*4(%esp), %edx\n"
+			"	movl %edx, 15*4(%esp)\n"
 			RESTORE_REGS_STRING
-			"	addl $4, %esp\n"	/* skip cs */
-			"	popf\n"
+			/* Skip flags entry */
+			"	addl $4, %esp\n"
+			"	popfl\n"
 #endif
 			".global optprobe_template_end\n"
 			"optprobe_template_end:\n"
@@ -165,10 +170,9 @@ optimized_callback(struct optimized_kpro
 	} else {
 		struct kprobe_ctlblk *kcb = get_kprobe_ctlblk();
 		/* Save skipped registers */
-#ifdef CONFIG_X86_64
 		regs->cs = __KERNEL_CS;
-#else
-		regs->cs = __KERNEL_CS | get_kernel_rpl();
+#ifdef CONFIG_X86_32
+		regs->cs |= get_kernel_rpl();
 		regs->gs = 0;
 #endif
 		regs->ip = (unsigned long)op->kp.addr + INT3_SIZE;
arch/x86/kernel/process_32.c +5 -11
@@ -62,27 +62,21 @@ void __show_regs(struct pt_regs *regs, e
 {
 	unsigned long cr0 = 0L, cr2 = 0L, cr3 = 0L, cr4 = 0L;
 	unsigned long d0, d1, d2, d3, d6, d7;
-	unsigned long sp;
-	unsigned short ss, gs;
+	unsigned short gs;
 
-	if (user_mode(regs)) {
-		sp = regs->sp;
-		ss = regs->ss;
+	if (user_mode(regs))
 		gs = get_user_gs(regs);
-	} else {
-		sp = kernel_stack_pointer(regs);
-		savesegment(ss, ss);
+	else
 		savesegment(gs, gs);
-	}
 
 	show_ip(regs, KERN_DEFAULT);
 
 	printk(KERN_DEFAULT "EAX: %08lx EBX: %08lx ECX: %08lx EDX: %08lx\n",
 		regs->ax, regs->bx, regs->cx, regs->dx);
 	printk(KERN_DEFAULT "ESI: %08lx EDI: %08lx EBP: %08lx ESP: %08lx\n",
-		regs->si, regs->di, regs->bp, sp);
+		regs->si, regs->di, regs->bp, regs->sp);
 	printk(KERN_DEFAULT "DS: %04x ES: %04x FS: %04x GS: %04x SS: %04x EFLAGS: %08lx\n",
-	       (u16)regs->ds, (u16)regs->es, (u16)regs->fs, gs, ss, regs->flags);
+	       (u16)regs->ds, (u16)regs->es, (u16)regs->fs, gs, regs->ss, regs->flags);
 
 	if (mode != SHOW_REGS_ALL)
 		return;
arch/x86/kernel/ptrace.c +0 -29
@@ -153,35 +153,6 @@ static inline bool invalid_selector(u16
 
 #define FLAG_MASK		FLAG_MASK_32
 
-/*
- * X86_32 CPUs don't save ss and esp if the CPU is already in kernel mode
- * when it traps.  The previous stack will be directly underneath the saved
- * registers, and 'sp/ss' won't even have been saved. Thus the '&regs->sp'.
- *
- * Now, if the stack is empty, '&regs->sp' is out of range. In this
- * case we try to take the previous stack. To always return a non-null
- * stack pointer we fall back to regs as stack if no previous stack
- * exists.
- *
- * This is valid only for kernel mode traps.
- */
-unsigned long kernel_stack_pointer(struct pt_regs *regs)
-{
-	unsigned long context = (unsigned long)regs & ~(THREAD_SIZE - 1);
-	unsigned long sp = (unsigned long)&regs->sp;
-	u32 *prev_esp;
-
-	if (context == (sp & ~(THREAD_SIZE - 1)))
-		return sp;
-
-	prev_esp = (u32 *)(context);
-	if (*prev_esp)
-		return (unsigned long)*prev_esp;
-
-	return (unsigned long)regs;
-}
-EXPORT_SYMBOL_GPL(kernel_stack_pointer);
-
 static unsigned long *pt_regs_access(struct pt_regs *regs, unsigned long regno)
 {
 	BUILD_BUG_ON(offsetof(struct pt_regs, bx) != 0);
arch/x86/kernel/time.c +1 -2
@@ -37,8 +37,7 @@ unsigned long profile_pc(struct pt_regs
 #ifdef CONFIG_FRAME_POINTER
 		return *(unsigned long *)(regs->bp + sizeof(long));
 #else
-		unsigned long *sp =
-			(unsigned long *)kernel_stack_pointer(regs);
+		unsigned long *sp = (unsigned long *)regs->sp;
 		/*
 		 * Return address is either directly at stack pointer
 		 * or above a saved flags. Eflags has bits 22-31 zero,
arch/x86/kernel/unwind_frame.c +4 -28
@@ -69,15 +69,6 @@ static void unwind_dump(struct unwind_st
 	}
 }
 
-static size_t regs_size(struct pt_regs *regs)
-{
-	/* x86_32 regs from kernel mode are two words shorter: */
-	if (IS_ENABLED(CONFIG_X86_32) && !user_mode(regs))
-		return sizeof(*regs) - 2*sizeof(long);
-
-	return sizeof(*regs);
-}
-
 static bool in_entry_code(unsigned long ip)
 {
 	char *addr = (char *)ip;
@@ -197,12 +188,6 @@ static struct pt_regs *decode_frame_poin
 }
 #endif
 
-#ifdef CONFIG_X86_32
-#define KERNEL_REGS_SIZE (sizeof(struct pt_regs) - 2*sizeof(long))
-#else
-#define KERNEL_REGS_SIZE (sizeof(struct pt_regs))
-#endif
-
 static bool update_stack_state(struct unwind_state *state,
 			       unsigned long *next_bp)
 {
@@ -213,7 +198,7 @@ static bool update_stack_state(struct un
 	size_t len;
 
 	if (state->regs)
-		prev_frame_end = (void *)state->regs + regs_size(state->regs);
+		prev_frame_end = (void *)state->regs + sizeof(*state->regs);
 	else
 		prev_frame_end = (void *)state->bp + FRAME_HEADER_SIZE;
 
@@ -221,7 +206,7 @@ static bool update_stack_state(struct un
 	regs = decode_frame_pointer(next_bp);
 	if (regs) {
 		frame = (unsigned long *)regs;
-		len = KERNEL_REGS_SIZE;
+		len = sizeof(*regs);
 		state->got_irq = true;
 	} else {
 		frame = next_bp;
@@ -245,14 +230,6 @@ static bool update_stack_state(struct un
 	    frame < prev_frame_end)
 		return false;
 
-	/*
-	 * On 32-bit with user mode regs, make sure the last two regs are safe
-	 * to access:
-	 */
-	if (IS_ENABLED(CONFIG_X86_32) && regs && user_mode(regs) &&
-	    !on_stack(info, frame, len + 2*sizeof(long)))
-		return false;
-
 	/* Move state to the next frame: */
 	if (regs) {
 		state->regs = regs;
@@ -411,10 +388,9 @@ void __unwind_start(struct unwind_state
 	 * Pretend that the frame is complete and that BP points to it, but save
 	 * the real BP so that we can use it when looking for the next frame.
 	 */
-	if (regs && regs->ip == 0 &&
-	    (unsigned long *)kernel_stack_pointer(regs) >= first_frame) {
+	if (regs && regs->ip == 0 && (unsigned long *)regs->sp >= first_frame) {
 		state->next_bp = bp;
-		bp = ((unsigned long *)kernel_stack_pointer(regs)) - 1;
+		bp = ((unsigned long *)regs->sp) - 1;
 	}
 
 	/* Initialize stack info and make sure the frame data is accessible: */
arch/x86/kernel/unwind_orc.c +1 -1
@@ -579,7 +579,7 @@ void __unwind_start(struct unwind_state
 			goto done;
 
 		state->ip = regs->ip;
-		state->sp = kernel_stack_pointer(regs);
+		state->sp = regs->sp;
 		state->bp = regs->bp;
 		state->regs = regs;
 		state->full_regs = true;
Signature · 1 line
Masami Hiramatsu <[email protected]>
Reviewed-by on v1 5/15
#14 lore raw

Re: [PATCH 05/15] x86_32: Provide consistent pt_regs

In reply to Peter Zijlstra
Quoted 21 lines · Peter Zijlstra

Currently pt_regs on x86_32 has an oddity in that kernel regs (!user_mode(regs)) are short two entries (esp/ss). This means that any code trying to use them (typically: regs->sp) needs to jump through some unfortunate hoops.

Change the entry code to fix this up and create a full pt_regs frame.

This then simplifies various trampolines in ftrace and kprobes, the stack unwinder, ptrace, kdump and kgdb.

---
 arch/x86/entry/entry_32.S         |  105 ++++++++++++++++++++++++++++++++++----
 arch/x86/include/asm/kexec.h      |   17 ------
 arch/x86/include/asm/ptrace.h     |   17 ------
 arch/x86/include/asm/stacktrace.h |    2 
 arch/x86/kernel/crash.c           |    8 --
 arch/x86/kernel/ftrace_32.S       |   77 +++++++++++++++------------
 arch/x86/kernel/kgdb.c            |    8 --
 arch/x86/kernel/kprobes/common.h  |    4 -
Quoted 8 lines
 arch/x86/kernel/kprobes/core.c    |   29 ++++------
 arch/x86/kernel/kprobes/opt.c     |   20 ++++---
 arch/x86/kernel/process_32.c      |   16 +----
 arch/x86/kernel/ptrace.c          |   29 ----------
 arch/x86/kernel/time.c            |    3 -
 arch/x86/kernel/unwind_frame.c    |   32 +----------
 arch/x86/kernel/unwind_orc.c      |    2 
 15 files changed, 178 insertions(+), 191 deletions(-)

I recall writing some of this code (some of the kernel_stack_pointer removal stuff) so please give me a shout-out ;-)

Otherwise:

Signature · 1 line
Josh
author maintainer
#15 lore raw

Re: [PATCH 05/15] x86_32: Provide consistent pt_regs

Quoted 31 lines · Josh Poimboeuf
Quoted 21 lines · Peter Zijlstra

Currently pt_regs on x86_32 has an oddity in that kernel regs (!user_mode(regs)) are short two entries (esp/ss). This means that any code trying to use them (typically: regs->sp) needs to jump through some unfortunate hoops.

Change the entry code to fix this up and create a full pt_regs frame.

This then simplifies various trampolines in ftrace and kprobes, the stack unwinder, ptrace, kdump and kgdb.

---
 arch/x86/entry/entry_32.S         |  105 ++++++++++++++++++++++++++++++++++----
 arch/x86/include/asm/kexec.h      |   17 ------
 arch/x86/include/asm/ptrace.h     |   17 ------
 arch/x86/include/asm/stacktrace.h |    2 
 arch/x86/kernel/crash.c           |    8 --
 arch/x86/kernel/ftrace_32.S       |   77 +++++++++++++++------------
 arch/x86/kernel/kgdb.c            |    8 --
 arch/x86/kernel/kprobes/common.h  |    4 -
Quoted 8 lines
 arch/x86/kernel/kprobes/core.c    |   29 ++++------
 arch/x86/kernel/kprobes/opt.c     |   20 ++++---
 arch/x86/kernel/process_32.c      |   16 +----
 arch/x86/kernel/ptrace.c          |   29 ----------
 arch/x86/kernel/time.c            |    3 -
 arch/x86/kernel/unwind_frame.c    |   32 +----------
 arch/x86/kernel/unwind_orc.c      |    2 
 15 files changed, 178 insertions(+), 191 deletions(-)
Quoted 2 lines

I recall writing some of this code (some of the kernel_stack_pointer removal stuff) so please give me a shout-out ;-)

Absolutely, sorry for not doing that. I've added:

"Much thanks to Josh for help with the cleanups!"

author maintainer
#16 lore raw

PATCH 6/15 x86_32: Allow int3_emulate_push()

In reply to Peter Zijlstra

Now that x86_32 has an unconditional gap on the kernel stack frame, the int3_emulate_push() thing will work without further changes.

---
 arch/x86/include/asm/text-patching.h |    2 --
 arch/x86/kernel/ftrace.c             |    7 -------
 2 files changed, 9 deletions(-)
arch/x86/include/asm/text-patching.h +0 -2
@@ -51,7 +51,6 @@ static inline void int3_emulate_jmp(stru
 #define INT3_INSN_SIZE 1
 #define CALL_INSN_SIZE 5
 
-#ifdef CONFIG_X86_64
 static inline void int3_emulate_push(struct pt_regs *regs, unsigned long val)
 {
 	/*
@@ -69,7 +68,6 @@ static inline void int3_emulate_call(str
 	int3_emulate_push(regs, regs->ip - INT3_INSN_SIZE + CALL_INSN_SIZE);
 	int3_emulate_jmp(regs, func);
 }
-#endif /* CONFIG_X86_64 */
 #endif /* !CONFIG_UML_X86 */
 
 #endif /* _ASM_X86_TEXT_PATCHING_H */
arch/x86/kernel/ftrace.c +0 -7
@@ -300,7 +300,6 @@ int ftrace_int3_handler(struct pt_regs *
 
 	ip = regs->ip - INT3_INSN_SIZE;
 
-#ifdef CONFIG_X86_64
 	if (ftrace_location(ip)) {
 		int3_emulate_call(regs, (unsigned long)ftrace_regs_caller);
 		return 1;
@@ -312,12 +311,6 @@ int ftrace_int3_handler(struct pt_regs *
 		int3_emulate_call(regs, ftrace_update_func_call);
 		return 1;
 	}
-#else
-	if (ftrace_location(ip) || is_ftrace_caller(ip)) {
-		int3_emulate_jmp(regs, ip + CALL_INSN_SIZE);
-		return 1;
-	}
-#endif
 
 	return 0;
 }
author maintainer
#17 lore raw

PATCH 7/15 x86: Add int3_emulate_call() selftest

In reply to Peter Zijlstra

Given that the entry_*.S changes for this functionality are somewhat tricky, make sure the paths are tested every boot, instead of on the rare occasion when we trip an INT3 while rewriting text.

---
 arch/x86/kernel/alternative.c |   81 +++++++++++++++++++++++++++++++++++++++---
 1 file changed, 77 insertions(+), 4 deletions(-)
arch/x86/kernel/alternative.c +77 -4
@@ -614,11 +614,83 @@ extern struct paravirt_patch_site __star
 	__stop_parainstructions[];
 #endif	/* CONFIG_PARAVIRT */
 
+/*
+ * Self-test for the INT3 based CALL emulation code.
+ *
+ * This exercises int3_emulate_call() to make sure INT3 pt_regs are set up
+ * properly and that there is a stack gap between the INT3 frame and the
+ * previous context. Without this gap doing a virtual PUSH on the interrupted
+ * stack would corrupt the INT3 IRET frame.
+ *
+ * See entry_{32,64}.S for more details.
+ */
+static void __init int3_magic(unsigned int *ptr)
+{
+	*ptr = 1;
+}
+
+extern __initdata unsigned long int3_selftest_ip; /* defined in asm below */
+
+static int __init
+int3_exception_notify(struct notifier_block *self, unsigned long val, void *data)
+{
+	struct die_args *args = data;
+	struct pt_regs *regs = args->regs;
+
+	if (!regs || user_mode(regs))
+		return NOTIFY_DONE;
+
+	if (val != DIE_INT3)
+		return NOTIFY_DONE;
+
+	if (regs->ip - INT3_INSN_SIZE != int3_selftest_ip)
+		return NOTIFY_DONE;
+
+	int3_emulate_call(regs, (unsigned long)&int3_magic);
+	return NOTIFY_STOP;
+}
+
+static void __init int3_selftest(void)
+{
+	static __initdata struct notifier_block int3_exception_nb = {
+		.notifier_call	= int3_exception_notify,
+		.priority	= INT_MAX-1, /* last */
+	};
+	unsigned int val = 0;
+
+	BUG_ON(register_die_notifier(&int3_exception_nb));
+
+	/*
+	 * Basically: int3_magic(&val); but really complicated :-)
+	 *
+	 * Stick the address of the INT3 instruction into int3_selftest_ip,
+	 * then trigger the INT3, padded with NOPs to match a CALL instruction
+	 * length.
+	 */
+	asm volatile ("1: int3; nop; nop; nop; nop\n\t"
+		      ".pushsection .init.data,\"aw\"\n\t"
+		      ".align " __ASM_SEL(4, 8) "\n\t"
+		      ".type int3_selftest_ip, @object\n\t"
+		      ".size int3_selftest_ip, " __ASM_SEL(4, 8) "\n\t"
+		      "int3_selftest_ip:\n\t"
+		      __ASM_SEL(.long, .quad) " 1b\n\t"
+		      ".popsection\n\t"
+		      : : __ASM_SEL_RAW(a, D) (&val) : "memory");
+
+	BUG_ON(val != 1);
+
+	unregister_die_notifier(&int3_exception_nb);
+}
+
 void __init alternative_instructions(void)
 {
-	/* The patching is not fully atomic, so try to avoid local interruptions
-	   that might execute the to be patched code.
-	   Other CPUs are not running. */
+	int3_selftest();
+
+	/*
+	 * The patching is not fully atomic, so try to avoid local
+	 * interruptions that might execute the to be patched code.
+	 * Other CPUs are not running.
+	 */
 	stop_nmi();
 
 	/*
@@ -643,10 +715,11 @@ void __init alternative_instructions(voi
 					    _text, _etext);
 	}
 
-	if (!uniproc_patched || num_possible_cpus() == 1)
+	if (!uniproc_patched || num_possible_cpus() == 1) {
 		free_init_pages("SMP alternatives",
 				(unsigned long)__smp_locks,
 				(unsigned long)__smp_locks_end);
+	}
 #endif
 
 	apply_paravirt(__parainstructions, __parainstructions_end);
Reviewed-by on v1 7/15
#18 lore raw

Re: [PATCH 07/15] x86: Add int3_emulate_call() selftest

Quoted 6 lines · Peter Zijlstra

Given that the entry_*.S changes for this functionality are somewhat tricky, make sure the paths are tested every boot, instead of on the rare occasion when we trip an INT3 while rewriting text.

Signature · 1 line
Josh
maintainer
#19 lore raw

Re: [PATCH 07/15] x86: Add int3_emulate_call() selftest

Quoted 11 lines · Josh Poimboeuf
Quoted 6 lines · Peter Zijlstra

Given that the entry_*.S changes for this functionality are somewhat tricky, make sure the paths are tested every boot, instead of on the rare occasion when we trip an INT3 while rewriting text.

Looks good to me, too, except that I seriously hate die notifiers that return NOTIFY_STOP, and I eventually want to remove support for them. This can wait, though.

author maintainer
#20 lore raw

Re: [PATCH 07/15] x86: Add int3_emulate_call() selftest

Quoted 7 lines · Andy Lutomirski
Quoted 6 lines · Josh Poimboeuf
Quoted 4 lines · Peter Zijlstra

Given that the entry_*.S changes for this functionality are somewhat tricky, make sure the paths are tested every boot, instead of on the rare occasion when we trip an INT3 while rewriting text.

Quoted 7 lines
Quoted 5 lines
Quoted 2 lines

Looks good to me, too,

I'll translate that into an Acked-by from you, if you don't mind :-)

Quoted 3 lines

except that I seriously hate die notifiers that
return NOTIFY_STOP, and I eventually want to remove support for them. This can wait, though.

Yes, I share your hatred for notifiers in general. But since they are still here and I do think it is a waste to have an unconditional function call in do_int3() just for this, I figured I'd use them.

author maintainer
#21 lore raw

PATCH 8/15 x86/alternatives: Teach text_poke_bp() to emulate instructions

In reply to Peter Zijlstra

In preparation for static_call support, teach text_poke_bp() to emulate instructions, including CALL.

The current text_poke_bp() takes a @handler argument which is used as a jump target when the temporary INT3 is hit by a different CPU.

When patching CALL instructions, this doesn't work because we'd miss the PUSH of the return address. Instead, teach poke_int3_handler() to emulate an instruction, typically the instruction we're patching in.

This fits almost all text_poke_bp() users, except arch_unoptimize_kprobe() which restores random text, and for that site we have to build an explicit emulate instruction.

Cc Daniel Bristot de Oliveira <[email protected]>
Cc Nadav Amit <[email protected]>
---
 arch/x86/include/asm/text-patching.h |    2 -
 arch/x86/kernel/alternative.c        |   47 ++++++++++++++++++++++++++---------
 arch/x86/kernel/jump_label.c         |    3 --
 arch/x86/kernel/kprobes/opt.c        |   11 +++++---
 4 files changed, 46 insertions(+), 17 deletions(-)
arch/x86/include/asm/text-patching.h +1 -1
@@ -37,7 +37,7 @@ extern void text_poke_early(void *addr,
 extern void *text_poke(void *addr, const void *opcode, size_t len);
 extern void *text_poke_kgdb(void *addr, const void *opcode, size_t len);
 extern int poke_int3_handler(struct pt_regs *regs);
-extern void text_poke_bp(void *addr, const void *opcode, size_t len, void *handler);
+extern void text_poke_bp(void *addr, const void *opcode, size_t len, const void *emulate);
 extern int after_bootmem;
 extern __ro_after_init struct mm_struct *poking_mm;
 extern __ro_after_init unsigned long poking_addr;
arch/x86/kernel/alternative.c +36 -11

89 lines not on this page · raw message

arch/x86/kernel/jump_label.c +1 -2
@@ -87,8 +87,7 @@ static void __ref __jump_label_transform
 		return;
 	}
 
-	text_poke_bp((void *)jump_entry_code(entry), code, JUMP_LABEL_NOP_SIZE,
-		     (void *)jump_entry_code(entry) + JUMP_LABEL_NOP_SIZE);
+	text_poke_bp((void *)jump_entry_code(entry), code, JUMP_LABEL_NOP_SIZE, NULL);
 }
 
 void arch_jump_label_transform(struct jump_entry *entry,
arch/x86/kernel/kprobes/opt.c +8 -3
@@ -437,8 +437,7 @@ void arch_optimize_kprobes(struct list_h
 		insn_buff[0] = RELATIVEJUMP_OPCODE;
 		*(s32 *)(&insn_buff[1]) = rel;
 
-		text_poke_bp(op->kp.addr, insn_buff, RELATIVEJUMP_SIZE,
-			     op->optinsn.insn);
+		text_poke_bp(op->kp.addr, insn_buff, RELATIVEJUMP_SIZE, NULL);
 
 		list_del_init(&op->list);
 	}
@@ -448,12 +447,18 @@ void arch_optimize_kprobes(struct list_h
 void arch_unoptimize_kprobe(struct optimized_kprobe *op)
 {
 	u8 insn_buff[RELATIVEJUMP_SIZE];
+	u8 emulate_buff[RELATIVEJUMP_SIZE];
 
 	/* Set int3 to first byte for kprobes */
 	insn_buff[0] = BREAKPOINT_INSTRUCTION;
 	memcpy(insn_buff + 1, op->optinsn.copied_insn, RELATIVE_ADDR_SIZE);
+
+	emulate_buff[0] = RELATIVEJUMP_OPCODE;
+	*(s32 *)(&emulate_buff[1]) = (s32)((long)op->optinsn.insn -
+			((long)op->kp.addr + RELATIVEJUMP_SIZE));
+
 	text_poke_bp(op->kp.addr, insn_buff, RELATIVEJUMP_SIZE,
-		     op->optinsn.insn);
+		     emulate_buff);
 }
 
 /*
#22 lore raw

Re: [PATCH 08/15] x86/alternatives: Teach text_poke_bp() to emulate instructions

Quoted 54 lines · Peter Zijlstra

In preparation for static_call support, teach text_poke_bp() to emulate instructions, including CALL.

The current text_poke_bp() takes a @handler argument which is used as a jump target when the temporary INT3 is hit by a different CPU.

When patching CALL instructions, this doesn't work because we'd miss the PUSH of the return address. Instead, teach poke_int3_handler() to emulate an instruction, typically the instruction we're patching in.

This fits almost all text_poke_bp() users, except arch_unoptimize_kprobe() which restores random text, and for that site we have to build an explicit emulate instruction.

Cc Daniel Bristot de Oliveira <[email protected]>
Cc Nadav Amit <[email protected]>

---

arch/x86/include/asm/text-patching.h |    2 -
arch/x86/kernel/alternative.c        |   47 ++++++++++++++++++++++++++---------
arch/x86/kernel/jump_label.c         |    3 --
arch/x86/kernel/kprobes/opt.c        |   11 +++++---
4 files changed, 46 insertions(+), 17 deletions(-)
arch/x86/include/asm/text-patching.h +1 -1
@@ -37,7 +37,7 @@ extern void text_poke_early(void *addr,
 extern void *text_poke(void *addr, const void *opcode, size_t len);
 extern void *text_poke_kgdb(void *addr, const void *opcode, size_t len);
 extern int poke_int3_handler(struct pt_regs *regs);
-extern void text_poke_bp(void *addr, const void *opcode, size_t len, void *handler);
+extern void text_poke_bp(void *addr, const void *opcode, size_t len, const void *emulate);
 extern int after_bootmem;
 extern __ro_after_init struct mm_struct *poking_mm;
 extern __ro_after_init unsigned long poking_addr;
arch/x86/kernel/alternative.c +7 -1
@@ -921,19 +921,25 @@ static void do_sync_core(void *info)
 }
 
 static bool bp_patching_in_progress;
-static void *bp_int3_handler, *bp_int3_addr;
+static const void *bp_int3_opcode, *bp_int3_addr;
 
 int poke_int3_handler(struct pt_regs *regs)
 {
+	long ip = regs->ip - INT3_INSN_SIZE + CALL_INSN_SIZE;
+	struct opcode {
+		u8 insn;
+		s32 rel;
+	} __packed opcode;
+

arch/x86/kernel/alternative.c static void do_sync_core(void *info) · line 940

Quoted 10 lines
arch/x86/kernel/alternative.c +3 -3
 	/*
 	 * Having observed our INT3 instruction, we now must observe
 	 * bp_patching_in_progress.
 	 *
-	 * 	in_progress = TRUE		INT3
-	 * 	WMB				RMB
-	 * 	write INT3			if (in_progress)
+	 *	in_progress = TRUE		INT3
+	 *	WMB				RMB
+	 *	write INT3			if (in_progress)

I don’t see what has changed in this chunk… Whitespaces?

Quoted 12 lines
arch/x86/kernel/alternative.c +1 -3
 	 *
-	 * Idem for bp_int3_handler.
+	 * Idem for bp_int3_opcode.
 	 */
 	smp_rmb();
 
@@ -943,8 +949,21 @@ int poke_int3_handler(struct pt_regs *re
 	if (user_mode(regs) || regs->ip != (unsigned long)bp_int3_addr)
 		return 0;
 
-	/* set up the specified breakpoint handler */
-	regs->ip = (unsigned long) bp_int3_handler;

arch/x86/kernel/alternative.c int poke_int3_handler(struct pt_regs *re · line 961

Quoted 10 lines
arch/x86/kernel/alternative.c +10 -0
+	opcode = *(struct opcode *)bp_int3_opcode;
+
+	switch (opcode.insn) {
+	case 0xE8: /* CALL */
+		int3_emulate_call(regs, ip + opcode.rel);
+		break;
+
+	case 0xE9: /* JMP */
+		int3_emulate_jmp(regs, ip + opcode.rel);
+		break;

Consider using RELATIVECALL_OPCODE and RELATIVEJUMP_OPCODE instead of the constants (0xE8, 0xE9), just as you do later in the patch.

author maintainer
#23 lore raw

Re: [PATCH 08/15] x86/alternatives: Teach text_poke_bp() to emulate instructions

Quoted 12 lines · Nadav Amit
Quoted 12 lines

int poke_int3_handler(struct pt_regs *regs)
{

(quoted diff) +6 -0
+	long ip = regs->ip - INT3_INSN_SIZE + CALL_INSN_SIZE;
+	struct opcode {
+		u8 insn;
+		s32 rel;
+	} __packed opcode;
+
	/*
	 * Having observed our INT3 instruction, we now must observe
	 * bp_patching_in_progress.
	 *
Quoted 8 lines

Review

Quoted 6 lines
(quoted diff) +3 -3
-	 * 	in_progress = TRUE		INT3
-	 * 	WMB				RMB
-	 * 	write INT3			if (in_progress)
+	 *	in_progress = TRUE		INT3
+	 *	WMB				RMB
+	 *	write INT3			if (in_progress)

I don’t see what has changed in this chunk… Whitespaces?

Yep, my editor kept marking that stuff red (space before tab), which annoyed me enough to fix it.

Quoted 10 lines
Quoted 10 lines
	 *
(quoted diff) +1 -1
-	 * Idem for bp_int3_handler.
+	 * Idem for bp_int3_opcode.
	 */
	smp_rmb();
(quoted diff) +0 -0
@@ -943,8 +949,21 @@ int poke_int3_handler(struct pt_regs *re
 	if (user_mode(regs) || regs->ip != (unsigned long)bp_int3_addr)
 		return 0;
 
Quoted 15 lines

Review int poke_int3_handler(struct pt_regs *re · line 961

Quoted 12 lines
(quoted diff) +10 -2
-	/* set up the specified breakpoint handler */
-	regs->ip = (unsigned long) bp_int3_handler;
+	opcode = *(struct opcode *)bp_int3_opcode;
+
+	switch (opcode.insn) {
+	case 0xE8: /* CALL */
+		int3_emulate_call(regs, ip + opcode.rel);
+		break;
+
+	case 0xE9: /* JMP */
+		int3_emulate_jmp(regs, ip + opcode.rel);
+		break;

Consider using RELATIVECALL_OPCODE and RELATIVEJUMP_OPCODE instead of the constants (0xE8, 0xE9), just as you do later in the patch.

Those are private to kprobes..

but I can do something like so:

arch/x86/include/asm/text-patching.h +8 -2
@@ -48,8 +48,14 @@ static inline void int3_emulate_jmp(stru
 	regs->ip = ip;
 }
 
-#define INT3_INSN_SIZE 1
-#define CALL_INSN_SIZE 5
+#define INT3_INSN_SIZE		1
+#define INT3_INSN_OPCODE	0xCC
+
+#define CALL_INSN_SIZE		5
+#define CALL_INSN_OPCODE	0xE8
+
+#define JMP_INSN_SIZE		5
+#define JMP_INSN_OPCODE		0xE9
 
 static inline void int3_emulate_push(struct pt_regs *regs, unsigned long val)
 {
arch/x86/kernel/alternative.c +2 -2
@@ -952,11 +952,11 @@ int poke_int3_handler(struct pt_regs *re
 	opcode = *(struct opcode *)bp_int3_opcode;
 
 	switch (opcode.insn) {
-	case 0xE8: /* CALL */
+	case CALL_INSN_OPCODE:
 		int3_emulate_call(regs, ip + opcode.rel);
 		break;
 
-	case 0xE9: /* JMP */
+	case JMP_INSN_OPCODE:
 		int3_emulate_jmp(regs, ip + opcode.rel);
 		break;
 
maintainer
#24 lore raw

Re: [PATCH 08/15] x86/alternatives: Teach text_poke_bp() to emulate instructions

Quoted 81 lines · On Fri, 7 Jun 2019 10:20:13 +0200 Peter Zijlstra <[email protected]> wrote:
Quoted 12 lines · Nadav Amit
Quoted 12 lines

int poke_int3_handler(struct pt_regs *regs)
{

(quoted diff) +6 -0
+	long ip = regs->ip - INT3_INSN_SIZE + CALL_INSN_SIZE;
+	struct opcode {
+		u8 insn;
+		s32 rel;
+	} __packed opcode;
+
	/*
	 * Having observed our INT3 instruction, we now must observe
	 * bp_patching_in_progress.
	 *
Quoted 8 lines

Review

Quoted 6 lines
(quoted diff) +3 -3
-	 * 	in_progress = TRUE		INT3
-	 * 	WMB				RMB
-	 * 	write INT3			if (in_progress)
+	 *	in_progress = TRUE		INT3
+	 *	WMB				RMB
+	 *	write INT3			if (in_progress)

I don’t see what has changed in this chunk… Whitespaces?

Yep, my editor kept marking that stuff red (space before tab), which annoyed me enough to fix it.

Quoted 10 lines
Quoted 10 lines
	 *
(quoted diff) +1 -1
-	 * Idem for bp_int3_handler.
+	 * Idem for bp_int3_opcode.
	 */
	smp_rmb();
(quoted diff) +0 -0
@@ -943,8 +949,21 @@ int poke_int3_handler(struct pt_regs *re
 	if (user_mode(regs) || regs->ip != (unsigned long)bp_int3_addr)
 		return 0;
 
Quoted 15 lines

Review int poke_int3_handler(struct pt_regs *re · line 961

Quoted 12 lines
(quoted diff) +10 -2
-	/* set up the specified breakpoint handler */
-	regs->ip = (unsigned long) bp_int3_handler;
+	opcode = *(struct opcode *)bp_int3_opcode;
+
+	switch (opcode.insn) {
+	case 0xE8: /* CALL */
+		int3_emulate_call(regs, ip + opcode.rel);
+		break;
+
+	case 0xE9: /* JMP */
+		int3_emulate_jmp(regs, ip + opcode.rel);
+		break;

Consider using RELATIVECALL_OPCODE and RELATIVEJUMP_OPCODE instead of the constants (0xE8, 0xE9), just as you do later in the patch.

Those are private to kprobes..

but I can do something like so:

arch/x86/include/asm/text-patching.h +8 -2
@@ -48,8 +48,14 @@ static inline void int3_emulate_jmp(stru
 	regs->ip = ip;
 }
 
-#define INT3_INSN_SIZE 1
-#define CALL_INSN_SIZE 5
+#define INT3_INSN_SIZE		1
+#define INT3_INSN_OPCODE	0xCC
+
+#define CALL_INSN_SIZE		5
+#define CALL_INSN_OPCODE	0xE8
+
+#define JMP_INSN_SIZE		5
+#define JMP_INSN_OPCODE		0xE9
 
 static inline void int3_emulate_push(struct pt_regs *regs, unsigned long val)
 {
arch/x86/kernel/alternative.c +0 -0
@@ -952,11 +952,11 @@ int poke_int3_handler(struct pt_regs *re
 	opcode = *(struct opcode *)bp_int3_opcode;
 
 	switch (opcode.insn) {

arch/x86/kernel/alternative.c int poke_int3_handler(struct pt_regs *re · line 962

Quoted 10 lines
arch/x86/kernel/alternative.c +2 -2
-	case 0xE8: /* CALL */
+	case CALL_INSN_OPCODE:
 		int3_emulate_call(regs, ip + opcode.rel);
 		break;
 
-	case 0xE9: /* JMP */
+	case JMP_INSN_OPCODE:
 		int3_emulate_jmp(regs, ip + opcode.rel);
 		break;
 

This looks good. I don't want to make those opcode as private. I would like to share it.

Thank you,

Signature · 1 line
Masami Hiramatsu <[email protected]>
maintainer
#25 lore raw

Re: [PATCH 08/15] x86/alternatives: Teach text_poke_bp() to emulate instructions

In reply to Peter Zijlstra
Quoted 6 lines · On Wed, 05 Jun 2019 15:08:01 +0200 Peter Zijlstra <[email protected]> wrote:

In preparation for static_call support, teach text_poke_bp() to emulate instructions, including CALL.

The current text_poke_bp() takes a @handler argument which is used as a jump target when the temporary INT3 is hit by a different CPU.

Quoted 7 lines

When patching CALL instructions, this doesn't work because we'd miss the PUSH of the return address. Instead, teach poke_int3_handler() to emulate an instruction, typically the instruction we're patching in.

This fits almost all text_poke_bp() users, except arch_unoptimize_kprobe() which restores random text, and for that site we have to build an explicit emulate instruction.

Hm, actually it doesn't restores randome text, since the first byte must always be int3. As the function name means, it just unoptimizes (jump based optprobe -> int3 based kprobe).
Anyway, that is not an issue. With this patch, optprobe must still work.

Quoted 63 lines
Cc Daniel Bristot de Oliveira <[email protected]>
Cc Nadav Amit <[email protected]>
---
 arch/x86/include/asm/text-patching.h |    2 -
 arch/x86/kernel/alternative.c        |   47 ++++++++++++++++++++++++++---------
 arch/x86/kernel/jump_label.c         |    3 --
 arch/x86/kernel/kprobes/opt.c        |   11 +++++---
 4 files changed, 46 insertions(+), 17 deletions(-)
arch/x86/include/asm/text-patching.h +1 -1
@@ -37,7 +37,7 @@ extern void text_poke_early(void *addr,
 extern void *text_poke(void *addr, const void *opcode, size_t len);
 extern void *text_poke_kgdb(void *addr, const void *opcode, size_t len);
 extern int poke_int3_handler(struct pt_regs *regs);
-extern void text_poke_bp(void *addr, const void *opcode, size_t len, void *handler);
+extern void text_poke_bp(void *addr, const void *opcode, size_t len, const void *emulate);
 extern int after_bootmem;
 extern __ro_after_init struct mm_struct *poking_mm;
 extern __ro_after_init unsigned long poking_addr;
arch/x86/kernel/alternative.c +13 -7
@@ -921,19 +921,25 @@ static void do_sync_core(void *info)
 }
 
 static bool bp_patching_in_progress;
-static void *bp_int3_handler, *bp_int3_addr;
+static const void *bp_int3_opcode, *bp_int3_addr;
 
 int poke_int3_handler(struct pt_regs *regs)
 {
+	long ip = regs->ip - INT3_INSN_SIZE + CALL_INSN_SIZE;
+	struct opcode {
+		u8 insn;
+		s32 rel;
+	} __packed opcode;
+
 	/*
 	 * Having observed our INT3 instruction, we now must observe
 	 * bp_patching_in_progress.
 	 *
-	 * 	in_progress = TRUE		INT3
-	 * 	WMB				RMB
-	 * 	write INT3			if (in_progress)
+	 *	in_progress = TRUE		INT3
+	 *	WMB				RMB
+	 *	write INT3			if (in_progress)
 	 *
-	 * Idem for bp_int3_handler.
+	 * Idem for bp_int3_opcode.
 	 */
 	smp_rmb();
 
@@ -943,8 +949,21 @@ int poke_int3_handler(struct pt_regs *re
 	if (user_mode(regs) || regs->ip != (unsigned long)bp_int3_addr)
 		return 0;
 
-	/* set up the specified breakpoint handler */
-	regs->ip = (unsigned long) bp_int3_handler;
+	opcode = *(struct opcode *)bp_int3_opcode;
+

arch/x86/kernel/alternative.c int poke_int3_handler(struct pt_regs *re · line 963

Quoted 10 lines
arch/x86/kernel/alternative.c +10 -0
+	switch (opcode.insn) {
+	case 0xE8: /* CALL */
+		int3_emulate_call(regs, ip + opcode.rel);
+		break;
+
+	case 0xE9: /* JMP */
+		int3_emulate_jmp(regs, ip + opcode.rel);
+		break;
+
+	default: /* assume NOP */

Shouldn't we check whether it is actually NOP here?

arch/x86/kernel/alternative.c int poke_int3_handler(struct pt_regs *re · line 966

Quoted 3 lines
arch/x86/kernel/alternative.c +3 -0
+		int3_emulate_jmp(regs, ip);
+		break;
+	}

BTW, if we fix the length of patching always 5 bytes and allow user to apply it only from/to jump/call/nop, we may be better to remove "len" and rename it, something like "text_poke_branch" etc.

Thank you,

Quoted 87 lines
arch/x86/kernel/alternative.c +10 -4
 	return 1;
 }
@@ -955,7 +974,7 @@ NOKPROBE_SYMBOL(poke_int3_handler);
  * @addr:	address to patch
  * @opcode:	opcode of new instruction
  * @len:	length to copy
- * @handler:	address to jump to when the temporary breakpoint is hit
+ * @emulate:	opcode to emulate, when NULL use @opcode
  *
  * Modify multi-byte instruction by using int3 breakpoint on SMP.
  * We completely avoid stop_machine() here, and achieve the
@@ -970,19 +989,25 @@ NOKPROBE_SYMBOL(poke_int3_handler);
  *	  replacing opcode
  *	- sync cores
  */
-void text_poke_bp(void *addr, const void *opcode, size_t len, void *handler)
+void text_poke_bp(void *addr, const void *opcode, size_t len, const void *emulate)
 {
 	unsigned char int3 = 0xcc;
 
-	bp_int3_handler = handler;
+	bp_int3_opcode = emulate ?: opcode;
 	bp_int3_addr = (u8 *)addr + sizeof(int3);
 	bp_patching_in_progress = true;
 
 	lockdep_assert_held(&text_mutex);
 
 	/*
+	 * poke_int3_handler() relies on @opcode being a 5 byte instruction;
+	 * notably a JMP, CALL or NOP5_ATOMIC.
+	 */
+	BUG_ON(len != 5);
+
+	/*
 	 * Corresponding read barrier in int3 notifier for making sure the
-	 * in_progress and handler are correctly ordered wrt. patching.
+	 * in_progress and opcode are correctly ordered wrt. patching.
 	 */
 	smp_wmb();
 
arch/x86/kernel/jump_label.c +1 -2
@@ -87,8 +87,7 @@ static void __ref __jump_label_transform
 		return;
 	}
 
-	text_poke_bp((void *)jump_entry_code(entry), code, JUMP_LABEL_NOP_SIZE,
-		     (void *)jump_entry_code(entry) + JUMP_LABEL_NOP_SIZE);
+	text_poke_bp((void *)jump_entry_code(entry), code, JUMP_LABEL_NOP_SIZE, NULL);
 }
 
 void arch_jump_label_transform(struct jump_entry *entry,
arch/x86/kernel/kprobes/opt.c +8 -3
@@ -437,8 +437,7 @@ void arch_optimize_kprobes(struct list_h
 		insn_buff[0] = RELATIVEJUMP_OPCODE;
 		*(s32 *)(&insn_buff[1]) = rel;
 
-		text_poke_bp(op->kp.addr, insn_buff, RELATIVEJUMP_SIZE,
-			     op->optinsn.insn);
+		text_poke_bp(op->kp.addr, insn_buff, RELATIVEJUMP_SIZE, NULL);
 
 		list_del_init(&op->list);
 	}
@@ -448,12 +447,18 @@ void arch_optimize_kprobes(struct list_h
 void arch_unoptimize_kprobe(struct optimized_kprobe *op)
 {
 	u8 insn_buff[RELATIVEJUMP_SIZE];
+	u8 emulate_buff[RELATIVEJUMP_SIZE];
 
 	/* Set int3 to first byte for kprobes */
 	insn_buff[0] = BREAKPOINT_INSTRUCTION;
 	memcpy(insn_buff + 1, op->optinsn.copied_insn, RELATIVE_ADDR_SIZE);
+
+	emulate_buff[0] = RELATIVEJUMP_OPCODE;
+	*(s32 *)(&emulate_buff[1]) = (s32)((long)op->optinsn.insn -
+			((long)op->kp.addr + RELATIVEJUMP_SIZE));
+
 	text_poke_bp(op->kp.addr, insn_buff, RELATIVEJUMP_SIZE,
-		     op->optinsn.insn);
+		     emulate_buff);
 }
 
 /*
Signature · 1 line
Masami Hiramatsu <[email protected]>
author maintainer
#26 lore raw

Re: [PATCH 08/15] x86/alternatives: Teach text_poke_bp() to emulate instructions

Quoted 8 lines · Masami Hiramatsu
Quoted 3 lines

This fits almost all text_poke_bp() users, except arch_unoptimize_kprobe() which restores random text, and for that site we have to build an explicit emulate instruction.

Hm, actually it doesn't restores randome text, since the first byte must always be int3. As the function name means, it just unoptimizes (jump based optprobe -> int3 based kprobe).
Anyway, that is not an issue. With this patch, optprobe must still work.

I thought it basically restored 5 bytes of original text (with no guarantee it is a single instruction, or even a complete instruction), with the first byte replaced with INT3.

Quoted 4 lines
Quoted 4 lines
arch/x86/kernel/alternative.c +0 -0
@@ -943,8 +949,21 @@ int poke_int3_handler(struct pt_regs *re
 	if (user_mode(regs) || regs->ip != (unsigned long)bp_int3_addr)
 		return 0;
 
Quoted 16 lines
Quoted 4 lines
arch/x86/kernel/alternative.c +2 -2
-	/* set up the specified breakpoint handler */
-	regs->ip = (unsigned long) bp_int3_handler;
+	opcode = *(struct opcode *)bp_int3_opcode;
+

arch/x86/kernel/alternative.c int poke_int3_handler(struct pt_regs *re · line 963

Quoted 10 lines
arch/x86/kernel/alternative.c +10 -0
+	switch (opcode.insn) {
+	case 0xE8: /* CALL */
+		int3_emulate_call(regs, ip + opcode.rel);
+		break;
+
+	case 0xE9: /* JMP */
+		int3_emulate_jmp(regs, ip + opcode.rel);
+		break;
+
+	default: /* assume NOP */

Shouldn't we check whether it is actually NOP here?

I was/am lazy and didn't want to deal with:

arch/x86/include/asm/nops.h:#define GENERIC_NOP5_ATOMIC NOP_DS_PREFIX,GENERIC_NOP4 arch/x86/include/asm/nops.h:#define K8_NOP5_ATOMIC 0x66,K8_NOP4 arch/x86/include/asm/nops.h:#define K7_NOP5_ATOMIC NOP_DS_PREFIX,K7_NOP4 arch/x86/include/asm/nops.h:#define P6_NOP5_ATOMIC P6_NOP5

But maybe we should check for all the various NOP5 variants and BUG() on anything unexpected.

Quoted 7 lines

arch/x86/kernel/alternative.c int poke_int3_handler(struct pt_regs *re · line 966

Quoted 3 lines
arch/x86/kernel/alternative.c +3 -0
+		int3_emulate_jmp(regs, ip);
+		break;
+	}

BTW, if we fix the length of patching always 5 bytes and allow user to apply it only from/to jump/call/nop, we may be better to remove "len" and rename it, something like "text_poke_branch" etc.

I considered it; but was thinking we could still allow patching other instructions, we'd just have to extend the emulation in poke_int3_handler().

Then again, if/when we want to do that, we can also restore the @len argument again.

#27 lore raw

Re: [PATCH 08/15] x86/alternatives: Teach text_poke_bp() to emulate instructions

Quoted 7 lines · Peter Zijlstra

I was/am lazy and didn't want to deal with:

arch/x86/include/asm/nops.h:#define GENERIC_NOP5_ATOMIC NOP_DS_PREFIX,GENERIC_NOP4 arch/x86/include/asm/nops.h:#define K8_NOP5_ATOMIC 0x66,K8_NOP4 arch/x86/include/asm/nops.h:#define K7_NOP5_ATOMIC NOP_DS_PREFIX,K7_NOP4 arch/x86/include/asm/nops.h:#define P6_NOP5_ATOMIC P6_NOP5

Ugh. Maybe we could just pick one atomic sequence, and not have the magic atomic nops be dynamic.

It's essentially what STATIC_KEY_INIT_NOP #define seems to do anyway.

NOP5_ATOMIC is already special, and not used for the normal nop rewriting, only for kprobe/jump_label/ftrace.

So I suspect we could just replace all cases of

ideal_nops[NOP_ATOMIC5]

with

STATIC_KEY_INIT_NOP

and get rid of the whole "let's optimize the atomic 5-byte nop" entirely.

Hmm?

By definition, NOP_ATOMIC5 is just a single nop anyway, it's not used for the potentially more complex alternative rewriting cases.

Linus

author maintainer
#28 lore raw

Re: [PATCH 08/15] x86/alternatives: Teach text_poke_bp() to emulate instructions

Quoted 11 lines · Linus Torvalds
Quoted 7 lines · Peter Zijlstra

I was/am lazy and didn't want to deal with:

arch/x86/include/asm/nops.h:#define GENERIC_NOP5_ATOMIC NOP_DS_PREFIX,GENERIC_NOP4 arch/x86/include/asm/nops.h:#define K8_NOP5_ATOMIC 0x66,K8_NOP4 arch/x86/include/asm/nops.h:#define K7_NOP5_ATOMIC NOP_DS_PREFIX,K7_NOP4 arch/x86/include/asm/nops.h:#define P6_NOP5_ATOMIC P6_NOP5

Ugh. Maybe we could just pick one atomic sequence, and not have the magic atomic nops be dynamic.

That'd be nice..

Quoted 1 line

It's essentially what STATIC_KEY_INIT_NOP #define seems to do anyway.

Well, that picks something, we'll overwrite it with the ideal nop later, once we've figured out what it should be.

Quoted 2 lines

NOP5_ATOMIC is already special, and not used for the normal nop rewriting, only for kprobe/jump_label/ftrace.

Right, but esp ftrace means there's a _lot_ of them around.

Quoted 11 lines

So I suspect we could just replace all cases of

ideal_nops[NOP_ATOMIC5]

with

STATIC_KEY_INIT_NOP

and get rid of the whole "let's optimize the atomic 5-byte nop" entirely.

Hmm?

So we have:

GENERIC (x86_32): leal ds:0x00(,%esi,1),%esi
K8 (x86_64): osp osp osp osp nop
K7 (x86_32): leal ds:0x00(,%eax,1),%eax
P6 (x86_64): nopl 0x00(%eax,%eax,1)

And I guess the $64k question is if there's any actual performance difference between the k8 and p6 variants on chips we still care about.

Most modern chips seem to end up selecting p6.

Anyway, the proposed patch looks like so:

---
Subject: x86: Remove ideal_nops[NOP_ATOMIC5]

By picking a single/fixed NOP5_ATOMIC instruction things become simpler.

---
 arch/x86/include/asm/jump_label.h | 12 +++---------
 arch/x86/include/asm/nops.h       | 17 ++++++++---------
 arch/x86/kernel/alternative.c     |  8 --------
 arch/x86/kernel/ftrace.c          |  3 ++-
 arch/x86/kernel/jump_label.c      | 37 ++++++-------------------------------
 arch/x86/kernel/kprobes/core.c    |  3 ++-
 6 files changed, 21 insertions(+), 59 deletions(-)
arch/x86/include/asm/jump_label.h +3 -9
index 65191ce8e1cf..a3d45abcda95 100644
@@ -4,12 +4,6 @@
 
 #define JUMP_LABEL_NOP_SIZE 5
 
-#ifdef CONFIG_X86_64
-# define STATIC_KEY_INIT_NOP P6_NOP5_ATOMIC
-#else
-# define STATIC_KEY_INIT_NOP GENERIC_NOP5_ATOMIC
-#endif
-
 #include <asm/asm.h>
 #include <asm/nops.h>
 
@@ -21,7 +15,7 @@
 static __always_inline bool arch_static_branch(struct static_key *key, bool branch)
 {
 	asm_volatile_goto("1:"
-		".byte " __stringify(STATIC_KEY_INIT_NOP) "\n\t"
+		".byte " __stringify(NOP5_ATOMIC) "\n\t"
 		".pushsection __jump_table,  \"aw\" \n\t"
 		_ASM_ALIGN "\n\t"
 		".long 1b - ., %l[l_yes] - . \n\t"
@@ -61,7 +55,7 @@ static __always_inline bool arch_static_branch_jump(struct static_key *key, bool
 	.long		\target - .Lstatic_jump_after_\@
 .Lstatic_jump_after_\@:
 	.else
-	.byte		STATIC_KEY_INIT_NOP
+	.byte		NOP5_ATOMIC
 	.endif
 	.pushsection __jump_table, "aw"
 	_ASM_ALIGN
@@ -73,7 +67,7 @@ static __always_inline bool arch_static_branch_jump(struct static_key *key, bool
 .macro STATIC_JUMP_IF_FALSE target, key, def
 .Lstatic_jump_\@:
 	.if \def
-	.byte		STATIC_KEY_INIT_NOP
+	.byte		NOP5_ATOMIC
 	.else
 	/* Equivalent to "jmp.d32 \target" */
 	.byte		0xe9
arch/x86/include/asm/nops.h +8 -9
index 12f12b5cf2ca..14cf05e645f5 100644
@@ -28,7 +28,6 @@
 #define GENERIC_NOP6 0x8d,0xb6,0x00,0x00,0x00,0x00
 #define GENERIC_NOP7 0x8d,0xb4,0x26,0x00,0x00,0x00,0x00
 #define GENERIC_NOP8 GENERIC_NOP1,GENERIC_NOP7
-#define GENERIC_NOP5_ATOMIC NOP_DS_PREFIX,GENERIC_NOP4
 
 /* Opteron 64bit nops
    1: nop
@@ -44,7 +43,6 @@
 #define K8_NOP6	K8_NOP3,K8_NOP3
 #define K8_NOP7	K8_NOP4,K8_NOP3
 #define K8_NOP8	K8_NOP4,K8_NOP4
-#define K8_NOP5_ATOMIC 0x66,K8_NOP4
 
 /* K7 nops
    uses eax dependencies (arbitrary choice)
@@ -63,7 +61,6 @@
 #define K7_NOP6	0x8d,0x80,0,0,0,0
 #define K7_NOP7	0x8D,0x04,0x05,0,0,0,0
 #define K7_NOP8	K7_NOP7,K7_NOP1
-#define K7_NOP5_ATOMIC NOP_DS_PREFIX,K7_NOP4
 
 /* P6 nops
    uses eax dependencies (Intel-recommended choice)
@@ -86,7 +83,12 @@
 #define P6_NOP6	0x66,0x0f,0x1f,0x44,0x00,0
 #define P6_NOP7	0x0f,0x1f,0x80,0,0,0,0
 #define P6_NOP8	0x0f,0x1f,0x84,0x00,0,0,0,0
-#define P6_NOP5_ATOMIC P6_NOP5
+
+#ifdef CONFIG_X86_64
+#define NOP5_ATOMIC	P6_NOP5
+#else
+#define NOP5_ATOMIC	NOP_DS_PREFIX,GENERIC_NOP4
+#endif
 
 #ifdef __ASSEMBLY__
 #define _ASM_MK_NOP(x) .byte x
@@ -103,7 +105,6 @@
 #define ASM_NOP6 _ASM_MK_NOP(K7_NOP6)
 #define ASM_NOP7 _ASM_MK_NOP(K7_NOP7)
 #define ASM_NOP8 _ASM_MK_NOP(K7_NOP8)
-#define ASM_NOP5_ATOMIC _ASM_MK_NOP(K7_NOP5_ATOMIC)
 #elif defined(CONFIG_X86_P6_NOP)
 #define ASM_NOP1 _ASM_MK_NOP(P6_NOP1)
 #define ASM_NOP2 _ASM_MK_NOP(P6_NOP2)
@@ -113,7 +114,6 @@
 #define ASM_NOP6 _ASM_MK_NOP(P6_NOP6)
 #define ASM_NOP7 _ASM_MK_NOP(P6_NOP7)
 #define ASM_NOP8 _ASM_MK_NOP(P6_NOP8)
-#define ASM_NOP5_ATOMIC _ASM_MK_NOP(P6_NOP5_ATOMIC)
 #elif defined(CONFIG_X86_64)
 #define ASM_NOP1 _ASM_MK_NOP(K8_NOP1)
 #define ASM_NOP2 _ASM_MK_NOP(K8_NOP2)
@@ -123,7 +123,6 @@
 #define ASM_NOP6 _ASM_MK_NOP(K8_NOP6)
 #define ASM_NOP7 _ASM_MK_NOP(K8_NOP7)
 #define ASM_NOP8 _ASM_MK_NOP(K8_NOP8)
-#define ASM_NOP5_ATOMIC _ASM_MK_NOP(K8_NOP5_ATOMIC)
 #else
 #define ASM_NOP1 _ASM_MK_NOP(GENERIC_NOP1)
 #define ASM_NOP2 _ASM_MK_NOP(GENERIC_NOP2)
@@ -133,11 +132,11 @@
 #define ASM_NOP6 _ASM_MK_NOP(GENERIC_NOP6)
 #define ASM_NOP7 _ASM_MK_NOP(GENERIC_NOP7)
 #define ASM_NOP8 _ASM_MK_NOP(GENERIC_NOP8)
-#define ASM_NOP5_ATOMIC _ASM_MK_NOP(GENERIC_NOP5_ATOMIC)
 #endif
 
+#define ASM_NOP5_ATOMIC _ASM_MK_NOP(NOP5_ATOMIC)
+
 #define ASM_NOP_MAX 8
-#define NOP_ATOMIC5 (ASM_NOP_MAX+1)	/* Entry for the 5-byte atomic NOP */
 
 #ifndef __ASSEMBLY__
 extern const unsigned char * const *ideal_nops;
arch/x86/kernel/alternative.c +0 -8
index 0d57015114e7..4c0250049d4f 100644
@@ -90,7 +90,6 @@ static const unsigned char intelnops[] =
 	GENERIC_NOP6,
 	GENERIC_NOP7,
 	GENERIC_NOP8,
-	GENERIC_NOP5_ATOMIC
 };
 static const unsigned char * const intel_nops[ASM_NOP_MAX+2] =
 {
@@ -103,7 +102,6 @@ static const unsigned char * const intel_nops[ASM_NOP_MAX+2] =
 	intelnops + 1 + 2 + 3 + 4 + 5,
 	intelnops + 1 + 2 + 3 + 4 + 5 + 6,
 	intelnops + 1 + 2 + 3 + 4 + 5 + 6 + 7,
-	intelnops + 1 + 2 + 3 + 4 + 5 + 6 + 7 + 8,
 };
 #endif
 
@@ -118,7 +116,6 @@ static const unsigned char k8nops[] =
 	K8_NOP6,
 	K8_NOP7,
 	K8_NOP8,
-	K8_NOP5_ATOMIC
 };
 static const unsigned char * const k8_nops[ASM_NOP_MAX+2] =
 {
@@ -131,7 +128,6 @@ static const unsigned char * const k8_nops[ASM_NOP_MAX+2] =
 	k8nops + 1 + 2 + 3 + 4 + 5,
 	k8nops + 1 + 2 + 3 + 4 + 5 + 6,
 	k8nops + 1 + 2 + 3 + 4 + 5 + 6 + 7,
-	k8nops + 1 + 2 + 3 + 4 + 5 + 6 + 7 + 8,
 };
 #endif
 
@@ -146,7 +142,6 @@ static const unsigned char k7nops[] =
 	K7_NOP6,
 	K7_NOP7,
 	K7_NOP8,
-	K7_NOP5_ATOMIC
 };
 static const unsigned char * const k7_nops[ASM_NOP_MAX+2] =
 {
@@ -159,7 +154,6 @@ static const unsigned char * const k7_nops[ASM_NOP_MAX+2] =
 	k7nops + 1 + 2 + 3 + 4 + 5,
 	k7nops + 1 + 2 + 3 + 4 + 5 + 6,
 	k7nops + 1 + 2 + 3 + 4 + 5 + 6 + 7,
-	k7nops + 1 + 2 + 3 + 4 + 5 + 6 + 7 + 8,
 };
 #endif
 
@@ -174,7 +168,6 @@ static const unsigned char p6nops[] =
 	P6_NOP6,
 	P6_NOP7,
 	P6_NOP8,
-	P6_NOP5_ATOMIC
 };
 static const unsigned char * const p6_nops[ASM_NOP_MAX+2] =
 {
@@ -187,7 +180,6 @@ static const unsigned char * const p6_nops[ASM_NOP_MAX+2] =
 	p6nops + 1 + 2 + 3 + 4 + 5,
 	p6nops + 1 + 2 + 3 + 4 + 5 + 6,
 	p6nops + 1 + 2 + 3 + 4 + 5 + 6 + 7,
-	p6nops + 1 + 2 + 3 + 4 + 5 + 6 + 7 + 8,
 };
 #endif
 
arch/x86/kernel/ftrace.c +2 -1
index 0927bb158ffc..6ea5ea506a5f 100644
@@ -101,7 +101,8 @@ static unsigned long text_ip_addr(unsigned long ip)
 
 static const unsigned char *ftrace_nop_replace(void)
 {
-	return ideal_nops[NOP_ATOMIC5];
+	static const unsigned char nop[] = { NOP5_ATOMIC };
+	return nop;
 }
 
 static int
arch/x86/kernel/jump_label.c +6 -31
index e631c358f7f4..5a27cf6e1c73 100644
@@ -40,8 +40,7 @@ static void __ref __jump_label_transform(struct jump_entry *entry,
 					 int init)
 {
 	union jump_code_union jmp;
-	const unsigned char default_nop[] = { STATIC_KEY_INIT_NOP };
-	const unsigned char *ideal_nop = ideal_nops[NOP_ATOMIC5];
+	const unsigned char nop[] = { NOP5_ATOMIC };
 	const void *expect, *code;
 	int line;
 
@@ -50,21 +49,13 @@ static void __ref __jump_label_transform(struct jump_entry *entry,
 		     (jump_entry_code(entry) + JUMP_LABEL_NOP_SIZE);
 
 	if (type == JUMP_LABEL_JMP) {
-		if (init) {
-			expect = default_nop; line = __LINE__;
-		} else {
-			expect = ideal_nop; line = __LINE__;
-		}
-
+		expect = nop;
+		line = __LINE__;
 		code = &jmp.code;
 	} else {
-		if (init) {
-			expect = default_nop; line = __LINE__;
-		} else {
-			expect = &jmp.code; line = __LINE__;
-		}
-
-		code = ideal_nop;
+		expect = &jmp.code;
+		line = __LINE__;
+		code = nop;
 	}
 
 	if (memcmp((void *)jump_entry_code(entry), expect, JUMP_LABEL_NOP_SIZE))
@@ -108,22 +99,6 @@ static enum {
 __init_or_module void arch_jump_label_transform_static(struct jump_entry *entry,
 				      enum jump_label_type type)
 {
-	/*
-	 * This function is called at boot up and when modules are
-	 * first loaded. Check if the default nop, the one that is
-	 * inserted at compile time, is the ideal nop. If it is, then
-	 * we do not need to update the nop, and we can leave it as is.
-	 * If it is not, then we need to update the nop to the ideal nop.
-	 */
-	if (jlstate == JL_STATE_START) {
-		const unsigned char default_nop[] = { STATIC_KEY_INIT_NOP };
-		const unsigned char *ideal_nop = ideal_nops[NOP_ATOMIC5];
-
-		if (memcmp(ideal_nop, default_nop, 5) != 0)
-			jlstate = JL_STATE_UPDATE;
-		else
-			jlstate = JL_STATE_NO_UPDATE;
-	}
 	if (jlstate == JL_STATE_UPDATE)
 		__jump_label_transform(entry, type, 1);
 }
arch/x86/kernel/kprobes/core.c +2 -1
index 6afd8061dbae..5b9aa5608d0d 100644
@@ -204,6 +204,7 @@ int can_boost(struct insn *insn, void *addr)
 static unsigned long
 __recover_probed_insn(kprobe_opcode_t *buf, unsigned long addr)
 {
+	static const unsigned char nop[] = { NOP5_ATOMIC };
 	struct kprobe *kp;
 	unsigned long faddr;
 
@@ -247,7 +248,7 @@ __recover_probed_insn(kprobe_opcode_t *buf, unsigned long addr)
 		return 0UL;
 
 	if (faddr)
-		memcpy(buf, ideal_nops[NOP_ATOMIC5], 5);
+		memcpy(buf, nop, 5);
 	else
 		buf[0] = kp->opcode;
 	return (unsigned long)buf;
#29 lore raw

Re: [PATCH 08/15] x86/alternatives: Teach text_poke_bp() to emulate instructions

In reply to Peter Zijlstra
Quoted 10 lines · Peter Zijlstra
Quoted 4 lines · Masami Hiramatsu
Quoted 3 lines

This fits almost all text_poke_bp() users, except arch_unoptimize_kprobe() which restores random text, and for that site we have to build an explicit emulate instruction.

Quoted 9 lines
Quoted 4 lines

Hm, actually it doesn't restores randome text, since the first byte must always be int3. As the function name means, it just unoptimizes (jump based optprobe -> int3 based kprobe).
Anyway, that is not an issue. With this patch, optprobe must still work.

I thought it basically restored 5 bytes of original text (with no guarantee it is a single instruction, or even a complete instruction), with the first byte replaced with INT3.

I am surely missing some kprobe context, but is it really safe to use this mechanism to replace more than one instruction?

#30 lore raw

Re: [PATCH 08/15] x86/alternatives: Teach text_poke_bp() to emulate instructions

Quoted 18 lines · Andy Lutomirski
Quoted 3 lines

On Jun 7, 2019, at 10:34 AM, Peter Zijlstra <[email protected]>

Quoted 5 lines · wrote:
Quoted 2 lines · Masami Hiramatsu
Quoted 2 lines

This fits almost all text_poke_bp() users, except arch_unoptimize_kprobe() which restores random text, and for that

site

Quoted 6 lines
Quoted 6 lines
Quoted 1 line

we have to build an explicit emulate instruction.

Hm, actually it doesn't restores randome text, since the first byte must always be int3. As the function name means, it just unoptimizes (jump based optprobe -> int3 based kprobe).
Anyway, that is not an issue. With this patch, optprobe must still

work.

Quoted 1 line
Quoted 8 lines
Quoted 2 lines

I thought it basically restored 5 bytes of original text (with no guarantee it is a single instruction, or even a complete

instruction),

Quoted 2 lines

with the first byte replaced with INT3.

I am surely missing some kprobe context, but is it really safe to use this mechanism to replace more than one instruction?

I don't see how it could be, except *perhaps* inside an NMI have, because you could have a preempted or interrupted now having an in-memory IP pointing inside the middle of the region you are intending to patch.

Signature · 1 line
Sent from my Android device with K-9 Mail. Please excuse my brevity.
author maintainer
#31 lore raw

Re: [PATCH 08/15] x86/alternatives: Teach text_poke_bp() to emulate instructions

In reply to Andy Lutomirski
Quoted 15 lines · Andy Lutomirski
Quoted 15 lines · Peter Zijlstra
Quoted 8 lines · Masami Hiramatsu
Quoted 3 lines

This fits almost all text_poke_bp() users, except arch_unoptimize_kprobe() which restores random text, and for that site we have to build an explicit emulate instruction.

Hm, actually it doesn't restores randome text, since the first byte must always be int3. As the function name means, it just unoptimizes (jump based optprobe -> int3 based kprobe).
Anyway, that is not an issue. With this patch, optprobe must still work.

Quoted 7 lines
Quoted 4 lines

I thought it basically restored 5 bytes of original text (with no guarantee it is a single instruction, or even a complete instruction), with the first byte replaced with INT3.

I am surely missing some kprobe context, but is it really safe to use this mechanism to replace more than one instruction?

I'm not entirely up-to-scratch here, so Masami, please correct me if I'm wrong.

So what happens is that arch_prepare_optimized_kprobe() <- copy_optimized_instructions() copies however much of the instruction stream is required such that we can overwrite the instruction at @addr with a 5 byte jump.

arch_optimize_kprobe() then does the text_poke_bp() that replaces the instruction @addr with int3, copies the rel jump address and overwrites the int3 with jmp.

And I'm thinking the problem is with something like:

@addr: nop nop nop nop nop

We copy out the nops into the trampoline, overwrite the first nop with an INT3, overwrite the remaining nops with the rel addr, but oops, another CPU can still be executing one of those NOPs, right?

I'm thinking we could fix this by first writing INT3 into all relevant instructions, which is going to be messy, given the current code base.

author maintainer
#32 lore raw

Re: [PATCH 08/15] x86/alternatives: Teach text_poke_bp() to emulate instructions

Quoted 19 lines · Peter Zijlstra
Quoted 2 lines · Andy Lutomirski

I am surely missing some kprobe context, but is it really safe to use this mechanism to replace more than one instruction?

I'm not entirely up-to-scratch here, so Masami, please correct me if I'm wrong.

So what happens is that arch_prepare_optimized_kprobe() <- copy_optimized_instructions() copies however much of the instruction stream is required such that we can overwrite the instruction at @addr with a 5 byte jump.

arch_optimize_kprobe() then does the text_poke_bp() that replaces the instruction @addr with int3, copies the rel jump address and overwrites the int3 with jmp.

And I'm thinking the problem is with something like:

Quoted 8 lines

@addr: nop nop nop nop nop

We copy out the nops into the trampoline, overwrite the first nop with an INT3, overwrite the remaining nops with the rel addr, but oops, another CPU can still be executing one of those NOPs, right?

I'm thinking we could fix this by first writing INT3 into all relevant instructions, which is going to be messy, given the current code base.

Maybe not that bad; how's something like this?

(completely untested)

---
 arch/x86/kernel/alternative.c | 20 ++++++++++++++++++--
 1 file changed, 18 insertions(+), 2 deletions(-)
arch/x86/kernel/alternative.c +18 -2
index 0d57015114e7..8f643dabea72 100644
@@ -24,6 +24,7 @@
 #include <asm/tlbflush.h>
 #include <asm/io.h>
 #include <asm/fixmap.h>
+#include <asm/insn.h>
 
 int __read_mostly alternatives_patched;
 
@@ -849,6 +850,7 @@ static void do_sync_core(void *info)
 
 static bool bp_patching_in_progress;
 static void *bp_int3_handler, *bp_int3_addr;
+static unsigned int bp_int3_length;
 
 int poke_int3_handler(struct pt_regs *regs)
 {
@@ -867,7 +869,11 @@ int poke_int3_handler(struct pt_regs *regs)
 	if (likely(!bp_patching_in_progress))
 		return 0;
 
-	if (user_mode(regs) || regs->ip != (unsigned long)bp_int3_addr)
+	if (user_mode(regs))
+		return 0;
+
+	if (regs->ip < (unsigned long)bp_int3_addr ||
+	    regs->ip >= (unsigned long)bp_int3_addr + bp_int3_length)
 		return 0;
 
 	/* set up the specified breakpoint handler */
@@ -900,9 +906,12 @@ NOKPROBE_SYMBOL(poke_int3_handler);
 void text_poke_bp(void *addr, const void *opcode, size_t len, void *handler)
 {
 	unsigned char int3 = 0xcc;
+	void *kaddr = addr;
+	struct insn insn;
 
 	bp_int3_handler = handler;
 	bp_int3_addr = (u8 *)addr + sizeof(int3);
+	bp_int3_length = len - sizeof(int3);
 	bp_patching_in_progress = true;
 
 	lockdep_assert_held(&text_mutex);
@@ -913,7 +922,14 @@ void text_poke_bp(void *addr, const void *opcode, size_t len, void *handler)
 	 */
 	smp_wmb();
 
-	text_poke(addr, &int3, sizeof(int3));
+	do {
+		kernel_insn_init(&insn, kaddr, MAX_INSN_SIZE);
+		insn_get_length(&insn);
+
+		text_poke(kaddr, &int3, sizeof(int3));
+
+		kaddr += insn.length;
+	} while (kaddr < addr + len);
 
 	on_each_cpu(do_sync_core, NULL, 1);
 
author maintainer
#33 lore raw

Re: [PATCH 08/15] x86/alternatives: Teach text_poke_bp() to emulate instructions

Quoted 58 lines · Peter Zijlstra
Quoted 19 lines · Peter Zijlstra
Quoted 2 lines · Andy Lutomirski

I am surely missing some kprobe context, but is it really safe to use this mechanism to replace more than one instruction?

I'm not entirely up-to-scratch here, so Masami, please correct me if I'm wrong.

So what happens is that arch_prepare_optimized_kprobe() <- copy_optimized_instructions() copies however much of the instruction stream is required such that we can overwrite the instruction at @addr with a 5 byte jump.

arch_optimize_kprobe() then does the text_poke_bp() that replaces the instruction @addr with int3, copies the rel jump address and overwrites the int3 with jmp.

And I'm thinking the problem is with something like:

Quoted 8 lines

@addr: nop nop nop nop nop

We copy out the nops into the trampoline, overwrite the first nop with an INT3, overwrite the remaining nops with the rel addr, but oops, another CPU can still be executing one of those NOPs, right?

I'm thinking we could fix this by first writing INT3 into all relevant instructions, which is going to be messy, given the current code base.

Maybe not that bad; how's something like this?

(completely untested)

---
 arch/x86/kernel/alternative.c | 20 ++++++++++++++++++--
 1 file changed, 18 insertions(+), 2 deletions(-)
arch/x86/kernel/alternative.c +2 -0
index 0d57015114e7..8f643dabea72 100644
@@ -24,6 +24,7 @@
 #include <asm/tlbflush.h>
 #include <asm/io.h>
 #include <asm/fixmap.h>
+#include <asm/insn.h>
 
 int __read_mostly alternatives_patched;
 
@@ -849,6 +850,7 @@ static void do_sync_core(void *info)
 
 static bool bp_patching_in_progress;
 static void *bp_int3_handler, *bp_int3_addr;
+static unsigned int bp_int3_length;
 
 int poke_int3_handler(struct pt_regs *regs)
 {

arch/x86/kernel/alternative.c int poke_int3_handler(struct pt_regs *regs) · line 877

Quoted 10 lines
arch/x86/kernel/alternative.c +5 -1
@@ -867,7 +869,11 @@ int poke_int3_handler(struct pt_regs *regs)
 	if (likely(!bp_patching_in_progress))
 		return 0;
 
-	if (user_mode(regs) || regs->ip != (unsigned long)bp_int3_addr)
+	if (user_mode(regs))
+		return 0;
+
+	if (regs->ip < (unsigned long)bp_int3_addr ||
+	    regs->ip >= (unsigned long)bp_int3_addr + bp_int3_length)
 		return 0;

Bugger, this isn't right. It'll jump to the beginning of the trampoline, even if it is multiple instructions in, which would lead to executing instructions twice, which would be BAD.

_maybe_, depending on what the slot looks like, we could do something like:

	offset = regs->ip - (unsigned long)bp_int3_addr;
	regs->ip = bp_int3_handler + offset;

That is; jump into the slot at the same offset we hit the INT3, but this is quickly getting yuck.

Quoted 30 lines
arch/x86/kernel/alternative.c +11 -1
 	/* set up the specified breakpoint handler */
@@ -900,9 +906,12 @@ NOKPROBE_SYMBOL(poke_int3_handler);
 void text_poke_bp(void *addr, const void *opcode, size_t len, void *handler)
 {
 	unsigned char int3 = 0xcc;
+	void *kaddr = addr;
+	struct insn insn;
 
 	bp_int3_handler = handler;
 	bp_int3_addr = (u8 *)addr + sizeof(int3);
+	bp_int3_length = len - sizeof(int3);
 	bp_patching_in_progress = true;
 
 	lockdep_assert_held(&text_mutex);
@@ -913,7 +922,14 @@ void text_poke_bp(void *addr, const void *opcode, size_t len, void *handler)
 	 */
 	smp_wmb();
 
-	text_poke(addr, &int3, sizeof(int3));
+	do {
+		kernel_insn_init(&insn, kaddr, MAX_INSN_SIZE);
+		insn_get_length(&insn);
+
+		text_poke(kaddr, &int3, sizeof(int3));
+
+		kaddr += insn.length;
+	} while (kaddr < addr + len);
 
 	on_each_cpu(do_sync_core, NULL, 1);
 
author maintainer
#34 lore raw

Re: [PATCH 08/15] x86/alternatives: Teach text_poke_bp() to emulate instructions

Quoted 4 lines · Peter Zijlstra

Bugger, this isn't right. It'll jump to the beginning of the trampoline, even if it is multiple instructions in, which would lead to executing instructions twice, which would be BAD.

Quoted 8 lines

_maybe_, depending on what the slot looks like, we could do something like:

	offset = regs->ip - (unsigned long)bp_int3_addr;
	regs->ip = bp_int3_handler + offset;

That is; jump into the slot at the same offset we hit the INT3, but this is quickly getting yuck.

Yeah, that won't work either... it needs something far more complex :/

maintainer
#35 lore raw

Re: [PATCH 08/15] x86/alternatives: Teach text_poke_bp() to emulate instructions

In reply to Peter Zijlstra
Quoted 5 lines · On Tue, 11 Jun 2019 10:03:07 +0200 Peter Zijlstra <[email protected]> wrote:

So what happens is that arch_prepare_optimized_kprobe() <- copy_optimized_instructions() copies however much of the instruction stream is required such that we can overwrite the instruction at @addr with a 5 byte jump.

Quoted 7 lines

arch_optimize_kprobe() then does the text_poke_bp() that replaces the instruction @addr with int3, copies the rel jump address and overwrites the int3 with jmp.

And I'm thinking the problem is with something like:

@addr: nop nop nop nop nop

What would work would be to:

	add breakpoint to first opcode.

	call synchronize_tasks();

	/* All tasks now hitting breakpoint and jumping over affected
	code */

	update the rest of the instructions.

	replace breakpoint with jmp.

One caveat is that the replaced instructions must not be a call function. As if the call function calls schedule then it will circumvent the synchronize_tasks(). It would be OK if that call is the last of the instructions. But I doubt we modify anything more then a call size anyway, so this should still work for all current instances.

-- Steve

Quoted 7 lines

We copy out the nops into the trampoline, overwrite the first nop with an INT3, overwrite the remaining nops with the rel addr, but oops, another CPU can still be executing one of those NOPs, right?

I'm thinking we could fix this by first writing INT3 into all relevant instructions, which is going to be messy, given the current code base.

maintainer
#36 lore raw

Re: [PATCH 08/15] x86/alternatives: Teach text_poke_bp() to emulate instructions

Quoted 5 lines · On Tue, 11 Jun 2019 11:22:54 -0400 Steven Rostedt <[email protected]> wrote:

What would work would be to:

	add breakpoint to first opcode.

	call synchronize_tasks();

BTW, that should be "synchronize_rcu_tasks()"

-- Steve

Quoted 7 lines
	/* All tasks now hitting breakpoint and jumping over affected
	code */

	update the rest of the instructions.

	replace breakpoint with jmp.
author maintainer
#37 lore raw

Re: [PATCH 08/15] x86/alternatives: Teach text_poke_bp() to emulate instructions

In reply to Steven Rostedt
Quoted 30 lines · Steven Rostedt
Quoted 5 lines · On Tue, 11 Jun 2019 10:03:07 +0200 Peter Zijlstra <[email protected]> wrote:

So what happens is that arch_prepare_optimized_kprobe() <- copy_optimized_instructions() copies however much of the instruction stream is required such that we can overwrite the instruction at @addr with a 5 byte jump.

Quoted 7 lines

arch_optimize_kprobe() then does the text_poke_bp() that replaces the instruction @addr with int3, copies the rel jump address and overwrites the int3 with jmp.

And I'm thinking the problem is with something like:

@addr: nop nop nop nop nop

What would work would be to:

	add breakpoint to first opcode.

	call synchronize_tasks();

	/* All tasks now hitting breakpoint and jumping over affected
	code */

	update the rest of the instructions.

	replace breakpoint with jmp.
Quoted 5 lines

One caveat is that the replaced instructions must not be a call function. As if the call function calls schedule then it will circumvent the synchronize_tasks(). It would be OK if that call is the last of the instructions. But I doubt we modify anything more then a call size anyway, so this should still work for all current instances.

Right, something like this could work (although I cannot currently find synchronize_tasks), but it would make the optprobe stuff fairly slow (iirc this sync_tasks() thing could be pretty horrible).

#38 lore raw

Re: [PATCH 08/15] x86/alternatives: Teach text_poke_bp() to emulate instructions

Quoted 39 lines · Peter Zijlstra
Quoted 30 lines · Steven Rostedt
Quoted 5 lines · On Tue, 11 Jun 2019 10:03:07 +0200 Peter Zijlstra <[email protected]> wrote:

So what happens is that arch_prepare_optimized_kprobe() <- copy_optimized_instructions() copies however much of the instruction stream is required such that we can overwrite the instruction at @addr with a 5 byte jump.

Quoted 7 lines

arch_optimize_kprobe() then does the text_poke_bp() that replaces the instruction @addr with int3, copies the rel jump address and overwrites the int3 with jmp.

And I'm thinking the problem is with something like:

@addr: nop nop nop nop nop

What would work would be to:

	add breakpoint to first opcode.

	call synchronize_tasks();

	/* All tasks now hitting breakpoint and jumping over affected
	code */

	update the rest of the instructions.

	replace breakpoint with jmp.
Quoted 5 lines

One caveat is that the replaced instructions must not be a call function. As if the call function calls schedule then it will circumvent the synchronize_tasks(). It would be OK if that call is the last of the instructions. But I doubt we modify anything more then a call size anyway, so this should still work for all current instances.

Quoted 3 lines

Right, something like this could work (although I cannot currently find synchronize_tasks), but it would make the optprobe stuff fairly slow (iirc this sync_tasks() thing could be pretty horrible).

I have run into similar problems before.

I had two problematic scenarios. In the first case, I had a “call” in the middle of the patched code-block, but this call was always followed by a “jump” to the end of the potentially patched code-block, so I did not have the problem.

In the second case, I had an indirect call (which is shorter than a direct call) being patched into a direct call. In this case, I preceded the indirect call with NOPs so indeed the indirect call was at the end of the patched block.

In certain cases, if a shorter instruction should be potentially patched into a longer one, the shorter one can be preceded by some prefixes. If there are multiple REX prefixes, for instance, the CPU only uses the last one, IIRC. This can allow to avoid synchronize_sched() when patching a single instruction into another instruction with a different length.

Not sure how helpful this information is, but sharing - just in case.

author maintainer
#39 lore raw

Re: [PATCH 08/15] x86/alternatives: Teach text_poke_bp() to emulate instructions

Quoted 8 lines · Nadav Amit

I have run into similar problems before.

I had two problematic scenarios. In the first case, I had a “call” in the middle of the patched code-block, but this call was always followed by a “jump” to the end of the potentially patched code-block, so I did not have the problem.

In the second case, I had an indirect call (which is shorter than a direct

Longer, 6 bytes vs 5 if I'm not mistaken.

Quoted 11 lines

call) being patched into a direct call. In this case, I preceded the indirect call with NOPs so indeed the indirect call was at the end of the patched block.

In certain cases, if a shorter instruction should be potentially patched into a longer one, the shorter one can be preceded by some prefixes. If there are multiple REX prefixes, for instance, the CPU only uses the last one, IIRC. This can allow to avoid synchronize_sched() when patching a single instruction into another instruction with a different length.

Not sure how helpful this information is, but sharing - just in case.

I think we can patch multiple instructions provided:

- all but one instruction are a NOP,
- there are no branch targets inside the range.

By poking INT3 at every instruction in the range and then doing the machine wide IPI+SYNC, we'll trap every CPU that is in-side the range.

Because all but one instruction are a NOP, we can emulate only the one instruction (assuming the real instruction is always last), otherwise NOP when we're behind the real instruction.

Then we can write new instructions, leaving the initial INT3 until last.

Something like this might be useful if we want to support immediate instructions (like patch_data_* in paravirt_patch.c) for static_call().

#40 lore raw

Re: [PATCH 08/15] x86/alternatives: Teach text_poke_bp() to emulate instructions

Quoted 6 lines · Peter Zijlstra
Quoted 2 lines · Nadav Amit

I have run into similar problems before.

Quoted 8 lines
Quoted 6 lines

I had two problematic scenarios. In the first case, I had a “call” in the middle of the patched code-block, but this call was always followed by a “jump” to the end of the potentially patched code-block, so I did not have the problem.

In the second case, I had an indirect call (which is shorter than a direct

Longer, 6 bytes vs 5 if I'm not mistaken.

Shorter (2-3 bytes IIRC), since the target was held in a register.

Quoted 21 lines
Quoted 5 lines

call) being patched into a direct call. In this case, I preceded the indirect call with NOPs so indeed the indirect call was at the end of the patched block.

Quoted 7 lines

In certain cases, if a shorter instruction should be potentially patched into a longer one, the shorter one can be preceded by some prefixes. If there are multiple REX prefixes, for instance, the CPU only uses the last one, IIRC. This can allow to avoid synchronize_sched() when patching a single instruction into another instruction with a different length.

Not sure how helpful this information is, but sharing - just in case.

I think we can patch multiple instructions provided:

- all but one instruction are a NOP,
- there are no branch targets inside the range.

By poking INT3 at every instruction in the range and then doing the machine wide IPI+SYNC, we'll trap every CPU that is in-side the range.

Quoted 8 lines

Because all but one instruction are a NOP, we can emulate only the one instruction (assuming the real instruction is always last), otherwise NOP when we're behind the real instruction.

Then we can write new instructions, leaving the initial INT3 until last.

Something like this might be useful if we want to support immediate instructions (like patch_data_* in paravirt_patch.c) for static_call().

I don’t know what you regard when you say SYNC, but if you regard something like sync_core() (in contrast to something like synchronize_sched() ), I am not sure it is sufficient.

Using IPI+sync_core(), I think, would make an assumption that IRQs are never enabled inside IRQ and exception handlers, or that these handlers would not be invoked while the patched code is executed. Otherwise, the IPI might be received inside the IRQ/exception handler, and then return from the handler will be into the middle of a patched instruction.

maintainer
#41 lore raw

Re: [PATCH 08/15] x86/alternatives: Teach text_poke_bp() to emulate instructions

In reply to Peter Zijlstra
Quoted 26 lines · Peter Zijlstra
Quoted 8 lines · Nadav Amit

I have run into similar problems before.

I had two problematic scenarios. In the first case, I had a “call” in the middle of the patched code-block, but this call was always followed by a “jump” to the end of the potentially patched code-block, so I did not have the problem.

In the second case, I had an indirect call (which is shorter than a direct

Longer, 6 bytes vs 5 if I'm not mistaken.

Quoted 11 lines

call) being patched into a direct call. In this case, I preceded the indirect call with NOPs so indeed the indirect call was at the end of the patched block.

In certain cases, if a shorter instruction should be potentially patched into a longer one, the shorter one can be preceded by some prefixes. If there are multiple REX prefixes, for instance, the CPU only uses the last one, IIRC. This can allow to avoid synchronize_sched() when patching a single instruction into another instruction with a different length.

Not sure how helpful this information is, but sharing - just in case.

Quoted 7 lines

I think we can patch multiple instructions provided:

- all but one instruction are a NOP,
- there are no branch targets inside the range.

By poking INT3 at every instruction in the range and then doing the machine wide IPI+SYNC, we'll trap every CPU that is in-side the range.

How do you know you'll trap them? You need to IPI, serialize, and get them to execute an instruction. If the CPU is in an interrupt and RIP just happens to be pointed to the INT3, you need them to execute a whole lot more than just one instruction.

author maintainer
#42 lore raw

Re: [PATCH 08/15] x86/alternatives: Teach text_poke_bp() to emulate instructions

Quoted 32 lines · Andy Lutomirski
Quoted 31 lines · Peter Zijlstra
Quoted 8 lines · Nadav Amit

I have run into similar problems before.

I had two problematic scenarios. In the first case, I had a “call” in the middle of the patched code-block, but this call was always followed by a “jump” to the end of the potentially patched code-block, so I did not have the problem.

In the second case, I had an indirect call (which is shorter than a direct

Longer, 6 bytes vs 5 if I'm not mistaken.

Quoted 11 lines

call) being patched into a direct call. In this case, I preceded the indirect call with NOPs so indeed the indirect call was at the end of the patched block.

In certain cases, if a shorter instruction should be potentially patched into a longer one, the shorter one can be preceded by some prefixes. If there are multiple REX prefixes, for instance, the CPU only uses the last one, IIRC. This can allow to avoid synchronize_sched() when patching a single instruction into another instruction with a different length.

Not sure how helpful this information is, but sharing - just in case.

I think we can patch multiple instructions provided:

- all but one instruction are a NOP,
- there are no branch targets inside the range.

Quoted 7 lines
Quoted 2 lines

By poking INT3 at every instruction in the range and then doing the machine wide IPI+SYNC, we'll trap every CPU that is in-side the range.

How do you know you'll trap them? You need to IPI, serialize, and get them to execute an instruction. If the CPU is in an interrupt and RIP just happens to be pointed to the INT3, you need them to execute a whole lot more than just one instruction.

Argh, yes, I'm an idiot.

#43 lore raw

Re: [PATCH 08/15] x86/alternatives: Teach text_poke_bp() to emulate instructions

In reply to Peter Zijlstra
Quoted 42 lines · Peter Zijlstra
Quoted 17 lines · Andy Lutomirski
Quoted 15 lines · Peter Zijlstra
Quoted 8 lines · Masami Hiramatsu
Quoted 3 lines

This fits almost all text_poke_bp() users, except arch_unoptimize_kprobe() which restores random text, and for that site we have to build an explicit emulate instruction.

Hm, actually it doesn't restores randome text, since the first byte must always be int3. As the function name means, it just unoptimizes (jump based optprobe -> int3 based kprobe).
Anyway, that is not an issue. With this patch, optprobe must still work.

Quoted 7 lines
Quoted 4 lines

I thought it basically restored 5 bytes of original text (with no guarantee it is a single instruction, or even a complete instruction), with the first byte replaced with INT3.

I am surely missing some kprobe context, but is it really safe to use this mechanism to replace more than one instruction?

I'm not entirely up-to-scratch here, so Masami, please correct me if I'm wrong.

So what happens is that arch_prepare_optimized_kprobe() <- copy_optimized_instructions() copies however much of the instruction stream is required such that we can overwrite the instruction at @addr with a 5 byte jump.

arch_optimize_kprobe() then does the text_poke_bp() that replaces the instruction @addr with int3, copies the rel jump address and overwrites the int3 with jmp.

And I'm thinking the problem is with something like:

Quoted 8 lines

@addr: nop nop nop nop nop

We copy out the nops into the trampoline, overwrite the first nop with an INT3, overwrite the remaining nops with the rel addr, but oops, another CPU can still be executing one of those NOPs, right?

I'm thinking we could fix this by first writing INT3 into all relevant instructions, which is going to be messy, given the current code base.

How does that help? If RIP == x+2 and you want to put a 5-byte jump at address x, no amount of 0xcc is going to change the fact that RIP is in the middle of the jump.

Live patching can handle this by detecting this condition on each CPU, but performance won’t be great. Maybe some synchronize_sched trickery could help.

maintainer
#44 lore raw

Re: [PATCH 08/15] x86/alternatives: Teach text_poke_bp() to emulate instructions

Quoted 7 lines · On Tue, 11 Jun 2019 08:54:23 -0700 Andy Lutomirski <[email protected]> wrote:

How does that help? If RIP == x+2 and you want to put a 5-byte jump at address x, no amount of 0xcc is going to change the fact that RIP is in the middle of the jump.

Live patching can handle this by detecting this condition on each CPU, but performance won’t be great. Maybe some synchronize_sched trickery could help.

We have synchronize_rcu_tasks() which return after all tasks have either entered user space or did a voluntary schedule (was not preempted). Or have not run (still in a sleeping state).

That way we guarantee that all tasks are no longer on any trampoline or code paths that do not call schedule. I use this to free dynamically allocated trampolines used by ftrace. And kprobes uses this too for its own trampolines.

-- Steve

author maintainer
#45 lore raw

Re: [PATCH 08/15] x86/alternatives: Teach text_poke_bp() to emulate instructions

In reply to Andy Lutomirski
Quoted 14 lines · Andy Lutomirski
Quoted 14 lines · Peter Zijlstra

arch_optimize_kprobe() then does the text_poke_bp() that replaces the instruction @addr with int3, copies the rel jump address and overwrites the int3 with jmp.

And I'm thinking the problem is with something like:

@addr: nop nop nop nop nop

We copy out the nops into the trampoline, overwrite the first nop with an INT3, overwrite the remaining nops with the rel addr, but oops, another CPU can still be executing one of those NOPs, right?

Quoted 6 lines
Quoted 2 lines

I'm thinking we could fix this by first writing INT3 into all relevant instructions, which is going to be messy, given the current code base.

How does that help? If RIP == x+2 and you want to put a 5-byte jump at address x, no amount of 0xcc is going to change the fact that RIP is in the middle of the jump.

What I proposed was doing 0xCC on every instruction that's inside of [x,x+5). After that we do machine wide IPI+SYNC.

So if RIP is at x+2, then it will observe 0xCC and trigger the exception there.

Now, the problem is that my exception cannot recover from anything except NOPs, so while it could work for some corner cases, it doesn't work for the optkprobe case in general.

Only then do we write the JMP offset and again to IPI+SYNC; then we write the 0xE8 and again IPI-SYNC.

But at that point we already have the guarantee nobody is inside [x,x+5). That is, except if we can get to x+2 without first going through x, IOW if x+2 is a branch target we're screwed any which way around and the poke is never valid.

Is that something optkprobes checks? If so, where and how?

author maintainer
#46 lore raw

Re: [PATCH 08/15] x86/alternatives: Teach text_poke_bp() to emulate instructions

In reply to Peter Zijlstra
Quoted 25 lines · Peter Zijlstra
Quoted 4 lines · Masami Hiramatsu
Quoted 4 lines
arch/x86/kernel/alternative.c +0 -0
@@ -943,8 +949,21 @@ int poke_int3_handler(struct pt_regs *re
 	if (user_mode(regs) || regs->ip != (unsigned long)bp_int3_addr)
 		return 0;
 
Quoted 16 lines
Quoted 4 lines
arch/x86/kernel/alternative.c +2 -2
-	/* set up the specified breakpoint handler */
-	regs->ip = (unsigned long) bp_int3_handler;
+	opcode = *(struct opcode *)bp_int3_opcode;
+

arch/x86/kernel/alternative.c int poke_int3_handler(struct pt_regs *re · line 963

Quoted 10 lines
arch/x86/kernel/alternative.c +10 -0
+	switch (opcode.insn) {
+	case 0xE8: /* CALL */
+		int3_emulate_call(regs, ip + opcode.rel);
+		break;
+
+	case 0xE9: /* JMP */
+		int3_emulate_jmp(regs, ip + opcode.rel);
+		break;
+
+	default: /* assume NOP */

Shouldn't we check whether it is actually NOP here?

I was/am lazy and didn't want to deal with:

Quoted 7 lines

arch/x86/include/asm/nops.h:#define GENERIC_NOP5_ATOMIC NOP_DS_PREFIX,GENERIC_NOP4 arch/x86/include/asm/nops.h:#define K8_NOP5_ATOMIC 0x66,K8_NOP4 arch/x86/include/asm/nops.h:#define K7_NOP5_ATOMIC NOP_DS_PREFIX,K7_NOP4 arch/x86/include/asm/nops.h:#define P6_NOP5_ATOMIC P6_NOP5

But maybe we should check for all the various NOP5 variants and BUG() on anything unexpected.

I realized we never actually poke a !ideal nop5_atomic, so I've added that to the latest versions.

Reviewed-by on v1 8/15
#47 lore raw

Re: [PATCH 08/15] x86/alternatives: Teach text_poke_bp() to emulate instructions

In reply to Peter Zijlstra
Quoted 10 lines · Peter Zijlstra

In preparation for static_call support, teach text_poke_bp() to emulate instructions, including CALL.

The current text_poke_bp() takes a @handler argument which is used as a jump target when the temporary INT3 is hit by a different CPU.

When patching CALL instructions, this doesn't work because we'd miss the PUSH of the return address. Instead, teach poke_int3_handler() to emulate an instruction, typically the instruction we're patching in.

Quoted 7 lines

This fits almost all text_poke_bp() users, except arch_unoptimize_kprobe() which restores random text, and for that site we have to build an explicit emulate instruction.

Cc Daniel Bristot de Oliveira <[email protected]>
Cc Nadav Amit <[email protected]>
Signature · 1 line
Josh
maintainer
#48 lore raw

Re: [PATCH 08/15] x86/alternatives: Teach text_poke_bp() to emulate instructions

In reply to Peter Zijlstra
Quoted 7 lines · On Wed, 05 Jun 2019 15:08:01 +0200 Peter Zijlstra <[email protected]> wrote:
arch/x86/kernel/alternative.c +2 -2
-void text_poke_bp(void *addr, const void *opcode, size_t len, void *handler)
+void text_poke_bp(void *addr, const void *opcode, size_t len, const void *emulate)
 {
 	unsigned char int3 = 0xcc;
 
-	bp_int3_handler = handler;
+	bp_int3_opcode = emulate ?: opcode;

arch/x86/kernel/alternative.c NOKPROBE_SYMBOL(poke_int3_handler); · line 1006

Quoted 10 lines
arch/x86/kernel/alternative.c +4 -0
 	bp_int3_addr = (u8 *)addr + sizeof(int3);
 	bp_patching_in_progress = true;
 
 	lockdep_assert_held(&text_mutex);
 
 	/*
+	 * poke_int3_handler() relies on @opcode being a 5 byte instruction;
+	 * notably a JMP, CALL or NOP5_ATOMIC.
+	 */
+	BUG_ON(len != 5);

If we have a bug on here, why bother with passing in len at all? Just force it to be 5.

We could make it a WARN_ON() and return without doing anything.

This also prevents us from ever changing two byte jmps.

-- Steve

Quoted 9 lines
arch/x86/kernel/alternative.c +3 -2
+
+	/*
 	 * Corresponding read barrier in int3 notifier for making sure the
-	 * in_progress and handler are correctly ordered wrt. patching.
+	 * in_progress and opcode are correctly ordered wrt. patching.
 	 */
 	smp_wmb();
 
-
author maintainer
#49 lore raw

Re: [PATCH 08/15] x86/alternatives: Teach text_poke_bp() to emulate instructions

Quoted 15 lines · Steven Rostedt
Quoted 12 lines · On Wed, 05 Jun 2019 15:08:01 +0200 Peter Zijlstra <[email protected]> wrote:
arch/x86/kernel/alternative.c +2 -2
-void text_poke_bp(void *addr, const void *opcode, size_t len, void *handler)
+void text_poke_bp(void *addr, const void *opcode, size_t len, const void *emulate)
 {
 	unsigned char int3 = 0xcc;
 
-	bp_int3_handler = handler;
+	bp_int3_opcode = emulate ?: opcode;
 	bp_int3_addr = (u8 *)addr + sizeof(int3);
 	bp_patching_in_progress = true;
 
 	lockdep_assert_held(&text_mutex);
Quoted 8 lines

arch/x86/kernel/alternative.c NOKPROBE_SYMBOL(poke_int3_handler); · line 1006

Quoted 5 lines
arch/x86/kernel/alternative.c +4 -0
 	/*
+	 * poke_int3_handler() relies on @opcode being a 5 byte instruction;
+	 * notably a JMP, CALL or NOP5_ATOMIC.
+	 */
+	BUG_ON(len != 5);

If we have a bug on here, why bother with passing in len at all? Just force it to be 5.

Masami said the same.

Quoted 3 lines

We could make it a WARN_ON() and return without doing anything.

This also prevents us from ever changing two byte jmps.

It doesn't; that is, we'd need to add emulation for the 3 byte jump, but that'd be pretty trivial.

author maintainer
#50 lore raw

Re: [PATCH 08/15] x86/alternatives: Teach text_poke_bp() to emulate instructions

Quoted 25 lines · Peter Zijlstra
Quoted 15 lines · Steven Rostedt
Quoted 12 lines · On Wed, 05 Jun 2019 15:08:01 +0200 Peter Zijlstra <[email protected]> wrote:
(quoted diff) +2 -2
-void text_poke_bp(void *addr, const void *opcode, size_t len, void *handler)
+void text_poke_bp(void *addr, const void *opcode, size_t len, const void *emulate)
 {
 	unsigned char int3 = 0xcc;
 
-	bp_int3_handler = handler;
+	bp_int3_opcode = emulate ?: opcode;
 	bp_int3_addr = (u8 *)addr + sizeof(int3);
 	bp_patching_in_progress = true;
 
 	lockdep_assert_held(&text_mutex);
Quoted 8 lines

Review

Quoted 5 lines
(quoted diff) +4 -0
 	/*
+	 * poke_int3_handler() relies on @opcode being a 5 byte instruction;
+	 * notably a JMP, CALL or NOP5_ATOMIC.
+	 */
+	BUG_ON(len != 5);

If we have a bug on here, why bother with passing in len at all? Just force it to be 5.

Quoted 8 lines

Masami said the same.

Quoted 3 lines

We could make it a WARN_ON() and return without doing anything.

This also prevents us from ever changing two byte jmps.

It doesn't; that is, we'd need to add emulation for the 3 byte jump, but that'd be pretty trivial.

I can't find a 3 byte jump on x86_64, I could only find a 2 byte one. But something like so should work I suppose, although at this point I'm thinking we should just used the instruction decode we have instead of playing iffy games with packed structures.

arch/x86/include/asm/text-patching.h +3 -0
index e1a4bb42eb92..abb9615dcb1d 100644
@@ -57,6 +57,9 @@ static inline void int3_emulate_jmp(struct pt_regs *regs, unsigned long ip)
 #define JMP_INSN_SIZE		5
 #define JMP_INSN_OPCODE		0xE9
 
+#define JMP8_INSN_SIZE		2
+#define JMP8_INSN_OPCODE	0xEB
+
 static inline void int3_emulate_push(struct pt_regs *regs, unsigned long val)
 {
 	/*
arch/x86/kernel/alternative.c +38 -12

83 lines not on this page · raw message

author maintainer
#51 lore raw

Re: [PATCH 08/15] x86/alternatives: Teach text_poke_bp() to emulate instructions

Quoted 3 lines · Peter Zijlstra

although at this point I'm
thinking we should just used the instruction decode we have instead of playing iffy games with packed structures.

How's something like this? It accepts jmp/32 jmp/8 call and nop5_atomic.

---
Subject: x86/alternatives: Teach text_poke_bp() to emulate instructions
From: Peter Zijlstra <[email protected]>
Date: Wed Jun 5 10:48:37 CEST 2019

In preparation for static_call support, teach text_poke_bp() to emulate instructions, including CALL.

The current text_poke_bp() takes a @handler argument which is used as a jump target when the temporary INT3 is hit by a different CPU.

When patching CALL instructions, this doesn't work because we'd miss the PUSH of the return address. Instead, teach poke_int3_handler() to emulate an instruction, typically the instruction we're patching in.

This fits almost all text_poke_bp() users, except arch_unoptimize_kprobe() which restores random text, and for that site we have to build an explicit emulate instruction.

---
 arch/x86/include/asm/text-patching.h |   15 +++++--
 arch/x86/kernel/alternative.c        |   73 +++++++++++++++++++++++++----------
 arch/x86/kernel/jump_label.c         |    3 -
 arch/x86/kernel/kprobes/opt.c        |   11 +++--
 4 files changed, 75 insertions(+), 27 deletions(-)
arch/x86/include/asm/text-patching.h +12 -3
@@ -37,7 +37,7 @@ extern void text_poke_early(void *addr,
 extern void *text_poke(void *addr, const void *opcode, size_t len);
 extern void *text_poke_kgdb(void *addr, const void *opcode, size_t len);
 extern int poke_int3_handler(struct pt_regs *regs);
-extern void text_poke_bp(void *addr, const void *opcode, size_t len, void *handler);
+extern void text_poke_bp(void *addr, const void *opcode, size_t len, const void *emulate);
 extern int after_bootmem;
 extern __ro_after_init struct mm_struct *poking_mm;
 extern __ro_after_init unsigned long poking_addr;
@@ -48,8 +48,17 @@ static inline void int3_emulate_jmp(stru
 	regs->ip = ip;
 }
 
-#define INT3_INSN_SIZE 1
-#define CALL_INSN_SIZE 5
+#define INT3_INSN_SIZE		1
+#define INT3_INSN_OPCODE	0xCC
+
+#define CALL_INSN_SIZE		5
+#define CALL_INSN_OPCODE	0xE8
+
+#define JMP_INSN_SIZE		5
+#define JMP_INSN_OPCODE		0xE9
+
+#define JMP8_INSN_SIZE		2
+#define JMP8_INSN_OPCODE	0xEB
 
 static inline void int3_emulate_push(struct pt_regs *regs, unsigned long val)
 {
arch/x86/kernel/alternative.c +54 -19

117 lines not on this page · raw message

arch/x86/kernel/jump_label.c +1 -2
@@ -87,8 +87,7 @@ static void __ref __jump_label_transform
 		return;
 	}
 
-	text_poke_bp((void *)jump_entry_code(entry), code, JUMP_LABEL_NOP_SIZE,
-		     (void *)jump_entry_code(entry) + JUMP_LABEL_NOP_SIZE);
+	text_poke_bp((void *)jump_entry_code(entry), code, JUMP_LABEL_NOP_SIZE, NULL);
 }
 
 void arch_jump_label_transform(struct jump_entry *entry,
arch/x86/kernel/kprobes/opt.c +8 -3
@@ -437,8 +437,7 @@ void arch_optimize_kprobes(struct list_h
 		insn_buff[0] = RELATIVEJUMP_OPCODE;
 		*(s32 *)(&insn_buff[1]) = rel;
 
-		text_poke_bp(op->kp.addr, insn_buff, RELATIVEJUMP_SIZE,
-			     op->optinsn.insn);
+		text_poke_bp(op->kp.addr, insn_buff, RELATIVEJUMP_SIZE, NULL);
 
 		list_del_init(&op->list);
 	}
@@ -448,12 +447,18 @@ void arch_optimize_kprobes(struct list_h
 void arch_unoptimize_kprobe(struct optimized_kprobe *op)
 {
 	u8 insn_buff[RELATIVEJUMP_SIZE];
+	u8 emulate_buff[RELATIVEJUMP_SIZE];
 
 	/* Set int3 to first byte for kprobes */
 	insn_buff[0] = BREAKPOINT_INSTRUCTION;
 	memcpy(insn_buff + 1, op->optinsn.copied_insn, RELATIVE_ADDR_SIZE);
+
+	emulate_buff[0] = RELATIVEJUMP_OPCODE;
+	*(s32 *)(&emulate_buff[1]) = (s32)((long)op->optinsn.insn -
+			((long)op->kp.addr + RELATIVEJUMP_SIZE));
+
 	text_poke_bp(op->kp.addr, insn_buff, RELATIVEJUMP_SIZE,
-		     op->optinsn.insn);
+		     emulate_buff);
 }
 
 /*
author maintainer
#52 lore raw

PATCH 9/15 compiler.h: Make __ADDRESSABLE() symbol truly unique

In reply to Peter Zijlstra

From: Josh Poimboeuf <[email protected]>

The __ADDRESSABLE() macro uses the __LINE__ macro to create a temporary symbol which has a unique name. However, if the macro is used multiple times from within another macro, the line number will always be the same, resulting in duplicate symbols.

Make the temporary symbols truly unique by using __UNIQUE_ID instead of __LINE__.

Cc Julia Cartwright <[email protected]>
Cc Steven Rostedt <[email protected]>
Cc Ingo Molnar <[email protected]>
Cc Daniel Bristot de Oliveira <[email protected]>
Cc Rasmus Villemoes <[email protected]>
Cc Jason Baron <[email protected]>
Cc Edward Cree <[email protected]>
Cc Jiri Kosina <[email protected]>
Cc Linus Torvalds <[email protected]>
Cc Borislav Petkov <[email protected]>
Cc Masami Hiramatsu <[email protected]>
Cc Thomas Gleixner <[email protected]>
Cc Jessica Yu <[email protected]>
Cc David Laight <[email protected]>
Cc Nadav Amit <[email protected]>
Cc "H. Peter Anvin" <[email protected]>
Cc Andy Lutomirski <[email protected]>
---
 include/linux/compiler.h |    2 +-
 1 file changed, 1 insertion(+), 1 deletion(-)
include/linux/compiler.h +1 -1
@@ -294,7 +294,7 @@ unsigned long read_word_at_a_time(const
  */
 #define __ADDRESSABLE(sym) \
 	static void * __section(".discard.addressable") __used \
-		__PASTE(__addressable_##sym, __LINE__) = (void *)&sym;
+		__UNIQUE_ID(__addressable_##sym) = (void *)&sym;
 
 /**
  * offset_to_ptr - convert a relative memory offset to an absolute pointer
author maintainer
#53 lore raw

PATCH 10/15 static_call: Add basic static call infrastructure

In reply to Peter Zijlstra

From: Josh Poimboeuf <[email protected]>

Static calls are a replacement for global function pointers. They use code patching to allow direct calls to be used instead of indirect calls. They give the flexibility of function pointers, but with improved performance. This is especially important for cases where retpolines would otherwise be used, as retpolines can significantly impact performance.

The concept and code are an extension of previous work done by Ard Biesheuvel and Steven Rostedt:

https://lkml.kernel.org/r/[email protected]
https://lkml.kernel.org/r/[email protected]

There are two implementations, depending on arch support:

1) out-of-line: patched trampolines (CONFIG_HAVE_STATIC_CALL)
2) basic function pointers

For more details, see the comments in include/linux/static_call.h.

Cc Steven Rostedt <[email protected]>
Cc Julia Cartwright <[email protected]>
Cc Ingo Molnar <[email protected]>
Cc Ard Biesheuvel <[email protected]>
Cc Jason Baron <[email protected]>
Cc Rasmus Villemoes <[email protected]>
Cc Daniel Bristot de Oliveira <[email protected]>
Cc Linus Torvalds <[email protected]>
Cc Jiri Kosina <[email protected]>
Cc Edward Cree <[email protected]>
Cc Thomas Gleixner <[email protected]>
Cc Masami Hiramatsu <[email protected]>
Cc Borislav Petkov <[email protected]>
Cc David Laight <[email protected]>
Cc Jessica Yu <[email protected]>
Cc Nadav Amit <[email protected]>
Cc Andy Lutomirski <[email protected]>
Cc "H. Peter Anvin" <[email protected]>
---
 arch/Kconfig                      |    3 
 include/linux/static_call.h       |  135 ++++++++++++++++++++++++++++++++++++++
 include/linux/static_call_types.h |   13 +++
 3 files changed, 151 insertions(+)
 create mode 100644 include/linux/static_call.h
 create mode 100644 include/linux/static_call_types.h
arch/Kconfig +3 -0
@@ -927,6 +927,9 @@ config LOCK_EVENT_COUNTS
 	  the chance of application behavior change because of timing
 	  differences. The counts are reported via debugfs.
 
+config HAVE_STATIC_CALL
+	bool
+
 source "kernel/gcov/Kconfig"
 
 source "scripts/gcc-plugins/Kconfig"
include/linux/static_call.h new file +135 -0

135 lines not on this page · raw message

include/linux/static_call_types.h new file +13 -0
@@ -0,0 +1,13 @@
+/* SPDX-License-Identifier: GPL-2.0 */
+#ifndef _STATIC_CALL_TYPES_H
+#define _STATIC_CALL_TYPES_H
+
+#include <linux/stringify.h>
+
+#define STATIC_CALL_TRAMP_PREFIX ____static_call_tramp_
+#define STATIC_CALL_TRAMP_PREFIX_STR __stringify(STATIC_CALL_TRAMP_PREFIX)
+
+#define STATIC_CALL_TRAMP(key) __PASTE(STATIC_CALL_TRAMP_PREFIX, key)
+#define STATIC_CALL_TRAMP_STR(key) __stringify(STATIC_CALL_TRAMP(key))
+
+#endif /* _STATIC_CALL_TYPES_H */
#54 lore raw

Re: [PATCH 10/15] static_call: Add basic static call infrastructure

Quoted 98 lines · Peter Zijlstra

From: Josh Poimboeuf <[email protected]>

Static calls are a replacement for global function pointers. They use code patching to allow direct calls to be used instead of indirect calls. They give the flexibility of function pointers, but with improved performance. This is especially important for cases where retpolines would otherwise be used, as retpolines can significantly impact performance.

The concept and code are an extension of previous work done by Ard Biesheuvel and Steven Rostedt:

https://nam04.safelinks.protection.outlook.com/?url=https%3A%2F%2Flkml.kernel.org%2Fr%2F20181005081333.15018-1-ard.biesheuvel%40linaro.org&amp;data=02%7C01%7Cnamit%40vmware.com%7C3f2ebbeff15e444d2fa008d6e9b9023f%7Cb39138ca3cee4b4aa4d6cd83d9dd62f0%7C0%7C0%7C636953378182765229&amp;sdata=WHceTWVt%2BNu1RFBv8jHp2Tw7VZuI5HxvHt%2FrWnjAmm4%3D&amp;reserved=0
https://nam04.safelinks.protection.outlook.com/?url=https%3A%2F%2Flkml.kernel.org%2Fr%2F20181006015110.653946300%40goodmis.org&amp;data=02%7C01%7Cnamit%40vmware.com%7C3f2ebbeff15e444d2fa008d6e9b9023f%7Cb39138ca3cee4b4aa4d6cd83d9dd62f0%7C0%7C0%7C636953378182765229&amp;sdata=12JcrUsOh7%2FjRwEV9ANHw5SA2A6D4qNJ6z3h5aMHpnE%3D&amp;reserved=0

There are two implementations, depending on arch support:

1) out-of-line: patched trampolines (CONFIG_HAVE_STATIC_CALL)
2) basic function pointers

For more details, see the comments in include/linux/static_call.h.

---

arch/Kconfig                      |    3 
include/linux/static_call.h       |  135 ++++++++++++++++++++++++++++++++++++++
include/linux/static_call_types.h |   13 +++
3 files changed, 151 insertions(+)

create mode 100644 include/linux/static_call.h create mode 100644 include/linux/static_call_types.h

arch/Kconfig +3 -0
@@ -927,6 +927,9 @@ config LOCK_EVENT_COUNTS
 	  the chance of application behavior change because of timing
 	  differences. The counts are reported via debugfs.
 
+config HAVE_STATIC_CALL
+	bool
+

source "kernel/gcov/Kconfig"

source "scripts/gcc-plugins/Kconfig"

include/linux/static_call.h new file +29 -0
@@ -0,0 +1,135 @@
+/* SPDX-License-Identifier: GPL-2.0 */
+#ifndef _LINUX_STATIC_CALL_H
+#define _LINUX_STATIC_CALL_H
+
+/*
+ * Static call support
+ *
+ * Static calls use code patching to hard-code function pointers into direct
+ * branch instructions.  They give the flexibility of function pointers, but
+ * with improved performance.  This is especially important for cases where
+ * retpolines would otherwise be used, as retpolines can significantly impact
+ * performance.
+ *
+ *
+ * API overview:
+ *
+ *   DECLARE_STATIC_CALL(key, func);
+ *   DEFINE_STATIC_CALL(key, func);
+ *   static_call(key, args...);
+ *   static_call_update(key, func);
+ *
+ *
+ * Usage example:
+ *
+ *   # Start with the following functions (with identical prototypes):
+ *   int func_a(int arg1, int arg2);
+ *   int func_b(int arg1, int arg2);
+ *
+ *   # Define a 'my_key' reference, associated with func_a() by default

include/linux/static_call.h line 39

Quoted 10 lines
include/linux/static_call.h new file +10 -0
+ *   DEFINE_STATIC_CALL(my_key, func_a);
+ *
+ *   # Call func_a()
+ *   static_call(my_key, arg1, arg2);
+ *
+ *   # Update 'my_key' to point to func_b()
+ *   static_call_update(my_key, func_b);
+ *
+ *   # Call func_b()
+ *   static_call(my_key, arg1, arg2);

I think that this calling interface is not very intuitive. I understand that the macros/objtool cannot allow the calling interface to be completely transparent (as compiler plugin could). But, can the macros be used to paste the key with the “static_call”? I think that having something like:

static_call__func(arg1, arg2)

Is more readable than

static_call(func, arg1, arg2)

include/linux/static_call.h line 95

Quoted 7 lines
include/linux/static_call.h new file +7 -0
+}
+
+#define static_call_update(key, func)					\
+({									\
+	BUILD_BUG_ON(!__same_type(func, STATIC_CALL_TRAMP(key)));	\
+	__static_call_update(&key, func);				\
+})

Is this safe against concurrent module removal?

author maintainer
#55 lore raw

Re: [PATCH 10/15] static_call: Add basic static call infrastructure

Quoted 18 lines · Nadav Amit
Quoted 5 lines

+ * Usage example:
+ *

+ *   # Start with the following functions (with identical prototypes):
+ *   int func_a(int arg1, int arg2);
+ *   int func_b(int arg1, int arg2);
Quoted 12 lines

+ *
+ * # Define a 'my_key' reference, associated with func_a() by default
+ * DEFINE_STATIC_CALL(my_key, func_a);
+ *
+ * # Call func_a()
+ * static_call(my_key, arg1, arg2);
+ *
+ * # Update 'my_key' to point to func_b()
+ * static_call_update(my_key, func_b);
+ *
+ * # Call func_b()
+ * static_call(my_key, arg1, arg2);

Quoted 1 line

I think that this calling interface is not very intuitive.

Yeah, it is somewhat unfortunate..

Quoted 10 lines

I understand that
the macros/objtool cannot allow the calling interface to be completely transparent (as compiler plugin could). But, can the macros be used to paste the key with the “static_call”? I think that having something like:

static_call__func(arg1, arg2)

Is more readable than

static_call(func, arg1, arg2)

Doesn't really make it much better for me; I think I'd prefer to switch to the GCC plugin scheme over this. ISTR there being some propotypes there, but I couldn't quickly locate them.

Quoted 9 lines

include/linux/static_call.h line 95

Quoted 7 lines
include/linux/static_call.h new file +7 -0
+}
+
+#define static_call_update(key, func)					\
+({									\
+	BUILD_BUG_ON(!__same_type(func, STATIC_CALL_TRAMP(key)));	\
+	__static_call_update(&key, func);				\
+})

Is this safe against concurrent module removal?

It is for CONFIG_MODULE=n :-)

#56 lore raw

Re: [PATCH 10/15] static_call: Add basic static call infrastructure

Quoted 20 lines · Peter Zijlstra
Quoted 18 lines · Nadav Amit
Quoted 17 lines

+ * Usage example:
+ *

+ *   # Start with the following functions (with identical prototypes):
+ *   int func_a(int arg1, int arg2);
+ *   int func_b(int arg1, int arg2);

+ *
+ * # Define a 'my_key' reference, associated with func_a() by default
+ * DEFINE_STATIC_CALL(my_key, func_a);
+ *
+ * # Call func_a()
+ * static_call(my_key, arg1, arg2);
+ *
+ * # Update 'my_key' to point to func_b()
+ * static_call_update(my_key, func_b);
+ *
+ * # Call func_b()
+ * static_call(my_key, arg1, arg2);

Quoted 4 lines
Quoted 1 line

I think that this calling interface is not very intuitive.

Yeah, it is somewhat unfortunate..

Another thing I brought up at the time is that it would be useful to have the ability to 'reset' a static call to its default target. E.g., for crypto modules that implement an accelerated version of a library interface, removing the module should revert those call sites back to the original target, without putting a disproportionate burden on the module itself to implement the logic to support this.

Quoted 7 lines
Quoted 7 lines

I understand that
the macros/objtool cannot allow the calling interface to be completely transparent (as compiler plugin could). But, can the macros be used to paste the key with the “static_call”? I think that having something like:

static_call__func(arg1, arg2)

Quoted 8 lines
Quoted 3 lines

Is more readable than

static_call(func, arg1, arg2)

Doesn't really make it much better for me; I think I'd prefer to switch to the GCC plugin scheme over this. ISTR there being some propotypes there, but I couldn't quickly locate them.

I implemented the GCC plugin here

https://git.kernel.org/pub/scm/linux/kernel/git/ardb/linux.git/log/?h=static-calls

but IIRC, all it does is annotate call sites exactly how objtool does it.

Quoted 11 lines
Quoted 9 lines

include/linux/static_call.h line 95

Quoted 7 lines
include/linux/static_call.h new file +7 -0
+}
+
+#define static_call_update(key, func)                                      \
+({                                                                 \
+   BUILD_BUG_ON(!__same_type(func, STATIC_CALL_TRAMP(key)));       \
+   __static_call_update(&key, func);                               \
+})

Is this safe against concurrent module removal?

It is for CONFIG_MODULE=n :-)

#57 lore raw

Re: [PATCH 10/15] static_call: Add basic static call infrastructure

Quoted 30 lines · Ard Biesheuvel
Quoted 22 lines · Peter Zijlstra
Quoted 18 lines · Nadav Amit
Quoted 17 lines

+ * Usage example:
+ *

+ *   # Start with the following functions (with identical prototypes):
+ *   int func_a(int arg1, int arg2);
+ *   int func_b(int arg1, int arg2);

+ *
+ * # Define a 'my_key' reference, associated with func_a() by default
+ * DEFINE_STATIC_CALL(my_key, func_a);
+ *
+ * # Call func_a()
+ * static_call(my_key, arg1, arg2);
+ *
+ * # Update 'my_key' to point to func_b()
+ * static_call_update(my_key, func_b);
+ *
+ * # Call func_b()
+ * static_call(my_key, arg1, arg2);

Quoted 4 lines
Quoted 1 line

I think that this calling interface is not very intuitive.

Yeah, it is somewhat unfortunate..

Quoted 6 lines

Another thing I brought up at the time is that it would be useful to have the ability to 'reset' a static call to its default target. E.g., for crypto modules that implement an accelerated version of a library interface, removing the module should revert those call sites back to the original target, without putting a disproportionate burden on the module itself to implement the logic to support this.

I was thinking this could be a layer on top. We could have a way to register a static call with the module core so that, when a GPL module with an appropriate symbol is loaded, the static call gets replaced.

KVM could use this too. Or we could just require KVM to be built in some day.

#58 lore raw

Re: [PATCH 10/15] static_call: Add basic static call infrastructure

In reply to Ard Biesheuvel
Quoted 49 lines · Ard Biesheuvel
Quoted 19 lines · Peter Zijlstra
Quoted 18 lines · Nadav Amit
Quoted 17 lines

+ * Usage example:
+ *

+ *   # Start with the following functions (with identical prototypes):
+ *   int func_a(int arg1, int arg2);
+ *   int func_b(int arg1, int arg2);

+ *
+ * # Define a 'my_key' reference, associated with func_a() by default
+ * DEFINE_STATIC_CALL(my_key, func_a);
+ *
+ * # Call func_a()
+ * static_call(my_key, arg1, arg2);
+ *
+ * # Update 'my_key' to point to func_b()
+ * static_call_update(my_key, func_b);
+ *
+ * # Call func_b()
+ * static_call(my_key, arg1, arg2);

Quoted 3 lines
Quoted 1 line

I think that this calling interface is not very intuitive.

Yeah, it is somewhat unfortunate..

Another thing I brought up at the time is that it would be useful to have the ability to 'reset' a static call to its default target. E.g., for crypto modules that implement an accelerated version of a library interface, removing the module should revert those call sites back to the original target, without putting a disproportionate burden on the module itself to implement the logic to support this.

Quoted 7 lines
Quoted 7 lines

I understand that
the macros/objtool cannot allow the calling interface to be completely transparent (as compiler plugin could). But, can the macros be used to paste the key with the “static_call”? I think that having something like:

static_call__func(arg1, arg2)

Quoted 7 lines
Quoted 3 lines

Is more readable than

static_call(func, arg1, arg2)

Doesn't really make it much better for me; I think I'd prefer to switch to the GCC plugin scheme over this. ISTR there being some propotypes there, but I couldn't quickly locate them.

Quoted 5 lines

I implemented the GCC plugin here

https://nam04.safelinks.protection.outlook.com/?url=https%3A%2F%2Fgit.kernel.org%2Fpub%2Fscm%2Flinux%2Fkernel%2Fgit%2Fardb%2Flinux.git%2Flog%2F%3Fh%3Dstatic-calls&amp;data=02%7C01%7Cnamit%40vmware.com%7Cd31c4713640c44a651bf08d6eb250faa%7Cb39138ca3cee4b4aa4d6cd83d9dd62f0%7C0%7C0%7C636954941771964758&amp;sdata=h7RtT33E9FMapLZbAu9aTfjREP5kXrM0o2QQ1WpbDCM%3D&amp;reserved=0

but IIRC, all it does is annotate call sites exactly how objtool does it.

I did not see your version before I made mine for a similar (but slightly different) purpose:

https://lore.kernel.org/lkml/[email protected]/

My version, I think, is more generic (I don’t think yours consider calls that have a return value). Anyhow, I am sure you know more about GCC plugins than I do.

I do have a version that can take annotations to say which call should be static and to get the symbol it uses.

I also think that this implementation would disallow keys that reside within structs. This would mean that paravirt, for instance, would need to go through many changes to use this infrastructure.

author maintainer
#59 lore raw

Re: [PATCH 10/15] static_call: Add basic static call infrastructure

In reply to Peter Zijlstra
Quoted 30 lines · Peter Zijlstra
Quoted 18 lines · Nadav Amit
Quoted 5 lines

+ * Usage example:
+ *

+ *   # Start with the following functions (with identical prototypes):
+ *   int func_a(int arg1, int arg2);
+ *   int func_b(int arg1, int arg2);
Quoted 12 lines

+ *
+ * # Define a 'my_key' reference, associated with func_a() by default
+ * DEFINE_STATIC_CALL(my_key, func_a);
+ *
+ * # Call func_a()
+ * static_call(my_key, arg1, arg2);
+ *
+ * # Update 'my_key' to point to func_b()
+ * static_call_update(my_key, func_b);
+ *
+ * # Call func_b()
+ * static_call(my_key, arg1, arg2);

Quoted 1 line

I think that this calling interface is not very intuitive.

Yeah, it is somewhat unfortunate..

Quoted 7 lines

I understand that
the macros/objtool cannot allow the calling interface to be completely transparent (as compiler plugin could). But, can the macros be used to paste the key with the “static_call”? I think that having something like:

static_call__func(arg1, arg2)

Quoted 7 lines
Quoted 3 lines

Is more readable than

static_call(func, arg1, arg2)

Doesn't really make it much better for me; I think I'd prefer to switch to the GCC plugin scheme over this. ISTR there being some propotypes there, but I couldn't quickly locate them.

How about something like:

	static_call(key)(arg1, arg2);

which is very close to the regular indirect call syntax. Furthermore, how about we put the trampolines in .static_call.text instead of relying on prefixes?

Also, I think I can shrink static_call_key by half:

- we can do away with static_call_key::tramp; there are only two usage sites:

o __static_call_update, the static_call() macro can provide the address of STATIC_CALL_TRAMP(key) directly

o static_call_add_module(), has two cases:

* the trampoline is from outside the module; in this case it will already have been updated by a previous call to __static_call_update.
* the trampoline is from inside the module; in this case it will have the default value and it doesn't need an update.

so in no case does static_call_add_module() need to modify a trampoline.

- we can change static_call_key::site_mods into a single next pointer, just like jump_label's static_key.

But so far all the schemes I've come up with require 'key' to be a name, it cannot be an actual 'struct static_call_key *' value. And therefore usage from within structures isn't allowed.

#60 lore raw

Re: [PATCH 10/15] static_call: Add basic static call infrastructure

Quoted 39 lines · Peter Zijlstra
Quoted 30 lines · Peter Zijlstra
Quoted 18 lines · Nadav Amit
Quoted 5 lines

+ * Usage example:
+ *

+ *   # Start with the following functions (with identical prototypes):
+ *   int func_a(int arg1, int arg2);
+ *   int func_b(int arg1, int arg2);
Quoted 12 lines

+ *
+ * # Define a 'my_key' reference, associated with func_a() by default
+ * DEFINE_STATIC_CALL(my_key, func_a);
+ *
+ * # Call func_a()
+ * static_call(my_key, arg1, arg2);
+ *
+ * # Update 'my_key' to point to func_b()
+ * static_call_update(my_key, func_b);
+ *
+ * # Call func_b()
+ * static_call(my_key, arg1, arg2);

Quoted 1 line

I think that this calling interface is not very intuitive.

Yeah, it is somewhat unfortunate..

Quoted 7 lines

I understand that
the macros/objtool cannot allow the calling interface to be completely transparent (as compiler plugin could). But, can the macros be used to paste the key with the “static_call”? I think that having something like:

static_call__func(arg1, arg2)

Quoted 7 lines
Quoted 3 lines

Is more readable than

static_call(func, arg1, arg2)

Doesn't really make it much better for me; I think I'd prefer to switch to the GCC plugin scheme over this. ISTR there being some propotypes there, but I couldn't quickly locate them.

Quoted 5 lines

How about something like:

	static_call(key)(arg1, arg2);

which is very close to the regular indirect call syntax.

Looks ok to me.

Quoted 2 lines

Furthermore, how about we put the trampolines in .static_call.text instead of relying on prefixes?

Yeah, that would probably be better.

Quoted 17 lines

Also, I think I can shrink static_call_key by half:

- we can do away with static_call_key::tramp; there are only two usage sites:

o __static_call_update, the static_call() macro can provide the address of STATIC_CALL_TRAMP(key) directly

o static_call_add_module(), has two cases:

* the trampoline is from outside the module; in this case it will already have been updated by a previous call to __static_call_update.
* the trampoline is from inside the module; in this case it will have the default value and it doesn't need an update.

Quoted 2 lines

so in no case does static_call_add_module() need to modify a trampoline.

Sounds plausible.

Quoted 2 lines

- we can change static_call_key::site_mods into a single next pointer, just like jump_label's static_key.

Yep.

Quoted 3 lines

But so far all the schemes I've come up with require 'key' to be a name, it cannot be an actual 'struct static_call_key *' value. And therefore usage from within structures isn't allowed.

Is that something we need? At least we were able to work around this limitation with tracepoints' usage of static calls. But I could see how it could be useful.

One way to solve that would be a completely different implementation:
have a global trampoline which detects the call site of the caller, associates it with the given key, schedules some work to patch the call site later, and then jumps to key->func. So the first call would trigger the patching.

Then we might not even need objtool :-) But it might be tricky to pull off.

Signature · 1 line
Josh
author maintainer
#61 lore raw

PATCH 11/15 static_call: Add inline static call infrastructure

In reply to Peter Zijlstra

From: Josh Poimboeuf <[email protected]>

Add infrastructure for an arch-specific CONFIG_HAVE_STATIC_CALL_INLINE option, which is a faster version of CONFIG_HAVE_STATIC_CALL. At runtime, the static call sites are patched directly, rather than using the out-of-line trampolines.

Compared to out-of-line static calls, the performance benefits are more modest, but still measurable. Steven Rostedt did some tracepoint measurements:

https://lkml.kernel.org/r/[email protected]

This code is heavily inspired by the jump label code (aka "static jumps"), as some of the concepts are very similar.

For more details, see the comments in include/linux/static_call.h.

Cc Steven Rostedt <[email protected]>
Cc Julia Cartwright <[email protected]>
Cc Ingo Molnar <[email protected]>
Cc Ard Biesheuvel <[email protected]>
Cc Jason Baron <[email protected]>
Cc Rasmus Villemoes <[email protected]>
Cc Daniel Bristot de Oliveira <[email protected]>
Cc Linus Torvalds <[email protected]>
Cc Jiri Kosina <[email protected]>
Cc Edward Cree <[email protected]>
Cc Thomas Gleixner <[email protected]>
Cc Masami Hiramatsu <[email protected]>
Cc Borislav Petkov <[email protected]>
Cc David Laight <[email protected]>
Cc Jessica Yu <[email protected]>
Cc Nadav Amit <[email protected]>
Cc Andy Lutomirski <[email protected]>
Cc "H. Peter Anvin" <[email protected]>
---
 arch/Kconfig                      |    4 
 include/asm-generic/vmlinux.lds.h |    7 
 include/linux/module.h            |   10 +
 include/linux/static_call.h       |   63 +++++++
 include/linux/static_call_types.h |    9 +
 kernel/Makefile                   |    1 
 kernel/module.c                   |    5 
 kernel/static_call.c              |  316 ++++++++++++++++++++++++++++++++++++++
 8 files changed, 414 insertions(+), 1 deletion(-)
 create mode 100644 kernel/static_call.c
arch/Kconfig +4 -0
@@ -930,6 +930,10 @@ config LOCK_EVENT_COUNTS
 config HAVE_STATIC_CALL
 	bool
 
+config HAVE_STATIC_CALL_INLINE
+	bool
+	depends on HAVE_STATIC_CALL
+
 source "kernel/gcov/Kconfig"
 
 source "scripts/gcc-plugins/Kconfig"
include/asm-generic/vmlinux.lds.h +7 -0
@@ -311,6 +311,12 @@
 	KEEP(*(__jump_table))						\
 	__stop___jump_table = .;
 
+#define STATIC_CALL_DATA						\
+	. = ALIGN(8);							\
+	__start_static_call_sites = .;					\
+	KEEP(*(.static_call_sites))					\
+	__stop_static_call_sites = .;
+
 /*
  * Allow architectures to handle ro_after_init data on their
  * own by defining an empty RO_AFTER_INIT_DATA.
@@ -320,6 +326,7 @@
 	__start_ro_after_init = .;					\
 	*(.data..ro_after_init)						\
 	JUMP_TABLE_DATA							\
+	STATIC_CALL_DATA						\
 	__end_ro_after_init = .;
 #endif
 
include/linux/module.h +10 -0
@@ -21,6 +21,7 @@
 #include <linux/rbtree_latch.h>
 #include <linux/error-injection.h>
 #include <linux/tracepoint-defs.h>
+#include <linux/static_call_types.h>
 
 #include <linux/percpu.h>
 #include <asm/module.h>
@@ -472,6 +473,10 @@ struct module {
 	unsigned int num_ftrace_callsites;
 	unsigned long *ftrace_callsites;
 #endif
+#ifdef CONFIG_HAVE_STATIC_CALL_INLINE
+	int num_static_call_sites;
+	struct static_call_site *static_call_sites;
+#endif
 
 #ifdef CONFIG_LIVEPATCH
 	bool klp; /* Is this a livepatch module? */
@@ -728,6 +733,11 @@ static inline bool within_module(unsigne
 {
 	return false;
 }
+
+static inline bool within_module_init(unsigned long addr, const struct module *mod)
+{
+	return false;
+}
 
 /* Get/put a kernel symbol (calls should be symmetric) */
 #define symbol_get(x) ({ extern typeof(x) x __attribute__((weak)); &(x); })
include/linux/static_call.h +62 -1

75 lines not on this page · raw message

include/linux/static_call_types.h +9 -0
@@ -10,4 +10,13 @@
 #define STATIC_CALL_TRAMP(key) __PASTE(STATIC_CALL_TRAMP_PREFIX, key)
 #define STATIC_CALL_TRAMP_STR(key) __stringify(STATIC_CALL_TRAMP(key))
 
+/*
+ * The static call site table needs to be created by external tooling (objtool
+ * or a compiler plugin).
+ */
+struct static_call_site {
+	s32 addr;
+	s32 key;
+};
+
 #endif /* _STATIC_CALL_TYPES_H */
kernel/Makefile +1 -0
@@ -100,6 +100,7 @@ obj-$(CONFIG_TRACEPOINTS) += trace/
 obj-$(CONFIG_IRQ_WORK) += irq_work.o
 obj-$(CONFIG_CPU_PM) += cpu_pm.o
 obj-$(CONFIG_BPF) += bpf/
+obj-$(CONFIG_HAVE_STATIC_CALL_INLINE) += static_call.o
 
 obj-$(CONFIG_PERF_EVENTS) += events/
 
kernel/module.c +5 -0
@@ -3117,6 +3117,11 @@ static int find_module_sections(struct m
 					    sizeof(*mod->ei_funcs),
 					    &mod->num_ei_funcs);
 #endif
+#ifdef CONFIG_HAVE_STATIC_CALL_INLINE
+	mod->static_call_sites = section_objs(info, ".static_call_sites",
+					      sizeof(*mod->static_call_sites),
+					      &mod->num_static_call_sites);
+#endif
 	mod->extable = section_objs(info, "__ex_table",
 				    sizeof(*mod->extable), &mod->num_exentries);
 
kernel/static_call.c new file +316 -0

316 lines not on this page · raw message

#62 lore raw

Re: [PATCH 11/15] static_call: Add inline static call infrastructure

Quoted 12 lines · Peter Zijlstra

From: Josh Poimboeuf <[email protected]>

Add infrastructure for an arch-specific CONFIG_HAVE_STATIC_CALL_INLINE option, which is a faster version of CONFIG_HAVE_STATIC_CALL. At runtime, the static call sites are patched directly, rather than using the out-of-line trampolines.

Compared to out-of-line static calls, the performance benefits are more modest, but still measurable. Steven Rostedt did some tracepoint measurements:

[ snip ]

Quoted 13 lines
kernel/static_call.c new file +13 -0
+static void static_call_del_module(struct module *mod)
+{
+	struct static_call_site *start = mod->static_call_sites;
+	struct static_call_site *stop = mod->static_call_sites +
+					mod->num_static_call_sites;
+	struct static_call_site *site;
+	struct static_call_key *key, *prev_key = NULL;
+	struct static_call_mod *site_mod;
+
+	for (site = start; site < stop; site++) {
+		key = static_call_key(site);
+		if (key == prev_key)
+			continue;

kernel/static_call.c line 227

Quoted 10 lines
kernel/static_call.c new file +10 -0
+		prev_key = key;
+
+		list_for_each_entry(site_mod, &key->site_mods, list) {
+			if (site_mod->mod == mod) {
+				list_del(&site_mod->list);
+				kfree(site_mod);
+				break;
+			}
+		}
+	}

I think that for safety, when a module is removed, all the static-calls should be traversed to check that none of them calls any function in the removed module. If that happens, perhaps it should be poisoned.

Quoted 6 lines
kernel/static_call.c new file +6 -0
+}
+
+static int static_call_module_notify(struct notifier_block *nb,
+				     unsigned long val, void *data)
+{
+	struct module *mod = data;

kernel/static_call.c line 243

Quoted 10 lines
kernel/static_call.c new file +10 -0
+	int ret = 0;
+
+	cpus_read_lock();
+	static_call_lock();
+
+	switch (val) {
+	case MODULE_STATE_COMING:
+		module_disable_ro(mod);
+		ret = static_call_add_module(mod);
+		module_enable_ro(mod, false);

Doesn’t it cause some pages to be W+X ? Can it be avoided?

kernel/static_call.c line 246

Quoted 3 lines
kernel/static_call.c new file +3 -0
+		if (ret) {
+			WARN(1, "Failed to allocate memory for static calls");
+			static_call_del_module(mod);

If static_call_add_module() succeeded in changing some of the calls, but not all, I don’t think that static_call_del_module() will correctly undo static_call_add_module(). The code transformations, I think, will remain.

author maintainer
#63 lore raw

Re: [PATCH 11/15] static_call: Add inline static call infrastructure

Quoted 19 lines · Nadav Amit
Quoted 19 lines
kernel/static_call.c new file +19 -0
+static void static_call_del_module(struct module *mod)
+{
+	struct static_call_site *start = mod->static_call_sites;
+	struct static_call_site *stop = mod->static_call_sites +
+					mod->num_static_call_sites;
+	struct static_call_site *site;
+	struct static_call_key *key, *prev_key = NULL;
+	struct static_call_mod *site_mod;
+
+	for (site = start; site < stop; site++) {
+		key = static_call_key(site);
+		if (key == prev_key)
+			continue;
+		prev_key = key;
+
+		list_for_each_entry(site_mod, &key->site_mods, list) {
+			if (site_mod->mod == mod) {
+				list_del(&site_mod->list);
+				kfree(site_mod);
Quoted 8 lines

kernel/static_call.c line 227

Quoted 4 lines
kernel/static_call.c new file +4 -0
+				break;
+			}
+		}
+	}

I think that for safety, when a module is removed, all the static-calls should be traversed to check that none of them calls any function in the removed module. If that happens, perhaps it should be poisoned.

We don't do that for normal indirect calls either.. I suppose we could here, but meh.

Quoted 10 lines
Quoted 10 lines
kernel/static_call.c new file +10 -0
+}
+
+static int static_call_module_notify(struct notifier_block *nb,
+				     unsigned long val, void *data)
+{
+	struct module *mod = data;
+	int ret = 0;
+
+	cpus_read_lock();
+	static_call_lock();
Quoted 8 lines

kernel/static_call.c line 243

Quoted 6 lines
kernel/static_call.c new file +6 -0
+
+	switch (val) {
+	case MODULE_STATE_COMING:
+		module_disable_ro(mod);
+		ret = static_call_add_module(mod);
+		module_enable_ro(mod, false);

Doesn’t it cause some pages to be W+X ? Can it be avoided?

I don't know why it does this, jump_labels doesn't seem to need this, and I'm not seeing what static_call needs differently.

Quoted 7 lines

kernel/static_call.c line 246

Quoted 3 lines
kernel/static_call.c new file +3 -0
+		if (ret) {
+			WARN(1, "Failed to allocate memory for static calls");
+			static_call_del_module(mod);

If static_call_add_module() succeeded in changing some of the calls, but not all, I don’t think that static_call_del_module() will correctly undo static_call_add_module(). The code transformations, I think, will remain.

Hurm, jump_labels has the same problem.

I wonder why kernel/module.c:prepare_coming_module() doesn't propagate the error from the notifier call. If it were to do that, I think we'll abort the module load and any modifications get lost anyway.

#64 lore raw

Re: [PATCH 11/15] static_call: Add inline static call infrastructure

Quoted 65 lines · Peter Zijlstra
Quoted 19 lines · Nadav Amit
Quoted 19 lines
kernel/static_call.c new file +19 -0
+static void static_call_del_module(struct module *mod)
+{
+	struct static_call_site *start = mod->static_call_sites;
+	struct static_call_site *stop = mod->static_call_sites +
+					mod->num_static_call_sites;
+	struct static_call_site *site;
+	struct static_call_key *key, *prev_key = NULL;
+	struct static_call_mod *site_mod;
+
+	for (site = start; site < stop; site++) {
+		key = static_call_key(site);
+		if (key == prev_key)
+			continue;
+		prev_key = key;
+
+		list_for_each_entry(site_mod, &key->site_mods, list) {
+			if (site_mod->mod == mod) {
+				list_del(&site_mod->list);
+				kfree(site_mod);
Quoted 8 lines

kernel/static_call.c line 227

Quoted 4 lines
kernel/static_call.c new file +4 -0
+				break;
+			}
+		}
+	}

I think that for safety, when a module is removed, all the static-calls should be traversed to check that none of them calls any function in the removed module. If that happens, perhaps it should be poisoned.

We don't do that for normal indirect calls either.. I suppose we could here, but meh.

Quoted 10 lines
Quoted 10 lines
kernel/static_call.c new file +10 -0
+}
+
+static int static_call_module_notify(struct notifier_block *nb,
+				     unsigned long val, void *data)
+{
+	struct module *mod = data;
+	int ret = 0;
+
+	cpus_read_lock();
+	static_call_lock();
Quoted 8 lines

kernel/static_call.c line 243

Quoted 6 lines
kernel/static_call.c new file +6 -0
+
+	switch (val) {
+	case MODULE_STATE_COMING:
+		module_disable_ro(mod);
+		ret = static_call_add_module(mod);
+		module_enable_ro(mod, false);

Doesn’t it cause some pages to be W+X ? Can it be avoided?

I don't know why it does this, jump_labels doesn't seem to need this, and I'm not seeing what static_call needs differently.

Quoted 7 lines

kernel/static_call.c line 246

Quoted 3 lines
kernel/static_call.c new file +3 -0
+		if (ret) {
+			WARN(1, "Failed to allocate memory for static calls");
+			static_call_del_module(mod);

If static_call_add_module() succeeded in changing some of the calls, but not all, I don’t think that static_call_del_module() will correctly undo static_call_add_module(). The code transformations, I think, will remain.

Quoted 5 lines

Hurm, jump_labels has the same problem.

I wonder why kernel/module.c:prepare_coming_module() doesn't propagate the error from the notifier call. If it were to do that, I think we'll abort the module load and any modifications get lost anyway.

This might be a security problem, since it can leave indirect branches, which are susceptible to Spectre v2, in the code.

author maintainer
#65 lore raw

Re: [PATCH 11/15] static_call: Add inline static call infrastructure

Quoted 11 lines · Nadav Amit
Quoted 11 lines · Peter Zijlstra
Quoted 7 lines · Nadav Amit

kernel/static_call.c line 246

Quoted 3 lines
kernel/static_call.c new file +3 -0
+		if (ret) {
+			WARN(1, "Failed to allocate memory for static calls");
+			static_call_del_module(mod);

If static_call_add_module() succeeded in changing some of the calls, but not all, I don’t think that static_call_del_module() will correctly undo static_call_add_module(). The code transformations, I think, will remain.

Quoted 8 lines
Quoted 5 lines

Hurm, jump_labels has the same problem.

I wonder why kernel/module.c:prepare_coming_module() doesn't propagate the error from the notifier call. If it were to do that, I think we'll abort the module load and any modifications get lost anyway.

This might be a security problem, since it can leave indirect branches, which are susceptible to Spectre v2, in the code.

It's a correctness problem too; for both jump_label and static_call, since if we don't patch the call site, we also don't patch the trampoline and who knows what random code it ends up running.

I'll go stare at the module code once my migrane goes again :/

#66 lore raw

Re: [PATCH 11/15] static_call: Add inline static call infrastructure

In reply to Peter Zijlstra
Quoted 10 lines · Peter Zijlstra
Quoted 10 lines
Quoted 10 lines
kernel/static_call.c new file +10 -0
+}
+
+static int static_call_module_notify(struct notifier_block *nb,
+				     unsigned long val, void *data)
+{
+	struct module *mod = data;
+	int ret = 0;
+
+	cpus_read_lock();
+	static_call_lock();
Quoted 8 lines
Quoted 8 lines

kernel/static_call.c line 243

Quoted 6 lines
kernel/static_call.c new file +6 -0
+
+	switch (val) {
+	case MODULE_STATE_COMING:
+		module_disable_ro(mod);
+		ret = static_call_add_module(mod);
+		module_enable_ro(mod, false);

Doesn’t it cause some pages to be W+X ?

How so?

Quoted 4 lines
Quoted 1 line

Can it be avoided?

I don't know why it does this, jump_labels doesn't seem to need this, and I'm not seeing what static_call needs differently.

I forgot why I did this, but it's probably for the case where there's a static call site in module init code. It deserves a comment.

Theoretically, jump labels need this to.

BTW, there's a change coming that will require the text_mutex before calling module_{disable,enable}_ro().

Signature · 1 line
Josh
#67 lore raw

Re: [PATCH 11/15] static_call: Add inline static call infrastructure

Quoted 29 lines · Josh Poimboeuf
Quoted 10 lines · Peter Zijlstra
Quoted 10 lines
Quoted 10 lines
kernel/static_call.c new file +10 -0
+}
+
+static int static_call_module_notify(struct notifier_block *nb,
+				     unsigned long val, void *data)
+{
+	struct module *mod = data;
+	int ret = 0;
+
+	cpus_read_lock();
+	static_call_lock();
Quoted 8 lines
Quoted 8 lines

kernel/static_call.c line 243

Quoted 6 lines
kernel/static_call.c new file +6 -0
+
+	switch (val) {
+	case MODULE_STATE_COMING:
+		module_disable_ro(mod);
+		ret = static_call_add_module(mod);
+		module_enable_ro(mod, false);

Doesn’t it cause some pages to be W+X ?

How so?

Quoted 4 lines
Quoted 1 line

Can it be avoided?

I don't know why it does this, jump_labels doesn't seem to need this, and I'm not seeing what static_call needs differently.

Quoted 7 lines

I forgot why I did this, but it's probably for the case where there's a static call site in module init code. It deserves a comment.

Theoretically, jump labels need this to.

BTW, there's a change coming that will require the text_mutex before calling module_{disable,enable}_ro().

I think that eventually, the most secure flow is for the module executable to be write-protected immediately after the module signature is checked and then use text_poke() to change the code without removing the write-protection in such manner.

Ideally, these pieces of code (module signature check and static-key/call mechanisms) would somehow be isolated.

I wonder whether static-calls in init-code cannot just be avoided. They would most likely introduce more overhead in patching than they would save in execution time.

#68 lore raw

Re: [PATCH 11/15] static_call: Add inline static call infrastructure

Quoted 42 lines · Nadav Amit
Quoted 29 lines · Josh Poimboeuf
Quoted 10 lines · Peter Zijlstra
Quoted 10 lines
Quoted 10 lines
kernel/static_call.c new file +10 -0
+}
+
+static int static_call_module_notify(struct notifier_block *nb,
+				     unsigned long val, void *data)
+{
+	struct module *mod = data;
+	int ret = 0;
+
+	cpus_read_lock();
+	static_call_lock();
Quoted 8 lines
Quoted 8 lines

kernel/static_call.c line 243

Quoted 6 lines
kernel/static_call.c new file +6 -0
+
+	switch (val) {
+	case MODULE_STATE_COMING:
+		module_disable_ro(mod);
+		ret = static_call_add_module(mod);
+		module_enable_ro(mod, false);

Doesn’t it cause some pages to be W+X ?

How so?

Quoted 4 lines
Quoted 1 line

Can it be avoided?

I don't know why it does this, jump_labels doesn't seem to need this, and I'm not seeing what static_call needs differently.

Quoted 7 lines

I forgot why I did this, but it's probably for the case where there's a static call site in module init code. It deserves a comment.

Theoretically, jump labels need this to.

BTW, there's a change coming that will require the text_mutex before calling module_{disable,enable}_ro().

I think that eventually, the most secure flow is for the module executable to be write-protected immediately after the module signature is checked and then use text_poke() to change the code without removing the write-protection in such manner.

Quoted 6 lines

Ideally, these pieces of code (module signature check and static-key/call mechanisms) would somehow be isolated.

I wonder whether static-calls in init-code cannot just be avoided. They would most likely introduce more overhead in patching than they would save in execution time.

It's a valid question. Are any tracepoints called from module init? Or -- thinking ahead -- are there any pv calls from module init? That might be plausible.

Signature · 1 line
Josh
author maintainer
#69 lore raw

Re: [PATCH 11/15] static_call: Add inline static call infrastructure

In reply to Josh Poimboeuf
Quoted 20 lines · Josh Poimboeuf
Quoted 10 lines · Peter Zijlstra
Quoted 10 lines
Quoted 10 lines
kernel/static_call.c new file +10 -0
+}
+
+static int static_call_module_notify(struct notifier_block *nb,
+				     unsigned long val, void *data)
+{
+	struct module *mod = data;
+	int ret = 0;
+
+	cpus_read_lock();
+	static_call_lock();
Quoted 8 lines
Quoted 8 lines

kernel/static_call.c line 243

Quoted 6 lines
kernel/static_call.c new file +6 -0
+
+	switch (val) {
+	case MODULE_STATE_COMING:
+		module_disable_ro(mod);
+		ret = static_call_add_module(mod);
+		module_enable_ro(mod, false);

Doesn’t it cause some pages to be W+X ?

Quoted 1 line

How so?

This is after complete_formation() which does RO,X. If we then disable RO we end up with W+X pages, which is bad.

That said, alternatives, ftrace, dynamic_debug all run before complete_formation() specifically such that they can directly poke text.

Possibly we should add a notifier callback for MODULE_STATE_UNFORMED, but that is for another day.

Quoted 5 lines
Quoted 4 lines
Quoted 1 line

Can it be avoided?

I don't know why it does this, jump_labels doesn't seem to need this, and I'm not seeing what static_call needs differently.

Quoted 7 lines

I forgot why I did this, but it's probably for the case where there's a static call site in module init code. It deserves a comment.

Theoretically, jump labels need this to.

BTW, there's a change coming that will require the text_mutex before calling module_{disable,enable}_ro().

I can't find why it would need this (and I'm going to remove it). Specifically complete_formation() does enable_ro(.after_init=false), which leaves .ro_after_init writable so {jump_label,static_call}_sort_entries() will work.

But both jump_label and static_call then use the full text_poke(), not text_poke_early(), for modules.

author maintainer
#70 lore raw

PATCH 12/15 x86/static_call: Add out-of-line static call implementation

In reply to Peter Zijlstra

From: Josh Poimboeuf <[email protected]>

Add the x86 out-of-line static call implementation. For each key, a permanent trampoline is created which is the destination for all static calls for the given key. The trampoline has a direct jump which gets patched by static_call_update() when the destination function changes.

Cc Steven Rostedt <[email protected]>
Cc Julia Cartwright <[email protected]>
Cc Ingo Molnar <[email protected]>
Cc Ard Biesheuvel <[email protected]>
Cc Jason Baron <[email protected]>
Cc Linus Torvalds <[email protected]>
Cc Jiri Kosina <[email protected]>
Cc Thomas Gleixner <[email protected]>
Cc Masami Hiramatsu <[email protected]>
Cc Borislav Petkov <[email protected]>
Cc David Laight <[email protected]>
Cc Jessica Yu <[email protected]>
Cc Andy Lutomirski <[email protected]>
Cc "H. Peter Anvin" <[email protected]>
---
 arch/x86/Kconfig                   |    1 
 arch/x86/include/asm/static_call.h |   28 +++++++++++++++++++++++++++
 arch/x86/kernel/Makefile           |    1 
 arch/x86/kernel/static_call.c      |   38 +++++++++++++++++++++++++++++++++++++
 4 files changed, 68 insertions(+)
 create mode 100644 arch/x86/include/asm/static_call.h
 create mode 100644 arch/x86/kernel/static_call.c
arch/x86/Kconfig +1 -0
@@ -198,6 +198,7 @@ config X86
 	select HAVE_FUNCTION_ARG_ACCESS_API
 	select HAVE_STACKPROTECTOR		if CC_HAS_SANE_STACKPROTECTOR
 	select HAVE_STACK_VALIDATION		if X86_64
+	select HAVE_STATIC_CALL
 	select HAVE_RSEQ
 	select HAVE_SYSCALL_TRACEPOINTS
 	select HAVE_UNSTABLE_SCHED_CLOCK
arch/x86/include/asm/static_call.h new file +28 -0
@@ -0,0 +1,28 @@
+/* SPDX-License-Identifier: GPL-2.0 */
+#ifndef _ASM_STATIC_CALL_H
+#define _ASM_STATIC_CALL_H
+
+/*
+ * Manually construct a 5-byte direct JMP to prevent the assembler from
+ * optimizing it into a 2-byte JMP.
+ */
+#define __ARCH_STATIC_CALL_JMP_LABEL(key) ".L" __stringify(key ## _after_jmp)
+#define __ARCH_STATIC_CALL_TRAMP_JMP(key, func)				\
+	".byte 0xe9						\n"	\
+	".long " #func " - " __ARCH_STATIC_CALL_JMP_LABEL(key) "\n"	\
+	__ARCH_STATIC_CALL_JMP_LABEL(key) ":"
+
+/*
+ * This is a permanent trampoline which does a direct jump to the function.
+ * The direct jump get patched by static_call_update().
+ */
+#define ARCH_DEFINE_STATIC_CALL_TRAMP(key, func)			\
+	asm(".pushsection .text, \"ax\"				\n"	\
+	    ".align 4						\n"	\
+	    ".globl " STATIC_CALL_TRAMP_STR(key) "		\n"	\
+	    ".type " STATIC_CALL_TRAMP_STR(key) ", @function	\n"	\
+	    STATIC_CALL_TRAMP_STR(key) ":			\n"	\
+	    __ARCH_STATIC_CALL_TRAMP_JMP(key, func) "           \n"	\
+	    ".popsection					\n")
+
+#endif /* _ASM_STATIC_CALL_H */
arch/x86/kernel/Makefile +1 -0
@@ -63,6 +63,7 @@ obj-y			+= tsc.o tsc_msr.o io_delay.o rt
 obj-y			+= pci-iommu_table.o
 obj-y			+= resource.o
 obj-y			+= irqflags.o
+obj-y			+= static_call.o
 
 obj-y				+= process.o
 obj-y				+= fpu/
arch/x86/kernel/static_call.c new file +38 -0
@@ -0,0 +1,38 @@
+// SPDX-License-Identifier: GPL-2.0
+#include <linux/static_call.h>
+#include <linux/memory.h>
+#include <linux/bug.h>
+#include <asm/text-patching.h>
+#include <asm/nospec-branch.h>
+
+#define CALL_INSN_SIZE 5
+
+void arch_static_call_transform(void *site, void *tramp, void *func)
+{
+	unsigned char opcodes[CALL_INSN_SIZE];
+	unsigned char insn_opcode;
+	unsigned long insn;
+	s32 dest_relative;
+
+	mutex_lock(&text_mutex);
+
+	insn = (unsigned long)tramp;
+
+	insn_opcode = *(unsigned char *)insn;
+	if (insn_opcode != 0xE9) {
+		WARN_ONCE(1, "unexpected static call insn opcode 0x%x at %pS",
+			  insn_opcode, (void *)insn);
+		goto unlock;
+	}
+
+	dest_relative = (long)(func) - (long)(insn + CALL_INSN_SIZE);
+
+	opcodes[0] = insn_opcode;
+	memcpy(&opcodes[1], &dest_relative, CALL_INSN_SIZE - 1);
+
+	text_poke_bp((void *)insn, opcodes, CALL_INSN_SIZE, NULL);
+
+unlock:
+	mutex_unlock(&text_mutex);
+}
+EXPORT_SYMBOL_GPL(arch_static_call_transform);
#71 lore raw

Re: [PATCH 12/15] x86/static_call: Add out-of-line static call implementation

Quoted 105 lines · Peter Zijlstra

From: Josh Poimboeuf <[email protected]>

Add the x86 out-of-line static call implementation. For each key, a permanent trampoline is created which is the destination for all static calls for the given key. The trampoline has a direct jump which gets patched by static_call_update() when the destination function changes.

---

arch/x86/Kconfig                   |    1 
arch/x86/include/asm/static_call.h |   28 +++++++++++++++++++++++++++
arch/x86/kernel/Makefile           |    1 
arch/x86/kernel/static_call.c      |   38 +++++++++++++++++++++++++++++++++++++
4 files changed, 68 insertions(+)

create mode 100644 arch/x86/include/asm/static_call.h create mode 100644 arch/x86/kernel/static_call.c

arch/x86/Kconfig +1 -0
@@ -198,6 +198,7 @@ config X86
 	select HAVE_FUNCTION_ARG_ACCESS_API
 	select HAVE_STACKPROTECTOR		if CC_HAS_SANE_STACKPROTECTOR
 	select HAVE_STACK_VALIDATION		if X86_64
+	select HAVE_STATIC_CALL
 	select HAVE_RSEQ
 	select HAVE_SYSCALL_TRACEPOINTS
 	select HAVE_UNSTABLE_SCHED_CLOCK
arch/x86/include/asm/static_call.h new file +28 -0
@@ -0,0 +1,28 @@
+/* SPDX-License-Identifier: GPL-2.0 */
+#ifndef _ASM_STATIC_CALL_H
+#define _ASM_STATIC_CALL_H
+
+/*
+ * Manually construct a 5-byte direct JMP to prevent the assembler from
+ * optimizing it into a 2-byte JMP.
+ */
+#define __ARCH_STATIC_CALL_JMP_LABEL(key) ".L" __stringify(key ## _after_jmp)
+#define __ARCH_STATIC_CALL_TRAMP_JMP(key, func)				\
+	".byte 0xe9						\n"	\
+	".long " #func " - " __ARCH_STATIC_CALL_JMP_LABEL(key) "\n"	\
+	__ARCH_STATIC_CALL_JMP_LABEL(key) ":"
+
+/*
+ * This is a permanent trampoline which does a direct jump to the function.
+ * The direct jump get patched by static_call_update().
+ */
+#define ARCH_DEFINE_STATIC_CALL_TRAMP(key, func)			\
+	asm(".pushsection .text, \"ax\"				\n"	\
+	    ".align 4						\n"	\
+	    ".globl " STATIC_CALL_TRAMP_STR(key) "		\n"	\
+	    ".type " STATIC_CALL_TRAMP_STR(key) ", @function	\n"	\
+	    STATIC_CALL_TRAMP_STR(key) ":			\n"	\
+	    __ARCH_STATIC_CALL_TRAMP_JMP(key, func) "           \n"	\
+	    ".popsection					\n")
+
+#endif /* _ASM_STATIC_CALL_H */
arch/x86/kernel/Makefile +0 -0
@@ -63,6 +63,7 @@ obj-y			+= tsc.o tsc_msr.o io_delay.o rt

obj-y += pci-iommu_table.o
obj-y += resource.o
obj-y += irqflags.o
+obj-y += static_call.o

obj-y += process.o
obj-y += fpu/

arch/x86/kernel/static_call.c new file +15 -0
@@ -0,0 +1,38 @@
+// SPDX-License-Identifier: GPL-2.0
+#include <linux/static_call.h>
+#include <linux/memory.h>
+#include <linux/bug.h>
+#include <asm/text-patching.h>
+#include <asm/nospec-branch.h>
+
+#define CALL_INSN_SIZE 5
+
+void arch_static_call_transform(void *site, void *tramp, void *func)
+{
+	unsigned char opcodes[CALL_INSN_SIZE];
+	unsigned char insn_opcode;
+	unsigned long insn;
+	s32 dest_relative;

arch/x86/kernel/static_call.c line 25

Quoted 10 lines
arch/x86/kernel/static_call.c new file +10 -0
+
+	mutex_lock(&text_mutex);
+
+	insn = (unsigned long)tramp;
+
+	insn_opcode = *(unsigned char *)insn;
+	if (insn_opcode != 0xE9) {
+		WARN_ONCE(1, "unexpected static call insn opcode 0x%x at %pS",
+			  insn_opcode, (void *)insn);
+		goto unlock;

This might happen if a kprobe is installed on the call, no?

I don’t know if you want to be more gentle handling of this case (or perhaps modify can_probe() to prevent such a case).

maintainer
#72 lore raw

Re: [PATCH 12/15] x86/static_call: Add out-of-line static call implementation

Quoted 118 lines · On Fri, 7 Jun 2019 06:13:58 +0000 Nadav Amit <[email protected]> wrote:
Quoted 105 lines · Peter Zijlstra

From: Josh Poimboeuf <[email protected]>

Add the x86 out-of-line static call implementation. For each key, a permanent trampoline is created which is the destination for all static calls for the given key. The trampoline has a direct jump which gets patched by static_call_update() when the destination function changes.

---

arch/x86/Kconfig                   |    1 
arch/x86/include/asm/static_call.h |   28 +++++++++++++++++++++++++++
arch/x86/kernel/Makefile           |    1 
arch/x86/kernel/static_call.c      |   38 +++++++++++++++++++++++++++++++++++++
4 files changed, 68 insertions(+)

create mode 100644 arch/x86/include/asm/static_call.h create mode 100644 arch/x86/kernel/static_call.c

arch/x86/Kconfig +1 -0
@@ -198,6 +198,7 @@ config X86
 	select HAVE_FUNCTION_ARG_ACCESS_API
 	select HAVE_STACKPROTECTOR		if CC_HAS_SANE_STACKPROTECTOR
 	select HAVE_STACK_VALIDATION		if X86_64
+	select HAVE_STATIC_CALL
 	select HAVE_RSEQ
 	select HAVE_SYSCALL_TRACEPOINTS
 	select HAVE_UNSTABLE_SCHED_CLOCK
arch/x86/include/asm/static_call.h new file +28 -0
@@ -0,0 +1,28 @@
+/* SPDX-License-Identifier: GPL-2.0 */
+#ifndef _ASM_STATIC_CALL_H
+#define _ASM_STATIC_CALL_H
+
+/*
+ * Manually construct a 5-byte direct JMP to prevent the assembler from
+ * optimizing it into a 2-byte JMP.
+ */
+#define __ARCH_STATIC_CALL_JMP_LABEL(key) ".L" __stringify(key ## _after_jmp)
+#define __ARCH_STATIC_CALL_TRAMP_JMP(key, func)				\
+	".byte 0xe9						\n"	\
+	".long " #func " - " __ARCH_STATIC_CALL_JMP_LABEL(key) "\n"	\
+	__ARCH_STATIC_CALL_JMP_LABEL(key) ":"
+
+/*
+ * This is a permanent trampoline which does a direct jump to the function.
+ * The direct jump get patched by static_call_update().
+ */
+#define ARCH_DEFINE_STATIC_CALL_TRAMP(key, func)			\
+	asm(".pushsection .text, \"ax\"				\n"	\
+	    ".align 4						\n"	\
+	    ".globl " STATIC_CALL_TRAMP_STR(key) "		\n"	\
+	    ".type " STATIC_CALL_TRAMP_STR(key) ", @function	\n"	\
+	    STATIC_CALL_TRAMP_STR(key) ":			\n"	\
+	    __ARCH_STATIC_CALL_TRAMP_JMP(key, func) "           \n"	\
+	    ".popsection					\n")
+
+#endif /* _ASM_STATIC_CALL_H */
arch/x86/kernel/Makefile +0 -0
@@ -63,6 +63,7 @@ obj-y			+= tsc.o tsc_msr.o io_delay.o rt

obj-y += pci-iommu_table.o
obj-y += resource.o
obj-y += irqflags.o
+obj-y += static_call.o

obj-y += process.o
obj-y += fpu/

arch/x86/kernel/static_call.c new file +15 -0
@@ -0,0 +1,38 @@
+// SPDX-License-Identifier: GPL-2.0
+#include <linux/static_call.h>
+#include <linux/memory.h>
+#include <linux/bug.h>
+#include <asm/text-patching.h>
+#include <asm/nospec-branch.h>
+
+#define CALL_INSN_SIZE 5
+
+void arch_static_call_transform(void *site, void *tramp, void *func)
+{
+	unsigned char opcodes[CALL_INSN_SIZE];
+	unsigned char insn_opcode;
+	unsigned long insn;
+	s32 dest_relative;

arch/x86/kernel/static_call.c line 25

Quoted 10 lines
arch/x86/kernel/static_call.c new file +10 -0
+
+	mutex_lock(&text_mutex);
+
+	insn = (unsigned long)tramp;
+
+	insn_opcode = *(unsigned char *)insn;
+	if (insn_opcode != 0xE9) {
+		WARN_ONCE(1, "unexpected static call insn opcode 0x%x at %pS",
+			  insn_opcode, (void *)insn);
+		goto unlock;  

This might happen if a kprobe is installed on the call, no?

Quoted 3 lines

I don’t know if you want to be more gentle handling of this case (or perhaps modify can_probe() to prevent such a case).

Perhaps it is better to block kprobes from attaching to a static call. Or have it use the static call directly as it does with ftrace. But that would probably be much more work.

-- Steve

author maintainer
#73 lore raw

Re: [PATCH 12/15] x86/static_call: Add out-of-line static call implementation

In reply to Nadav Amit
Quoted 21 lines · Nadav Amit
Quoted 8 lines · Peter Zijlstra
arch/x86/kernel/static_call.c new file +6 -0
+void arch_static_call_transform(void *site, void *tramp, void *func)
+{
+	unsigned char opcodes[CALL_INSN_SIZE];
+	unsigned char insn_opcode;
+	unsigned long insn;
+	s32 dest_relative;

arch/x86/kernel/static_call.c line 25

Quoted 10 lines
arch/x86/kernel/static_call.c new file +10 -0
+
+	mutex_lock(&text_mutex);
+
+	insn = (unsigned long)tramp;
+
+	insn_opcode = *(unsigned char *)insn;
+	if (insn_opcode != 0xE9) {
+		WARN_ONCE(1, "unexpected static call insn opcode 0x%x at %pS",
+			  insn_opcode, (void *)insn);
+		goto unlock;

This might happen if a kprobe is installed on the call, no?

Quoted 3 lines

I don’t know if you want to be more gentle handling of this case (or perhaps modify can_probe() to prevent such a case).

yuck.. yes, that's something that needs consideration.

author maintainer
#74 lore raw

Re: [PATCH 12/15] x86/static_call: Add out-of-line static call implementation

Quoted 22 lines · Peter Zijlstra
Quoted 21 lines · Nadav Amit
Quoted 8 lines · Peter Zijlstra
arch/x86/kernel/static_call.c new file +6 -0
+void arch_static_call_transform(void *site, void *tramp, void *func)
+{
+	unsigned char opcodes[CALL_INSN_SIZE];
+	unsigned char insn_opcode;
+	unsigned long insn;
+	s32 dest_relative;

arch/x86/kernel/static_call.c line 25

Quoted 10 lines
arch/x86/kernel/static_call.c new file +10 -0
+
+	mutex_lock(&text_mutex);
+
+	insn = (unsigned long)tramp;
+
+	insn_opcode = *(unsigned char *)insn;
+	if (insn_opcode != 0xE9) {
+		WARN_ONCE(1, "unexpected static call insn opcode 0x%x at %pS",
+			  insn_opcode, (void *)insn);
+		goto unlock;

This might happen if a kprobe is installed on the call, no?

Quoted 5 lines
Quoted 3 lines

I don’t know if you want to be more gentle handling of this case (or perhaps modify can_probe() to prevent such a case).

yuck.. yes, that's something that needs consideration.

For jump_label this is avoided by jump_label_text_reserved(), I'm thinking static_call should do the same.

author maintainer
#75 lore raw

PATCH 13/15 x86/static_call: Add inline static call implementation for x86-64

In reply to Peter Zijlstra

From: Josh Poimboeuf <[email protected]>

Add the inline static call implementation for x86-64. For each key, a temporary trampoline is created, named __static_call_tramp_<key>. The trampoline has an indirect jump to the destination function.

Objtool uses the trampoline naming convention to detect all the call sites. It then annotates those call sites in the .static_call_sites section.

During boot (and module init), the call sites are patched to call directly into the destination function. The temporary trampoline is then no longer used.

Cc Steven Rostedt <[email protected]>
Cc Julia Cartwright <[email protected]>
Cc Ingo Molnar <[email protected]>
Cc Ard Biesheuvel <[email protected]>
Cc Jason Baron <[email protected]>
Cc Linus Torvalds <[email protected]>
Cc Jiri Kosina <[email protected]>
Cc Thomas Gleixner <[email protected]>
Cc Masami Hiramatsu <[email protected]>
Cc Borislav Petkov <[email protected]>
Cc David Laight <[email protected]>
Cc Jessica Yu <[email protected]>
Cc Andy Lutomirski <[email protected]>
Cc "H. Peter Anvin" <[email protected]>
---
 arch/x86/Kconfig                                |    3 
 arch/x86/include/asm/static_call.h              |   28 ++++-
 arch/x86/kernel/asm-offsets.c                   |    6 +
 arch/x86/kernel/static_call.c                   |   12 +-
 include/linux/static_call.h                     |    2 
 tools/objtool/Makefile                          |    3 
 tools/objtool/check.c                           |  125 +++++++++++++++++++++++-
 tools/objtool/check.h                           |    2 
 tools/objtool/elf.h                             |    1 
 tools/objtool/include/linux/static_call_types.h |   19 +++
 tools/objtool/sync-check.sh                     |    1 
 11 files changed, 193 insertions(+), 9 deletions(-)
 create mode 100644 tools/objtool/include/linux/static_call_types.h
arch/x86/Kconfig +2 -1
@@ -199,6 +199,7 @@ config X86
 	select HAVE_STACKPROTECTOR		if CC_HAS_SANE_STACKPROTECTOR
 	select HAVE_STACK_VALIDATION		if X86_64
 	select HAVE_STATIC_CALL
+	select HAVE_STATIC_CALL_INLINE		if HAVE_STACK_VALIDATION
 	select HAVE_RSEQ
 	select HAVE_SYSCALL_TRACEPOINTS
 	select HAVE_UNSTABLE_SCHED_CLOCK
@@ -213,6 +214,7 @@ config X86
 	select RTC_MC146818_LIB
 	select SPARSE_IRQ
 	select SRCU
+	select STACK_VALIDATION			if HAVE_STACK_VALIDATION && (HAVE_STATIC_CALL_INLINE || RETPOLINE)
 	select SYSCTL_EXCEPTION_TRACE
 	select THREAD_INFO_IN_TASK
 	select USER_STACKTRACE_SUPPORT
@@ -439,7 +441,6 @@ config GOLDFISH
 config RETPOLINE
 	bool "Avoid speculative indirect branches in kernel"
 	default y
-	select STACK_VALIDATION if HAVE_STACK_VALIDATION
 	help
 	  Compile kernel with the retpoline compiler options to guard against
 	  kernel-to-user data leaks by avoiding speculative indirect
arch/x86/include/asm/static_call.h +26 -2
@@ -2,6 +2,20 @@
 #ifndef _ASM_STATIC_CALL_H
 #define _ASM_STATIC_CALL_H
 
+#include <asm/asm-offsets.h>
+
+#ifdef CONFIG_HAVE_STATIC_CALL_INLINE
+
+/*
+ * This trampoline is only used during boot / module init, so it's safe to use
+ * the indirect branch without a retpoline.
+ */
+#define __ARCH_STATIC_CALL_TRAMP_JMP(key, func)				\
+	ANNOTATE_RETPOLINE_SAFE						\
+	"jmpq *" __stringify(key) "+" __stringify(SC_KEY_func) "(%rip) \n"
+
+#else /* !CONFIG_HAVE_STATIC_CALL_INLINE */
+
 /*
  * Manually construct a 5-byte direct JMP to prevent the assembler from
  * optimizing it into a 2-byte JMP.
@@ -12,9 +26,19 @@
 	".long " #func " - " __ARCH_STATIC_CALL_JMP_LABEL(key) "\n"	\
 	__ARCH_STATIC_CALL_JMP_LABEL(key) ":"
 
+#endif /* !CONFIG_HAVE_STATIC_CALL_INLINE */
+
 /*
- * This is a permanent trampoline which does a direct jump to the function.
- * The direct jump get patched by static_call_update().
+ * For CONFIG_HAVE_STATIC_CALL_INLINE, this is a temporary trampoline which
+ * uses the current value of the key->func pointer to do an indirect jump to
+ * the function.  This trampoline is only used during boot, before the call
+ * sites get patched by static_call_update().  The name of this trampoline has
+ * a magical aspect: objtool uses it to find static call sites so it can create
+ * the .static_call_sites section.
+ *
+ * For CONFIG_HAVE_STATIC_CALL, this is a permanent trampoline which
+ * does a direct jump to the function.  The direct jump gets patched by
+ * static_call_update().
  */
 #define ARCH_DEFINE_STATIC_CALL_TRAMP(key, func)			\
 	asm(".pushsection .text, \"ax\"				\n"	\
arch/x86/kernel/asm-offsets.c +6 -0
@@ -12,6 +12,7 @@
 #include <linux/hardirq.h>
 #include <linux/suspend.h>
 #include <linux/kbuild.h>
+#include <linux/static_call.h>
 #include <asm/processor.h>
 #include <asm/thread_info.h>
 #include <asm/sigframe.h>
@@ -104,4 +105,9 @@ static void __used common(void)
 	OFFSET(TSS_sp0, tss_struct, x86_tss.sp0);
 	OFFSET(TSS_sp1, tss_struct, x86_tss.sp1);
 	OFFSET(TSS_sp2, tss_struct, x86_tss.sp2);
+
+#ifdef CONFIG_HAVE_STATIC_CALL_INLINE
+	BLANK();
+	OFFSET(SC_KEY_func, static_call_key, func);
+#endif
 }
arch/x86/kernel/static_call.c +9 -3
@@ -10,16 +10,22 @@
 void arch_static_call_transform(void *site, void *tramp, void *func)
 {
 	unsigned char opcodes[CALL_INSN_SIZE];
-	unsigned char insn_opcode;
+	unsigned char insn_opcode, expected;
 	unsigned long insn;
 	s32 dest_relative;
 
 	mutex_lock(&text_mutex);
 
-	insn = (unsigned long)tramp;
+	if (IS_ENABLED(CONFIG_HAVE_STATIC_CALL_INLINE)) {
+		insn = (unsigned long)site;
+		expected = 0xE8; /* CALL */
+	} else {
+		insn = (unsigned long)tramp;
+		expected = 0xE9; /* JMP */
+	}
 
 	insn_opcode = *(unsigned char *)insn;
-	if (insn_opcode != 0xE9) {
+	if (insn_opcode != expected) {
 		WARN_ONCE(1, "unexpected static call insn opcode 0x%x at %pS",
 			  insn_opcode, (void *)insn);
 		goto unlock;
include/linux/static_call.h +1 -1
@@ -59,7 +59,7 @@
 #include <linux/cpu.h>
 #include <linux/static_call_types.h>
 
-#ifdef CONFIG_HAVE_STATIC_CALL
+#if defined(CONFIG_HAVE_STATIC_CALL) && !defined(COMPILE_OFFSETS)
 #include <asm/static_call.h>
 extern void arch_static_call_transform(void *site, void *tramp, void *func);
 #endif
tools/objtool/Makefile +2 -1
@@ -33,7 +33,8 @@ all: $(OBJTOOL)
 
 INCLUDES := -I$(srctree)/tools/include \
 	    -I$(srctree)/tools/arch/$(HOSTARCH)/include/uapi \
-	    -I$(srctree)/tools/objtool/arch/$(ARCH)/include
+	    -I$(srctree)/tools/objtool/arch/$(ARCH)/include \
+	    -I$(srctree)/tools/objtool/include
 WARNINGS := $(EXTRA_WARNINGS) -Wno-switch-default -Wno-switch-enum -Wno-packed
 CFLAGS   += -Werror $(WARNINGS) $(KBUILD_HOSTCFLAGS) -g $(INCLUDES) $(LIBELF_FLAGS)
 LDFLAGS  += $(LIBELF_LIBS) $(LIBSUBCMD) $(KBUILD_HOSTLDFLAGS)
tools/objtool/check.c +124 -1

185 lines not on this page · raw message

tools/objtool/check.h +2 -0
@@ -28,6 +28,7 @@ struct insn_state {
 struct instruction {
 	struct list_head list;
 	struct hlist_node hash;
+	struct list_head static_call_node;
 	struct section *sec;
 	unsigned long offset;
 	unsigned int len;
@@ -49,6 +50,7 @@ struct objtool_file {
 	struct elf *elf;
 	struct list_head insn_list;
 	DECLARE_HASHTABLE(insn_hash, 16);
+	struct list_head static_call_list;
 	bool ignore_unreachables, c_file, hints, rodata;
 };
 
tools/objtool/elf.h +1 -0
@@ -51,6 +51,7 @@ struct symbol {
 	unsigned int len;
 	struct symbol *pfunc, *cfunc, *alias;
 	bool uaccess_safe;
+	bool static_call_tramp;
 };
 
 struct rela {
tools/objtool/include/linux/static_call_types.h new file +19 -0
@@ -0,0 +1,19 @@
+/* SPDX-License-Identifier: GPL-2.0 */
+#ifndef _STATIC_CALL_TYPES_H
+#define _STATIC_CALL_TYPES_H
+
+#include <linux/stringify.h>
+
+#define STATIC_CALL_TRAMP_PREFIX ____static_call_tramp_
+#define STATIC_CALL_TRAMP_PREFIX_STR __stringify(STATIC_CALL_TRAMP_PREFIX)
+
+#define STATIC_CALL_TRAMP(key) __PASTE(STATIC_CALL_TRAMP_PREFIX, key)
+#define STATIC_CALL_TRAMP_STR(key) __stringify(STATIC_CALL_TRAMP(key))
+
+/* The static call site table is created by objtool. */
+struct static_call_site {
+	s32 addr;
+	s32 key;
+};
+
+#endif /* _STATIC_CALL_TYPES_H */
tools/objtool/sync-check.sh +1 -0
@@ -10,6 +10,7 @@ arch/x86/include/asm/insn.h
 arch/x86/include/asm/inat.h
 arch/x86/include/asm/inat_types.h
 arch/x86/include/asm/orc_types.h
+include/linux/static_call_types.h
 '
 
 check()
#76 lore raw

Re: [PATCH 13/15] x86/static_call: Add inline static call implementation for x86-64

Quoted 147 lines · Peter Zijlstra

From: Josh Poimboeuf <[email protected]>

Add the inline static call implementation for x86-64. For each key, a temporary trampoline is created, named __static_call_tramp_<key>. The trampoline has an indirect jump to the destination function.

Objtool uses the trampoline naming convention to detect all the call sites. It then annotates those call sites in the .static_call_sites section.

During boot (and module init), the call sites are patched to call directly into the destination function. The temporary trampoline is then no longer used.

---

arch/x86/Kconfig                                |    3 
arch/x86/include/asm/static_call.h              |   28 ++++-
arch/x86/kernel/asm-offsets.c                   |    6 +
arch/x86/kernel/static_call.c                   |   12 +-
include/linux/static_call.h                     |    2 
tools/objtool/Makefile                          |    3 
tools/objtool/check.c                           |  125 +++++++++++++++++++++++-
tools/objtool/check.h                           |    2 
tools/objtool/elf.h                             |    1 
tools/objtool/include/linux/static_call_types.h |   19 +++
tools/objtool/sync-check.sh                     |    1 
11 files changed, 193 insertions(+), 9 deletions(-)

create mode 100644 tools/objtool/include/linux/static_call_types.h

arch/x86/Kconfig +2 -0
@@ -199,6 +199,7 @@ config X86
 	select HAVE_STACKPROTECTOR		if CC_HAS_SANE_STACKPROTECTOR
 	select HAVE_STACK_VALIDATION		if X86_64
 	select HAVE_STATIC_CALL
+	select HAVE_STATIC_CALL_INLINE		if HAVE_STACK_VALIDATION
 	select HAVE_RSEQ
 	select HAVE_SYSCALL_TRACEPOINTS
 	select HAVE_UNSTABLE_SCHED_CLOCK
@@ -213,6 +214,7 @@ config X86
 	select RTC_MC146818_LIB
 	select SPARSE_IRQ
 	select SRCU
+	select STACK_VALIDATION			if HAVE_STACK_VALIDATION && (HAVE_STATIC_CALL_INLINE || RETPOLINE)
 	select SYSCTL_EXCEPTION_TRACE
 	select THREAD_INFO_IN_TASK
 	select USER_STACKTRACE_SUPPORT
@@ -439,7 +441,6 @@ config GOLDFISH

config RETPOLINE
bool "Avoid speculative indirect branches in kernel" default y

arch/x86/Kconfig +0 -1
-	select STACK_VALIDATION if HAVE_STACK_VALIDATION

help
Compile kernel with the retpoline compiler options to guard against kernel-to-user data leaks by avoiding speculative indirect

arch/x86/include/asm/static_call.h +26 -2
@@ -2,6 +2,20 @@
 #ifndef _ASM_STATIC_CALL_H
 #define _ASM_STATIC_CALL_H
 
+#include <asm/asm-offsets.h>
+
+#ifdef CONFIG_HAVE_STATIC_CALL_INLINE
+
+/*
+ * This trampoline is only used during boot / module init, so it's safe to use
+ * the indirect branch without a retpoline.
+ */
+#define __ARCH_STATIC_CALL_TRAMP_JMP(key, func)				\
+	ANNOTATE_RETPOLINE_SAFE						\
+	"jmpq *" __stringify(key) "+" __stringify(SC_KEY_func) "(%rip) \n"
+
+#else /* !CONFIG_HAVE_STATIC_CALL_INLINE */
+
 /*
 * Manually construct a 5-byte direct JMP to prevent the assembler from
 * optimizing it into a 2-byte JMP.
@@ -12,9 +26,19 @@
 	".long " #func " - " __ARCH_STATIC_CALL_JMP_LABEL(key) "\n"	\
 	__ARCH_STATIC_CALL_JMP_LABEL(key) ":"
 
+#endif /* !CONFIG_HAVE_STATIC_CALL_INLINE */
+
 /*
- * This is a permanent trampoline which does a direct jump to the function.
- * The direct jump get patched by static_call_update().
+ * For CONFIG_HAVE_STATIC_CALL_INLINE, this is a temporary trampoline which
+ * uses the current value of the key->func pointer to do an indirect jump to
+ * the function.  This trampoline is only used during boot, before the call
+ * sites get patched by static_call_update().  The name of this trampoline has
+ * a magical aspect: objtool uses it to find static call sites so it can create
+ * the .static_call_sites section.
+ *
+ * For CONFIG_HAVE_STATIC_CALL, this is a permanent trampoline which
+ * does a direct jump to the function.  The direct jump gets patched by
+ * static_call_update().
 */
 #define ARCH_DEFINE_STATIC_CALL_TRAMP(key, func)			\
 	asm(".pushsection .text, \"ax\"				\n"	\
arch/x86/kernel/asm-offsets.c +6 -0
@@ -12,6 +12,7 @@
 #include <linux/hardirq.h>
 #include <linux/suspend.h>
 #include <linux/kbuild.h>
+#include <linux/static_call.h>
 #include <asm/processor.h>
 #include <asm/thread_info.h>
 #include <asm/sigframe.h>
@@ -104,4 +105,9 @@ static void __used common(void)
 	OFFSET(TSS_sp0, tss_struct, x86_tss.sp0);
 	OFFSET(TSS_sp1, tss_struct, x86_tss.sp1);
 	OFFSET(TSS_sp2, tss_struct, x86_tss.sp2);
+
+#ifdef CONFIG_HAVE_STATIC_CALL_INLINE
+	BLANK();
+	OFFSET(SC_KEY_func, static_call_key, func);
+#endif
 }
arch/x86/kernel/static_call.c +0 -1
@@ -10,16 +10,22 @@
 void arch_static_call_transform(void *site, void *tramp, void *func)
 {
 	unsigned char opcodes[CALL_INSN_SIZE];
-	unsigned char insn_opcode;

arch/x86/kernel/static_call.c line 21

Quoted 10 lines
arch/x86/kernel/static_call.c +4 -1
+	unsigned char insn_opcode, expected;
 	unsigned long insn;
 	s32 dest_relative;
 
 	mutex_lock(&text_mutex);
 
-	insn = (unsigned long)tramp;
+	if (IS_ENABLED(CONFIG_HAVE_STATIC_CALL_INLINE)) {
+		insn = (unsigned long)site;
+		expected = 0xE8; /* CALL */

RELATIVECALL_OPCODE ?

arch/x86/kernel/static_call.c line 24

Quoted 3 lines
arch/x86/kernel/static_call.c +3 -0
+	} else {
+		insn = (unsigned long)tramp;
+		expected = 0xE9; /* JMP */

RELATIVEJUMP_OPCODE ?

( I did not review the objtool parts )

#77 lore raw

Re: [PATCH 13/15] x86/static_call: Add inline static call implementation for x86-64

In reply to Peter Zijlstra
Quoted 9 lines · Peter Zijlstra
arch/x86/include/asm/static_call.h +3 -0
@@ -2,6 +2,20 @@
 #ifndef _ASM_STATIC_CALL_H
 #define _ASM_STATIC_CALL_H
 
+#include <asm/asm-offsets.h>
+
+#ifdef CONFIG_HAVE_STATIC_CALL_INLINE

arch/x86/include/asm/static_call.h line 17

Quoted 10 lines
arch/x86/include/asm/static_call.h +10 -0
+
+/*
+ * This trampoline is only used during boot / module init, so it's safe to use
+ * the indirect branch without a retpoline.
+ */
+#define __ARCH_STATIC_CALL_TRAMP_JMP(key, func)				\
+	ANNOTATE_RETPOLINE_SAFE						\
+	"jmpq *" __stringify(key) "+" __stringify(SC_KEY_func) "(%rip) \n"
+
+#else /* !CONFIG_HAVE_STATIC_CALL_INLINE */

I wonder if we can simplify this (and drop the indirect branch) by getting rid of the above cruft, and instead just use the out-of-line trampoline as the default for inline as well.

Then the inline case could fall back to the out-of-line implementation (by patching the trampoline's jmp dest) before static_call_initialized is set.

Signature · 1 line
Josh
#78 lore raw

Re: [PATCH 13/15] x86/static_call: Add inline static call implementation for x86-64

Quoted 27 lines · Josh Poimboeuf
Quoted 9 lines · Peter Zijlstra
arch/x86/include/asm/static_call.h +3 -0
@@ -2,6 +2,20 @@
 #ifndef _ASM_STATIC_CALL_H
 #define _ASM_STATIC_CALL_H
 
+#include <asm/asm-offsets.h>
+
+#ifdef CONFIG_HAVE_STATIC_CALL_INLINE

arch/x86/include/asm/static_call.h line 17

Quoted 10 lines
arch/x86/include/asm/static_call.h +10 -0
+
+/*
+ * This trampoline is only used during boot / module init, so it's safe to use
+ * the indirect branch without a retpoline.
+ */
+#define __ARCH_STATIC_CALL_TRAMP_JMP(key, func)				\
+	ANNOTATE_RETPOLINE_SAFE						\
+	"jmpq *" __stringify(key) "+" __stringify(SC_KEY_func) "(%rip) \n"
+
+#else /* !CONFIG_HAVE_STATIC_CALL_INLINE */

I wonder if we can simplify this (and drop the indirect branch) by getting rid of the above cruft, and instead just use the out-of-line trampoline as the default for inline as well.

Quoted 3 lines

Then the inline case could fall back to the out-of-line implementation (by patching the trampoline's jmp dest) before static_call_initialized is set.

I must be missing some context - but what guarantees that this indirect branch would be exactly 5 bytes long? Isn’t there an assumption that this would be the case? Shouldn’t there be some handling of the padding?

#79 lore raw

Re: [PATCH 13/15] x86/static_call: Add inline static call implementation for x86-64

Quoted 27 lines · Nadav Amit
Quoted 27 lines · Josh Poimboeuf
Quoted 9 lines · Peter Zijlstra
arch/x86/include/asm/static_call.h +3 -0
@@ -2,6 +2,20 @@
 #ifndef _ASM_STATIC_CALL_H
 #define _ASM_STATIC_CALL_H
 
+#include <asm/asm-offsets.h>
+
+#ifdef CONFIG_HAVE_STATIC_CALL_INLINE

arch/x86/include/asm/static_call.h line 17

Quoted 10 lines
arch/x86/include/asm/static_call.h +10 -0
+
+/*
+ * This trampoline is only used during boot / module init, so it's safe to use
+ * the indirect branch without a retpoline.
+ */
+#define __ARCH_STATIC_CALL_TRAMP_JMP(key, func)				\
+	ANNOTATE_RETPOLINE_SAFE						\
+	"jmpq *" __stringify(key) "+" __stringify(SC_KEY_func) "(%rip) \n"
+
+#else /* !CONFIG_HAVE_STATIC_CALL_INLINE */

I wonder if we can simplify this (and drop the indirect branch) by getting rid of the above cruft, and instead just use the out-of-line trampoline as the default for inline as well.

Quoted 7 lines
Quoted 3 lines

Then the inline case could fall back to the out-of-line implementation (by patching the trampoline's jmp dest) before static_call_initialized is set.

I must be missing some context - but what guarantees that this indirect branch would be exactly 5 bytes long? Isn’t there an assumption that this would be the case? Shouldn’t there be some handling of the padding?

We don't patch the indirect branch. It's just part of a temporary trampoline which is called by the call site, and which does "jmp key->func" during boot until static call initialization is done.

(Though I'm suggesting removing that.)

Signature · 1 line
Josh
#80 lore raw

Re: [PATCH 13/15] x86/static_call: Add inline static call implementation for x86-64

Quoted 38 lines · Josh Poimboeuf
Quoted 27 lines · Nadav Amit
Quoted 27 lines · Josh Poimboeuf
Quoted 9 lines · Peter Zijlstra
arch/x86/include/asm/static_call.h +3 -0
@@ -2,6 +2,20 @@
 #ifndef _ASM_STATIC_CALL_H
 #define _ASM_STATIC_CALL_H
 
+#include <asm/asm-offsets.h>
+
+#ifdef CONFIG_HAVE_STATIC_CALL_INLINE

arch/x86/include/asm/static_call.h line 17

Quoted 10 lines
arch/x86/include/asm/static_call.h +10 -0
+
+/*
+ * This trampoline is only used during boot / module init, so it's safe to use
+ * the indirect branch without a retpoline.
+ */
+#define __ARCH_STATIC_CALL_TRAMP_JMP(key, func)				\
+	ANNOTATE_RETPOLINE_SAFE						\
+	"jmpq *" __stringify(key) "+" __stringify(SC_KEY_func) "(%rip) \n"
+
+#else /* !CONFIG_HAVE_STATIC_CALL_INLINE */

I wonder if we can simplify this (and drop the indirect branch) by getting rid of the above cruft, and instead just use the out-of-line trampoline as the default for inline as well.

Quoted 7 lines
Quoted 3 lines

Then the inline case could fall back to the out-of-line implementation (by patching the trampoline's jmp dest) before static_call_initialized is set.

I must be missing some context - but what guarantees that this indirect branch would be exactly 5 bytes long? Isn’t there an assumption that this would be the case? Shouldn’t there be some handling of the padding?

Quoted 5 lines

We don't patch the indirect branch. It's just part of a temporary trampoline which is called by the call site, and which does "jmp key->func" during boot until static call initialization is done.

(Though I'm suggesting removing that.)

Oh... I see.

On another note - even if this branch is only executed during module initialization, it does seem safer to use a retpoline instead of an indirect branch (consider a branch that is run many times on one hardware thread on SMT, when STIBP is not set, and attacker code is run on the second thread).

I guess you don’t simply call the retpoline code so since you don’t have a clobbered register to hold the target. But it still seems possible to use a retpoline. Anyhow, it might be a moot discussion if this code is removed.

author maintainer
#81 lore raw

Re: [PATCH 13/15] x86/static_call: Add inline static call implementation for x86-64

In reply to Josh Poimboeuf
Quoted 25 lines · Josh Poimboeuf
Quoted 9 lines · Peter Zijlstra
arch/x86/include/asm/static_call.h +3 -0
@@ -2,6 +2,20 @@
 #ifndef _ASM_STATIC_CALL_H
 #define _ASM_STATIC_CALL_H
 
+#include <asm/asm-offsets.h>
+
+#ifdef CONFIG_HAVE_STATIC_CALL_INLINE

arch/x86/include/asm/static_call.h line 17

Quoted 10 lines
arch/x86/include/asm/static_call.h +10 -0
+
+/*
+ * This trampoline is only used during boot / module init, so it's safe to use
+ * the indirect branch without a retpoline.
+ */
+#define __ARCH_STATIC_CALL_TRAMP_JMP(key, func)				\
+	ANNOTATE_RETPOLINE_SAFE						\
+	"jmpq *" __stringify(key) "+" __stringify(SC_KEY_func) "(%rip) \n"
+
+#else /* !CONFIG_HAVE_STATIC_CALL_INLINE */

I wonder if we can simplify this (and drop the indirect branch) by getting rid of the above cruft, and instead just use the out-of-line trampoline as the default for inline as well.

Quoted 3 lines

Then the inline case could fall back to the out-of-line implementation (by patching the trampoline's jmp dest) before static_call_initialized is set.

I think I've got that covered. I changed arch_static_call_transform() to (always) first rewrite the trampoline and then the in-line site.

That way, when early/module crud comes in that still points at the trampoline, it will jump to the right place.

I've not even compiled yet, but it ought to work ;-)

author maintainer
#82 lore raw

PATCH 14/15 static_call: Simple self-test module

In reply to Peter Zijlstra
---
 lib/Kconfig.debug      |    8 ++++++++
 lib/Makefile           |    1 +
 lib/test_static_call.c |   41 +++++++++++++++++++++++++++++++++++++++++
 3 files changed, 50 insertions(+)
lib/Kconfig.debug +8 -0
@@ -1955,6 +1955,14 @@ config TEST_STATIC_KEYS
 
 	  If unsure, say N.
 
+config TEST_STATIC_CALL
+	tristate "Test static call"
+	depends on m
+	help
+	  Test the static call interfaces.
+
+	  If unsure, say N.
+
 config TEST_KMOD
 	tristate "kmod stress tester"
 	depends on m
lib/Makefile +1 -0
@@ -79,6 +79,7 @@ obj-$(CONFIG_TEST_SORT) += test_sort.o
 obj-$(CONFIG_TEST_USER_COPY) += test_user_copy.o
 obj-$(CONFIG_TEST_STATIC_KEYS) += test_static_keys.o
 obj-$(CONFIG_TEST_STATIC_KEYS) += test_static_key_base.o
+obj-$(CONFIG_TEST_STATIC_CALL) += test_static_call.o
 obj-$(CONFIG_TEST_PRINTF) += test_printf.o
 obj-$(CONFIG_TEST_BITMAP) += test_bitmap.o
 obj-$(CONFIG_TEST_STRSCPY) += test_strscpy.o
lib/test_static_call.c new file +41 -0

41 lines not on this page · raw message

#83 lore raw

Re: [PATCH 14/15] static_call: Simple self-test module

Quoted 12 lines · Peter Zijlstra
---
 lib/Kconfig.debug      |    8 ++++++++
 lib/Makefile           |    1 +
 lib/test_static_call.c |   41 +++++++++++++++++++++++++++++++++++++++++
 3 files changed, 50 insertions(+)
lib/Kconfig.debug +0 -0
@@ -1955,6 +1955,14 @@ config TEST_STATIC_KEYS
 

lib/Kconfig.debug config TEST_STATIC_KEYS · line 1965

Quoted 10 lines
lib/Kconfig.debug +8 -0
 	  If unsure, say N.
 
+config TEST_STATIC_CALL
+	tristate "Test static call"
+	depends on m
+	help
+	  Test the static call interfaces.
+
+	  If unsure, say N.
+

Any reason why we wouldn't just make this a built-in boot time test?

Signature · 1 line
Josh
author maintainer
#84 lore raw

Re: [PATCH 14/15] static_call: Simple self-test module

Quoted 15 lines · Josh Poimboeuf
Quoted 14 lines · Peter Zijlstra
---
 lib/Kconfig.debug      |    8 ++++++++
 lib/Makefile           |    1 +
 lib/test_static_call.c |   41 +++++++++++++++++++++++++++++++++++++++++
 3 files changed, 50 insertions(+)
lib/Kconfig.debug +0 -0
@@ -1955,6 +1955,14 @@ config TEST_STATIC_KEYS
 
 	  If unsure, say N.
 
Quoted 10 lines

lib/Kconfig.debug config TEST_STATIC_KEYS · line 1965

Quoted 8 lines
lib/Kconfig.debug +8 -0
+config TEST_STATIC_CALL
+	tristate "Test static call"
+	depends on m
+	help
+	  Test the static call interfaces.
+
+	  If unsure, say N.
+

Any reason why we wouldn't just make this a built-in boot time test?

None what so ever; but I did copy paste from the static_key stuff and that has it for some rasin.

#85 lore raw

Re: [PATCH 14/15] static_call: Simple self-test module

Quoted 27 lines · Peter Zijlstra
Quoted 15 lines · Josh Poimboeuf
Quoted 14 lines · Peter Zijlstra
---
 lib/Kconfig.debug      |    8 ++++++++
 lib/Makefile           |    1 +
 lib/test_static_call.c |   41 +++++++++++++++++++++++++++++++++++++++++
 3 files changed, 50 insertions(+)
lib/Kconfig.debug +0 -0
@@ -1955,6 +1955,14 @@ config TEST_STATIC_KEYS
 
 	  If unsure, say N.
 
Quoted 10 lines

lib/Kconfig.debug config TEST_STATIC_KEYS · line 1965

Quoted 8 lines
lib/Kconfig.debug +8 -0
+config TEST_STATIC_CALL
+	tristate "Test static call"
+	depends on m
+	help
+	  Test the static call interfaces.
+
+	  If unsure, say N.
+

Any reason why we wouldn't just make this a built-in boot time test?

Quoted 2 lines

None what so ever; but I did copy paste from the static_key stuff and that has it for some rasin.

Their functionality is pretty crucial, and I doubt anybody is manually building and loading these tests? Seems like built-in tests would be wiser for both static calls/keys.

Signature · 1 line
Josh
author maintainer
#86 lore raw

PATCH 15/15 tracepoints: Use static_call

In reply to Peter Zijlstra

From: Steven Rostedt (VMware) <[email protected]>

... Changelog goes here ...

---
 include/linux/tracepoint-defs.h |    4 ++
 include/linux/tracepoint.h      |   73 ++++++++++++++++++++++++++--------------
 include/trace/define_trace.h    |   14 +++----
 kernel/tracepoint.c             |   25 +++++++++++--
 4 files changed, 80 insertions(+), 36 deletions(-)
include/linux/tracepoint-defs.h +4 -0
@@ -11,6 +11,8 @@
 #include <linux/atomic.h>
 #include <linux/static_key.h>
 
+struct static_call_key;
+
 struct trace_print_flags {
 	unsigned long		mask;
 	const char		*name;
@@ -30,6 +32,8 @@ struct tracepoint_func {
 struct tracepoint {
 	const char *name;		/* Tracepoint name */
 	struct static_key key;
+	struct static_call_key *static_call_key;
+	void *iterator;
 	int (*regfunc)(void);
 	void (*unregfunc)(void);
 	struct tracepoint_func __rcu *funcs;
include/linux/tracepoint.h +48 -25

134 lines not on this page · raw message

include/trace/define_trace.h +7 -7
@@ -25,7 +25,7 @@
 
 #undef TRACE_EVENT
 #define TRACE_EVENT(name, proto, args, tstruct, assign, print)	\
-	DEFINE_TRACE(name)
+	DEFINE_TRACE(name, PARAMS(proto), PARAMS(args))
 
 #undef TRACE_EVENT_CONDITION
 #define TRACE_EVENT_CONDITION(name, proto, args, cond, tstruct, assign, print) \
@@ -39,12 +39,12 @@
 #undef TRACE_EVENT_FN
 #define TRACE_EVENT_FN(name, proto, args, tstruct,		\
 		assign, print, reg, unreg)			\
-	DEFINE_TRACE_FN(name, reg, unreg)
+	DEFINE_TRACE_FN(name, reg, unreg, PARAMS(proto), PARAMS(args))
 
 #undef TRACE_EVENT_FN_COND
 #define TRACE_EVENT_FN_COND(name, proto, args, cond, tstruct,		\
 		assign, print, reg, unreg)			\
-	DEFINE_TRACE_FN(name, reg, unreg)
+	DEFINE_TRACE_FN(name, reg, unreg, PARAMS(proto), PARAMS(args))
 
 #undef TRACE_EVENT_NOP
 #define TRACE_EVENT_NOP(name, proto, args, struct, assign, print)
@@ -54,15 +54,15 @@
 
 #undef DEFINE_EVENT
 #define DEFINE_EVENT(template, name, proto, args) \
-	DEFINE_TRACE(name)
+	DEFINE_TRACE(name, PARAMS(proto), PARAMS(args))
 
 #undef DEFINE_EVENT_FN
 #define DEFINE_EVENT_FN(template, name, proto, args, reg, unreg) \
-	DEFINE_TRACE_FN(name, reg, unreg)
+	DEFINE_TRACE_FN(name, reg, unreg, PARAMS(proto), PARAMS(args))
 
 #undef DEFINE_EVENT_PRINT
 #define DEFINE_EVENT_PRINT(template, name, proto, args, print)	\
-	DEFINE_TRACE(name)
+	DEFINE_TRACE(name, PARAMS(proto), PARAMS(args))
 
 #undef DEFINE_EVENT_CONDITION
 #define DEFINE_EVENT_CONDITION(template, name, proto, args, cond) \
@@ -70,7 +70,7 @@
 
 #undef DECLARE_TRACE
 #define DECLARE_TRACE(name, proto, args)	\
-	DEFINE_TRACE(name)
+	DEFINE_TRACE(name, PARAMS(proto), PARAMS(args))
 
 #undef TRACE_INCLUDE
 #undef __TRACE_INCLUDE
kernel/tracepoint.c +21 -4
@@ -127,7 +127,7 @@ static void debug_print_probes(struct tr
 
 static struct tracepoint_func *
 func_add(struct tracepoint_func **funcs, struct tracepoint_func *tp_func,
-	 int prio)
+	 int prio, int *tot_probes)
 {
 	struct tracepoint_func *old, *new;
 	int nr_probes = 0;
@@ -170,11 +170,12 @@ func_add(struct tracepoint_func **funcs,
 	new[nr_probes + 1].func = NULL;
 	*funcs = new;
 	debug_print_probes(*funcs);
+	*tot_probes = nr_probes + 1;
 	return old;
 }
 
 static void *func_remove(struct tracepoint_func **funcs,
-		struct tracepoint_func *tp_func)
+		struct tracepoint_func *tp_func, int *left)
 {
 	int nr_probes = 0, nr_del = 0, i;
 	struct tracepoint_func *old, *new;
@@ -228,6 +229,7 @@ static int tracepoint_add_func(struct tr
 			       struct tracepoint_func *func, int prio)
 {
 	struct tracepoint_func *old, *tp_funcs;
+	int probes = 0;
 	int ret;
 
 	if (tp->regfunc && !static_key_enabled(&tp->key)) {
@@ -238,7 +240,7 @@ static int tracepoint_add_func(struct tr
 
 	tp_funcs = rcu_dereference_protected(tp->funcs,
 			lockdep_is_held(&tracepoints_mutex));
-	old = func_add(&tp_funcs, func, prio);
+	old = func_add(&tp_funcs, func, prio, &probes);
 	if (IS_ERR(old)) {
 		WARN_ON_ONCE(PTR_ERR(old) != -ENOMEM);
 		return PTR_ERR(old);
@@ -253,6 +255,13 @@ static int tracepoint_add_func(struct tr
 	rcu_assign_pointer(tp->funcs, tp_funcs);
 	if (!static_key_enabled(&tp->key))
 		static_key_slow_inc(&tp->key);
+
+	if (probes == 1) {
+		__static_call_update(tp->static_call_key, tp_funcs->func);
+	} else {
+		__static_call_update(tp->static_call_key, tp->iterator);
+	}
+
 	release_probes(old);
 	return 0;
 }
@@ -267,10 +276,11 @@ static int tracepoint_remove_func(struct
 		struct tracepoint_func *func)
 {
 	struct tracepoint_func *old, *tp_funcs;
+	int probes_left = 0;
 
 	tp_funcs = rcu_dereference_protected(tp->funcs,
 			lockdep_is_held(&tracepoints_mutex));
-	old = func_remove(&tp_funcs, func);
+	old = func_remove(&tp_funcs, func, &probes_left);
 	if (IS_ERR(old)) {
 		WARN_ON_ONCE(PTR_ERR(old) != -ENOMEM);
 		return PTR_ERR(old);
@@ -284,6 +294,13 @@ static int tracepoint_remove_func(struct
 		if (static_key_enabled(&tp->key))
 			static_key_slow_dec(&tp->key);
 	}
+
+	if (probes_left == 1) {
+		__static_call_update(tp->static_call_key, tp_funcs->func);
+	} else {
+		__static_call_update(tp->static_call_key, tp->iterator);
+	}
+
 	rcu_assign_pointer(tp->funcs, tp_funcs);
 	release_probes(old);
 	return 0;