x86 cleanups and static_call()
Subsystems (12)
Peter Zijlstra
maintainercover x86 cleanups and static_call()
Hi!
Now that all the x86_64 int3_emulate_call() stuff is upstream, here are the i386 patches and all the rest of the cleanups that resulted from that discussion.
And I figured I should have a go at making that static_call() thing working now that we have the prerequisites sorted. This is mostly a combination of the v2 and v3 static_call() patches done on top of an 'enhanced' text_poke_bp().
I wrote a little self-test and added Steve's ftrace convertion on top for testing, and suspicously I've not had it explode, so I'm sure this is going to set all your computers on fire.
Enjoy!
Peter Zijlstra
maintainerPATCH 1/15 x86/entry/32: Clean up return from interrupt preemption path
The code flow around the return from interrupt preemption point seems needlesly complicated.
There is only one site jumping to resume_kernel, and none (outside of resume_kernel) jumping to restore_all_kernel. Inline resume_kernel in restore_all_kernel and avoid the CONFIG_PREEMPT dependent label.
--- arch/x86/entry/entry_32.S | 24 ++++++++++-------------- 1 file changed, 10 insertions(+), 14 deletions(-)
arch/x86/entry/entry_32.S +10 -14
@@ -67,7 +67,6 @@ # define preempt_stop(clobbers) DISABLE_INTERRUPTS(clobbers); TRACE_IRQS_OFF #else # define preempt_stop(clobbers) -# define resume_kernel restore_all_kernel #endif .macro TRACE_IRQS_IRET
@@ -755,7 +754,7 @@ END(ret_from_fork) andl $SEGMENT_RPL_MASK, %eax #endif cmpl $USER_RPL, %eax - jb resume_kernel # not returning to v8086 or userspace + jb restore_all_kernel # not returning to v8086 or userspace ENTRY(resume_userspace) DISABLE_INTERRUPTS(CLBR_ANY)
@@ -765,18 +764,6 @@ ENTRY(resume_userspace) jmp restore_all END(ret_from_exception) -#ifdef CONFIG_PREEMPT -ENTRY(resume_kernel) - DISABLE_INTERRUPTS(CLBR_ANY) - cmpl $0, PER_CPU_VAR(__preempt_count) - jnz restore_all_kernel - testl $X86_EFLAGS_IF, PT_EFLAGS(%esp) # interrupts off (exception path) ? - jz restore_all_kernel - call preempt_schedule_irq - jmp restore_all_kernel -END(resume_kernel) -#endif - GLOBAL(__begin_SYSENTER_singlestep_region) /* * All code from here through __end_SYSENTER_singlestep_region is subject
@@ -1027,6 +1014,15 @@ ENTRY(entry_INT80_32) INTERRUPT_RETURN restore_all_kernel: +#ifdef CONFIG_PREEMPT + DISABLE_INTERRUPTS(CLBR_ANY) + cmpl $0, PER_CPU_VAR(__preempt_count) + jnz .Lno_preempt + testl $X86_EFLAGS_IF, PT_EFLAGS(%esp) # interrupts off (exception path) ? + jz .Lno_preempt + call preempt_schedule_irq +.Lno_preempt: +#endif TRACE_IRQS_IRET PARANOID_EXIT_TO_KERNEL_MODE BUG_IF_WRONG_CR3
Re: [PATCH 01/15] x86/entry/32: Clean up return from interrupt preemption path
Quoted 2 lines · Peter Zijlstra
The code flow around the return from interrupt preemption point seems needlesly complicated.
"needlessly"
Quoted 6 lines
There is only one site jumping to
resume_kernel, and none (outside ofresume_kernel) jumping torestore_all_kernel. Inlineresume_kernelinrestore_all_kerneland avoid theCONFIG_PREEMPTdependent label.Signed-off-by Peter Zijlstra (Intel) <[email protected]>
Signature · 1 line
Josh
Peter Zijlstra
maintainerPATCH 2/15 x86: Move ENCODE_FRAME_POINTER to asm/frame.h
In reply to Peter ZijlstraIn preparation for wider use, move the ENCODE_FRAME_POINTER macros to a common header and provide inline asm versions.
These macros are used to encode a pt_regs frame for the unwinder; see unwind_frame.c:decode_frame_pointer().
--- arch/x86/entry/calling.h | 15 ------------- arch/x86/entry/entry_32.S | 16 -------------- arch/x86/include/asm/frame.h | 49 +++++++++++++++++++++++++++++++++++++++++++ 3 files changed, 49 insertions(+), 31 deletions(-)
arch/x86/entry/calling.h +0 -15
@@ -172,21 +172,6 @@ For 32-bit we have the following convent .endif .endm -/* -* This is a sneaky trick to help the unwinder find pt_regs on the stack. The -* frame pointer is replaced with an encoded pointer to pt_regs. The encoding -* is just setting the LSB, which makes it an invalid stack address and is also -* a signal to the unwinder that it's a pt_regs pointer in disguise. -* -* NOTE: This macro must be used *after* PUSH_AND_CLEAR_REGS because it corrupts -* the original rbp. -*/ -.macro ENCODE_FRAME_POINTER ptregs_offset=0 -#ifdef CONFIG_FRAME_POINTER - leaq 1+\ptregs_offset(%rsp), %rbp -#endif -.endm -#ifdef CONFIG_PAGE_TABLE_ISOLATION /*
arch/x86/entry/entry_32.S +0 -16
@@ -246,22 +246,6 @@ .Lend_\@: .endm -/* - * This is a sneaky trick to help the unwinder find pt_regs on the stack. The - * frame pointer is replaced with an encoded pointer to pt_regs. The encoding - * is just clearing the MSB, which makes it an invalid stack address and is also - * a signal to the unwinder that it's a pt_regs pointer in disguise. - * - * NOTE: This macro must be used *after* SAVE_ALL because it corrupts the - * original rbp. - */ -.macro ENCODE_FRAME_POINTER -#ifdef CONFIG_FRAME_POINTER - mov %esp, %ebp - andl $0x7fffffff, %ebp -#endif -.endm - .macro RESTORE_INT_REGS popl %ebx popl %ecx
arch/x86/include/asm/frame.h +49 -0
67 lines not on this page · raw message
Re: [PATCH 02/15] x86: Move ENCODE_FRAME_POINTER to asm/frame.h
Quoted 7 lines · Peter Zijlstra
In preparation for wider use, move the
ENCODE_FRAME_POINTERmacros to a common header and provide inline asm versions.These macros are used to encode a
pt_regsframe for the unwinder; see unwind_frame.c:decode_frame_pointer().Signed-off-by Peter Zijlstra (Intel) <[email protected]>
Signature · 1 line
Josh
Peter Zijlstra
maintainerPATCH 3/15 x86/kprobes: Fix frame pointer annotations
In reply to Peter ZijlstraThe kprobe trampolines have a FRAME_POINTER annotation that makes no sense. It marks the frame in the middle of pt_regs, at the place of saving BP.
Change it to mark the pt_regs frame as per the ENCODE_FRAME_POINTER from the respective entry_*.S.
--- arch/x86/kernel/kprobes/common.h | 24 ++++++++++++------------ 1 file changed, 12 insertions(+), 12 deletions(-)
arch/x86/kernel/kprobes/common.h +12 -12
@@ -5,15 +5,10 @@/* Kprobes and Optprobes common header */ #include <asm/asm.h> - -#ifdef CONFIG_FRAME_POINTER -# define SAVE_RBP_STRING " push %" _ASM_BP "\n" \ - " mov %" _ASM_SP ", %" _ASM_BP "\n" -#else -# define SAVE_RBP_STRING " push %" _ASM_BP "\n" -#endif +#include <asm/frame.h> #ifdef CONFIG_X86_64 +#define SAVE_REGS_STRING \/* Skip cs, ip, orig_ax. */ \ " subq $24, %rsp\n" \
@@ -27,11 +22,13 @@" pushq %r10\n" \" pushq %r11\n" \" pushq %rbx\n" \ -SAVE_RBP_STRING \ +" pushq %rbp\n" \" pushq %r12\n" \" pushq %r13\n" \" pushq %r14\n" \ -" pushq %r15\n" +" pushq %r15\n" \ +ENCODE_FRAME_POINTER + #define RESTORE_REGS_STRING \" popq %r15\n" \" popq %r14\n" \
@@ -51,19 +48,22 @@/* Skip orig_ax, ip, cs */ \ " addq $24, %rsp\n"#else +#define SAVE_REGS_STRING \/* Skip cs, ip, orig_ax and gs. */ \ - " subl $16, %esp\n" \ + " subl $4*4, %esp\n" \ " pushl %fs\n" \ " pushl %es\n" \ " pushl %ds\n" \ " pushl %eax\n" \ - SAVE_RBP_STRING \ + " pushl %ebp\n" \ " pushl %edi\n" \ " pushl %esi\n" \ " pushl %edx\n" \ " pushl %ecx\n" \ - " pushl %ebx\n" + " pushl %ebx\n" \ + ENCODE_FRAME_POINTER +#define RESTORE_REGS_STRING \ " popl %ebx\n" \ " popl %ecx\n" \
Masami Hiramatsu
maintainerRe: [PATCH 03/15] x86/kprobes: Fix frame pointer annotations
Quoted 3 lines · On Wed, 05 Jun 2019 15:07:56 +0200 Peter Zijlstra <[email protected]> wrote:
The kprobe trampolines have a
FRAME_POINTERannotation that makes no sense. It marks the frame in the middle ofpt_regs, at the place of saving BP.
commit ee213fc72fd67 introduced this code, and this is for unwinder which uses frame pointer. I think current code stores the address of previous (original context's) frame pointer into %rbp. So with that, if unwinder tries to decode frame pointer, it can get the original %rbp value, instead of &pt_regs from current %rbp.
Quoted 4 lines
Change it to mark the
pt_regsframe as per theENCODE_FRAME_POINTERfrom the respective entry_*.S.
With this change, I think stack unwinder can not get the original %rbp value. Peter, could you check the above commit?
Thank you,
Quoted 69 lines
Signed-off-by Peter Zijlstra (Intel) <[email protected]>--- arch/x86/kernel/kprobes/common.h | 24 ++++++++++++------------ 1 file changed, 12 insertions(+), 12 deletions(-)arch/x86/kernel/kprobes/common.h +12
-12@@ -5,15 +5,10 @@/* Kprobes and Optprobes common header */ #include <asm/asm.h> - -#ifdef CONFIG_FRAME_POINTER -# define SAVE_RBP_STRING " push %" _ASM_BP "\n" \ - " mov %" _ASM_SP ", %" _ASM_BP "\n" -#else -# define SAVE_RBP_STRING " push %" _ASM_BP "\n" -#endif +#include <asm/frame.h> #ifdef CONFIG_X86_64 +#define SAVE_REGS_STRING \/* Skip cs, ip, orig_ax. */ \ " subq $24, %rsp\n" \@@ -27,11 +22,13 @@" pushq %r10\n" \" pushq %r11\n" \" pushq %rbx\n" \ -SAVE_RBP_STRING \ +" pushq %rbp\n" \" pushq %r12\n" \" pushq %r13\n" \" pushq %r14\n" \ -" pushq %r15\n" +" pushq %r15\n" \ +ENCODE_FRAME_POINTER + #define RESTORE_REGS_STRING \" popq %r15\n" \" popq %r14\n" \@@ -51,19 +48,22 @@/* Skip orig_ax, ip, cs */ \ " addq $24, %rsp\n"#else +#define SAVE_REGS_STRING \/* Skip cs, ip, orig_ax and gs. */ \ - " subl $16, %esp\n" \ + " subl $4*4, %esp\n" \ " pushl %fs\n" \ " pushl %es\n" \ " pushl %ds\n" \ " pushl %eax\n" \ - SAVE_RBP_STRING \ + " pushl %ebp\n" \ " pushl %edi\n" \ " pushl %esi\n" \ " pushl %edx\n" \ " pushl %ecx\n" \ - " pushl %ebx\n" + " pushl %ebx\n" \ + ENCODE_FRAME_POINTER +#define RESTORE_REGS_STRING \ " popl %ebx\n" \ " popl %ecx\n" \
Signature · 1 line
Masami Hiramatsu <[email protected]>
Josh Poimboeuf
Reviewed-byRe: [PATCH 03/15] x86/kprobes: Fix frame pointer annotations
Quoted 14 lines · Masami Hiramatsu
Quoted 3 lines · On Wed, 05 Jun 2019 15:07:56 +0200 Peter Zijlstra <[email protected]> wrote:
The kprobe trampolines have a
FRAME_POINTERannotation that makes no sense. It marks the frame in the middle ofpt_regs, at the place of saving BP.commit
ee213fc72fd67introduced this code, and this is for unwinder which uses frame pointer. I think current code stores the address of previous (original context's) frame pointer into %rbp. So with that, if unwinder tries to decode frame pointer, it can get the original %rbp value, instead of&pt_regsfrom current %rbp.Quoted 1 line
Quoted 6 lines
Quoted 3 lines
Change it to mark the
pt_regsframe as per theENCODE_FRAME_POINTERfrom the respective entry_*.S.With this change, I think stack unwinder can not get the original %rbp value. Peter, could you check the above commit?
The unwinder knows how to decode the encoded frame pointer. So it can find regs by decoding the new rbp value, and it also knows that regs->bp is the original rbp value.
Signature · 1 line
Josh
Re: [PATCH 03/15] x86/kprobes: Fix frame pointer annotations
Quoted 19 lines · On Fri, 7 Jun 2019 09:36:02 -0400 Josh Poimboeuf <[email protected]> wrote:
Quoted 18 lines · Masami Hiramatsu
Quoted 3 lines · On Wed, 05 Jun 2019 15:07:56 +0200 Peter Zijlstra <[email protected]> wrote:
The kprobe trampolines have a
FRAME_POINTERannotation that makes no sense. It marks the frame in the middle ofpt_regs, at the place of saving BP.commit
ee213fc72fd67introduced this code, and this is for unwinder which uses frame pointer. I think current code stores the address of previous (original context's) frame pointer into %rbp. So with that, if unwinder tries to decode frame pointer, it can get the original %rbp value, instead of&pt_regsfrom current %rbp.Quoted 4 lines
Change it to mark the
pt_regsframe as per theENCODE_FRAME_POINTERfrom the respective entry_*.S.
Quoted 9 lines
Quoted 2 lines
With this change, I think stack unwinder can not get the original %rbp value. Peter, could you check the above commit?
The unwinder knows how to decode the encoded frame pointer. So it can find regs by decoding the new rbp value, and it also knows that
regs->bpis the original rbp value.Reviewed-by Josh Poimboeuf <[email protected]>
Ah, OK. My misunderstood. So this encode framepointer as same as other interrupt entry stack.
Then, it looks good to me too.
Thank you Josh!
Quoted 2 lines
Signature · 1 line
Josh
Signature · 1 line
Masami Hiramatsu <[email protected]>
Peter Zijlstra
maintainerRe: [PATCH 03/15] x86/kprobes: Fix frame pointer annotations
In reply to Josh PoimboeufQuoted 8 lines · Josh Poimboeuf
Quoted 7 lines · Masami Hiramatsu
Quoted 3 lines · On Wed, 05 Jun 2019 15:07:56 +0200 Peter Zijlstra <[email protected]> wrote:
The kprobe trampolines have a
FRAME_POINTERannotation that makes no sense. It marks the frame in the middle ofpt_regs, at the place of saving BP.
Quoted 5 lines
Quoted 5 lines
commit
ee213fc72fd67introduced this code, and this is for unwinder which uses frame pointer. I think current code stores the address of previous (original context's) frame pointer into %rbp. So with that, if unwinder tries to decode frame pointer, it can get the original %rbp value, instead of&pt_regsfrom current %rbp.
The way I read that code is that we'll put the value of SP into BP at the point where we've done 'PUSH BP', which is right in the middle of that PUSH sequence. So while it works for a FP based unwinder, it doesn't 'properly' identify the current frame.
Quoted 10 lines
Quoted 6 lines
Quoted 3 lines
Change it to mark the
pt_regsframe as per theENCODE_FRAME_POINTERfrom the respective entry_*.S.With this change, I think stack unwinder can not get the original %rbp value. Peter, could you check the above commit?
The unwinder knows how to decode the encoded frame pointer. So it can find regs by decoding the new rbp value, and it also knows that
regs->bpis the original rbp value.
Right, as Josh says the unwinder has a special case for this and it knows these 'odd' BP values (either MSB or LSB set) indicate a pt_regs set.
Peter Zijlstra
maintainerPATCH 4/15 x86/ftrace: Add pt_regs frame annotations
In reply to Peter ZijlstraWhen CONFIG_FRAME_POINTER, we should mark pt_regs frames.
--- arch/x86/kernel/ftrace_32.S | 3 +++ arch/x86/kernel/ftrace_64.S | 3 +++ 2 files changed, 6 insertions(+)
arch/x86/kernel/ftrace_32.S +3 -0
@@ -9,6 +9,7 @@ #include <asm/export.h> #include <asm/ftrace.h> #include <asm/nospec-branch.h> +#include <asm/frame.h> # define function_hook __fentry__ EXPORT_SYMBOL(__fentry__)
@@ -116,6 +117,8 @@ ENTRY(ftrace_regs_caller) pushl %ecx pushl %ebx + ENCODE_FRAME_POINTER + movl 12*4(%esp), %eax /* Load ip (1st parameter) */ subl $MCOUNT_INSN_SIZE, %eax /* Adjust ip */ movl 15*4(%esp), %edx /* Load parent ip (2nd parameter) */
arch/x86/kernel/ftrace_64.S +3 -0
@@ -9,6 +9,7 @@ #include <asm/export.h> #include <asm/nospec-branch.h> #include <asm/unwind_hints.h> +#include <asm/frame.h> .code64 .section .entry.text, "ax"
@@ -203,6 +204,8 @@ GLOBAL(ftrace_regs_caller_op_ptr) leaq MCOUNT_REG_SIZE+8*2(%rsp), %rcx movq %rcx, RSP(%rsp) + ENCODE_FRAME_POINTER + /* regs go into 4th parameter */ leaq (%rsp), %rcx
Re: [PATCH 04/15] x86/ftrace: Add pt_regs frame annotations
Quoted 3 lines · Peter Zijlstra
When
CONFIG_FRAME_POINTER, we should markpt_regsframes.Signed-off-by Peter Zijlstra (Intel) <[email protected]>
Signature · 1 line
Josh
Peter Zijlstra
maintainerPATCH 5/15 x86_32: Provide consistent pt_regs
In reply to Peter ZijlstraCurrently pt_regs on x86_32 has an oddity in that kernel regs (!user_mode(regs)) are short two entries (esp/ss). This means that any code trying to use them (typically: regs->sp) needs to jump through some unfortunate hoops.
Change the entry code to fix this up and create a full pt_regs frame.
This then simplifies various trampolines in ftrace and kprobes, the stack unwinder, ptrace, kdump and kgdb.
--- arch/x86/entry/entry_32.S | 105 ++++++++++++++++++++++++++++++++++---- arch/x86/include/asm/kexec.h | 17 ------ arch/x86/include/asm/ptrace.h | 17 ------ arch/x86/include/asm/stacktrace.h | 2 arch/x86/kernel/crash.c | 8 -- arch/x86/kernel/ftrace_32.S | 77 +++++++++++++++------------ arch/x86/kernel/kgdb.c | 8 -- arch/x86/kernel/kprobes/common.h | 4 - arch/x86/kernel/kprobes/core.c | 29 ++++------ arch/x86/kernel/kprobes/opt.c | 20 ++++--- arch/x86/kernel/process_32.c | 16 +---- arch/x86/kernel/ptrace.c | 29 ---------- arch/x86/kernel/time.c | 3 - arch/x86/kernel/unwind_frame.c | 32 +---------- arch/x86/kernel/unwind_orc.c | 2 15 files changed, 178 insertions(+), 191 deletions(-)
arch/x86/entry/entry_32.S +95 -10
@@ -202,9 +202,102 @@ .Lend_\@: .endm +#define CS_FROM_ENTRY_STACK (1 << 31) +#define CS_FROM_USER_CR3 (1 << 30) +#define CS_FROM_KERNEL (1 << 29) + +.macro FIXUP_FRAME + /* + * The high bits of the CS dword (__csh) are used for CS_FROM_*. + * Clear them in case hardware didn't do this for us. + */ + andl $0x0000ffff, 3*4(%esp) + +#ifdef CONFIG_VM86 + testl $X86_EFLAGS_VM, 4*4(%esp) + jnz .Lfrom_usermode_no_fixup_\@ +#endif + testl $SEGMENT_RPL_MASK, 3*4(%esp) + jnz .Lfrom_usermode_no_fixup_\@ + + orl $CS_FROM_KERNEL, 3*4(%esp) + + /* + * When we're here from kernel mode; the (exception) stack looks like: + * + * 5*4(%esp) - <previous context> + * 4*4(%esp) - flags + * 3*4(%esp) - cs + * 2*4(%esp) - ip + * 1*4(%esp) - orig_eax + * 0*4(%esp) - gs / function + * + * Lets build a 5 entry IRET frame after that, such that struct pt_regs + * is complete and in particular regs->sp is correct. This gives us + * the original 5 enties as gap: + * + * 12*4(%esp) - <previous context> + * 11*4(%esp) - gap / flags + * 10*4(%esp) - gap / cs + * 9*4(%esp) - gap / ip + * 8*4(%esp) - gap / orig_eax + * 7*4(%esp) - gap / gs / function + * 6*4(%esp) - ss + * 5*4(%esp) - sp + * 4*4(%esp) - flags + * 3*4(%esp) - cs + * 2*4(%esp) - ip + * 1*4(%esp) - orig_eax + * 0*4(%esp) - gs / function + */ + + pushl %ss # ss + pushl %esp # sp (points at ss) + addl $6*4, (%esp) # point sp back at the previous context + pushl 6*4(%esp) # flags + pushl 6*4(%esp) # cs + pushl 6*4(%esp) # ip + pushl 6*4(%esp) # orig_eax + pushl 6*4(%esp) # gs / function +.Lfrom_usermode_no_fixup_\@: +.endm + +.macro IRET_FRAME + testl $CS_FROM_KERNEL, 1*4(%esp) + jz .Lfinished_frame_\@ + + /* + * Reconstruct the 3 entry IRET frame right after the (modified) + * regs->sp without lowering %esp in between, such that an NMI in the + * middle doesn't scribble our stack. + */ + pushl %eax + pushl %ecx + movl 5*4(%esp), %eax # (modified) regs->sp + + movl 4*4(%esp), %ecx # flags + movl %ecx, -4(%eax) + + movl 3*4(%esp), %ecx # cs + andl $0x0000ffff, %ecx + movl %ecx, -8(%eax) + + movl 2*4(%esp), %ecx # ip + movl %ecx, -12(%eax) + + movl 1*4(%esp), %ecx # eax + movl %ecx, -16(%eax) + + popl %ecx + lea -16(%eax), %esp + popl %eax +.Lfinished_frame_\@: +.endm + .macro SAVE_ALL pt_regs_ax=%eax switch_stacks=0 cld PUSH_GS + FIXUP_FRAME pushl %fs pushl %es pushl %ds
@@ -358,9 +451,6 @@ * switch to it before we do any copying. */ -#define CS_FROM_ENTRY_STACK (1 << 31) -#define CS_FROM_USER_CR3 (1 << 30) - .macro SWITCH_TO_KERNEL_STACK ALTERNATIVE "", "jmp .Lend_\@", X86_FEATURE_XENPV
@@ -374,13 +464,6 @@ * that register for the time this macro runs */ - /* - * The high bits of the CS dword (__csh) are used for - * CS_FROM_ENTRY_STACK and CS_FROM_USER_CR3. Clear them in case - * hardware didn't do this for us. - */ - andl $(0x0000ffff), PT_CS(%esp) - /* Are we on the entry stack? Bail out if not! */ movl PER_CPU_VAR(cpu_entry_area), %ecx addl $CPU_ENTRY_AREA_entry_stack + SIZEOF_entry_stack, %ecx
@@ -990,6 +1073,7 @@ ENTRY(entry_INT80_32) /* Restore user state */ RESTORE_REGS pop=4 # skip orig_eax/error_code .Lirq_return: + IRET_FRAME /* * ARCH_HAS_MEMBARRIER_SYNC_CORE rely on IRET core serialization * when returning from IPI handler and when returning from
@@ -1340,6 +1424,7 @@ END(page_fault) common_exception: /* the function address is in %gs's slot on the stack */ + FIXUP_FRAME pushl %fs pushl %es pushl %ds
arch/x86/include/asm/kexec.h +0 -17
@@ -71,22 +71,6 @@ struct kimage;#define KEXEC_BACKUP_SRC_END (640 * 1024UL - 1)/* 640K */ /* -* CPU does not save ss and sp on stack if execution is already -* running in kernel mode at the time of NMI occurrence. This code -* fixes it. -*/ -static inline void crash_fixup_ss_esp (struct pt_regs *newregs, -struct pt_regs *oldregs) -{ -#ifdef CONFIG_X86_32 - newregs->sp = (unsigned long )&(oldregs->sp); - asmvolatile ("xorl %%eax, %%eax\n\t" -"movw %%ss, %%ax\n\t" - :"=a" (newregs->ss)); -#endif -} - -/* * This function is responsible for capturing register states if coming * via panic otherwise just fix up the ss and sp if coming via kernel * mode exception.
@@ -96,7 +80,6 @@ static inline void crash_setup_regs(stru {if (oldregs) {memcpy (newregs, oldregs,sizeof (*newregs)); -crash_fixup_ss_esp (newregs, oldregs); }else {#ifdef CONFIG_X86_32 asmvolatile ("movl %%ebx,%0" :"=m" (newregs->bx));
arch/x86/include/asm/ptrace.h +2 -15
@@ -166,14 +166,10 @@ static inline bool user_64bit_mode(struc#define compat_user_stack_pointer () current_pt_regs()->sp#endif -#ifdef CONFIG_X86_32 -extern unsigned long kernel_stack_pointer (struct pt_regs *regs); -#else static inline unsigned long kernel_stack_pointer (struct pt_regs *regs) {return regs->sp; } -#endif #define GET_IP (regs) ((regs)->ip)#define GET_FP (regs) ((regs)->bp)
@@ -201,14 +197,6 @@ static inline unsigned long regs_get_regif (unlikely (offset >MAX_REG_OFFSET ))return 0 ;#ifdef CONFIG_X86_32 -/* -* Traps from the kernel do not save sp and ss. -* Use the helper function to retrieve sp. -*/ -if (offset == offsetof(struct pt_regs , sp) && - regs->cs == __KERNEL_CS) -return kernel_stack_pointer (regs); -/* The selector fields are 16-bit. */ if (offset == offsetof(struct pt_regs, cs) || offset == offsetof(struct pt_regs, ss) ||
@@ -234,8 +222,7 @@ static inline unsigned long regs_get_regstatic inline int regs_within_kernel_stack (struct pt_regs *regs,unsigned long addr) { -return ((addr & ~(THREAD_SIZE -1 )) == - (kernel_stack_pointer (regs) & ~(THREAD_SIZE -1 ))); +return ((addr & ~(THREAD_SIZE -1 )) == (regs->sp & ~(THREAD_SIZE -1 ))); }/**
@@ -249,7 +236,7 @@ static inline int regs_within_kernel_sta*/ static inline unsigned long *regs_get_kernel_stack_nth_addr (struct pt_regs *regs,unsigned int n) { -unsigned long *addr = (unsigned long *)kernel_stack_pointer (regs); +unsigned long *addr = (unsigned long *)regs->sp; addr += n;if (regs_within_kernel_stack (regs, (unsigned long )addr))
arch/x86/include/asm/stacktrace.h +1 -1
@@ -78,7 +78,7 @@ static inline unsigned long * get_stack_pointer(struct task_struct *task, struct pt_regs *regs) { if (regs) - return (unsigned long *)kernel_stack_pointer(regs); + return (unsigned long *)regs->sp; if (task == current) return __builtin_frame_address(0);
arch/x86/kernel/crash.c +0 -8
@@ -72,14 +72,6 @@ static inline void cpu_crash_vmclear_loastatic void kdump_nmi_callback (int cpu,struct pt_regs *regs) { -#ifdef CONFIG_X86_32 -struct pt_regs fixed_regs; - -if (!user_mode (regs)) { -crash_fixup_ss_esp (&fixed_regs, regs); - regs = &fixed_regs; - } -#endif crash_save_cpu (regs, cpu);/*
arch/x86/kernel/ftrace_32.S +43 -34
113 lines not on this page · raw message
arch/x86/kernel/kgdb.c +0 -8
@@ -127,14 +127,6 @@ char *dbg_get_reg(int regno, void *mem,#ifdef CONFIG_X86_32 switch (regno) { -case GDB_SS : -if (!user_mode (regs)) - *(unsigned long *)mem = __KERNEL_DS; -break ; -case GDB_SP : -if (!user_mode (regs)) - *(unsigned long *)mem =kernel_stack_pointer (regs); -break ;case GDB_GS :case GDB_FS : *(unsigned long *)mem =0xFFFF ;
arch/x86/kernel/kprobes/common.h +2 -2
@@ -72,8 +72,8 @@" popl %edi\n" \" popl %ebp\n" \" popl %eax\n" \ -/* Skip ds, es, fs, gs, orig_ax, and ip. Note: don't pop cs here*/ \ - " addl $24, %esp\n" +/* Skip ds, es, fs, gs, orig_ax, ip, and cs. */ \ + " addl $7*4, %esp\n"#endif /* Ensure if the instruction can be boostable */
arch/x86/kernel/kprobes/core.c +12 -17
@@ -69,7 +69,7 @@ DEFINE_PER_CPU(struct kprobe *, current_kprobe) = NULL; DEFINE_PER_CPU(struct kprobe_ctlblk, kprobe_ctlblk); -#define stack_addr (regs) ((unsigned long *)kernel_stack_pointer(regs)) +#define stack_addr (regs) ((unsigned long *)regs->sp)#define W (row, b0, b1, b2, b3, b4, b5, b6, b7, b8, b9, ba, bb, bc, bd, be, bf)\ (((b0##UL << 0x0)|(b1##UL << 0x1)|(b2##UL << 0x2)|(b3##UL << 0x3) | \
@@ -731,29 +731,27 @@ asm(".global kretprobe_trampoline\n" ".type kretprobe_trampoline, @function\n" "kretprobe_trampoline:\n" -#ifdef CONFIG_X86_64/* We don't bother saving the ss register */ +#ifdef CONFIG_X86_64" pushq %rsp\n" " pushfq\n" SAVE_REGS_STRING " movq %rsp, %rdi\n" " call trampoline_handler\n" /* Replace saved sp with true return address. */ - " movq %rax, 152(%rsp)\n" +" movq %rax, 19*8(%rsp)\n" RESTORE_REGS_STRING " popfq\n" #else - " pushf\n" + " pushl %esp\n" + " pushfl\n"SAVE_REGS_STRING " movl %esp, %eax\n" " call trampoline_handler\n" - /* Move flags to cs */ - " movl 56(%esp), %edx\n" - " movl %edx, 52(%esp)\n" - /* Replace saved flags with true return address. */ - " movl %eax, 56(%esp)\n" +/* Replace saved sp with true return address. */ +" movl %eax, 15*4(%esp)\n" RESTORE_REGS_STRING - " popf\n" +" popfl\n" #endif " ret\n" ".size kretprobe_trampoline, .-kretprobe_trampoline\n"
@@ -794,16 +792,13 @@ __used __visible void *trampoline_handleINIT_HLIST_HEAD (&empty_rp);kretprobe_hash_lock (current, &head, &flags);/* fixup registers */ -#ifdef CONFIG_X86_64 regs->cs = __KERNEL_CS; -/* On x86-64, we use pt_regs->sp for return address holder. */ - frame_pointer = ®s->sp; -#else - regs->cs = __KERNEL_CS |get_kernel_rpl (); +#ifdef CONFIG_X86_32 + regs->cs |=get_kernel_rpl (); regs->gs =0 ; -/* On x86-32, we use pt_regs->flags for return address holder. */ - frame_pointer = ®s->flags;#endif +/* We use pt_regs->sp for return address holder. */ + frame_pointer = ®s->sp; regs->ip = trampoline_address; regs->orig_ax = ~0UL ;
arch/x86/kernel/kprobes/opt.c +12 -8
@@ -115,14 +115,15 @@ asm ("optprobe_template_call:\n" ASM_NOP5 /* Move flags to rsp */ -" movq 144(%rsp), %rdx\n" -" movq %rdx, 152(%rsp)\n" +" movq 18*8(%rsp), %rdx\n" +" movq %rdx, 19*8(%rsp)\n" RESTORE_REGS_STRING /* Skip flags entry */ " addq $8, %rsp\n" " popfq\n"#else /* CONFIG_X86_32 */ -" pushf\n" +" pushl %esp\n" +" pushfl\n" SAVE_REGS_STRING " movl %esp, %edx\n" ".global optprobe_template_val\n"
@@ -131,9 +132,13 @@ asm (".global optprobe_template_call\n" "optprobe_template_call:\n" ASM_NOP5 +/* Move flags into esp */ +" movl 14*4(%esp), %edx\n" +" movl %edx, 15*4(%esp)\n" RESTORE_REGS_STRING -" addl $4, %esp\n" /* skip cs */ - " popf\n" +/* Skip flags entry */ +" addl $4, %esp\n" + " popfl\n"#endif ".global optprobe_template_end\n" "optprobe_template_end:\n"
@@ -165,10 +170,9 @@ optimized_callback(struct optimized_kpro }else {struct kprobe_ctlblk *kcb =get_kprobe_ctlblk ();/* Save skipped registers */ -#ifdef CONFIG_X86_64 regs->cs = __KERNEL_CS; -#else - regs->cs = __KERNEL_CS |get_kernel_rpl (); +#ifdef CONFIG_X86_32 + regs->cs |=get_kernel_rpl (); regs->gs =0 ;#endif regs->ip = (unsigned long )op->kp.addr +INT3_SIZE ;
arch/x86/kernel/process_32.c +5 -11
@@ -62,27 +62,21 @@ void __show_regs(struct pt_regs *regs, e {unsigned long cr0 =0L , cr2 =0L , cr3 =0L , cr4 =0L ;unsigned long d0, d1, d2, d3, d6, d7; -unsigned long sp; -unsigned short ss, gs; +unsigned short gs; -if (user_mode (regs)) { - sp = regs->sp; - ss = regs->ss; +if (user_mode (regs)) gs =get_user_gs (regs); - }else { - sp =kernel_stack_pointer (regs); -savesegment (ss, ss); +else savesegment (gs, gs); - }show_ip (regs,KERN_DEFAULT );printk (KERN_DEFAULT "EAX: %08lx EBX: %08lx ECX: %08lx EDX: %08lx\n" , regs->ax, regs->bx, regs->cx, regs->dx);printk (KERN_DEFAULT "ESI: %08lx EDI: %08lx EBP: %08lx ESP: %08lx\n" , - regs->si, regs->di, regs->bp, sp); + regs->si, regs->di, regs->bp, regs->sp);printk (KERN_DEFAULT "DS: %04x ES: %04x FS: %04x GS: %04x SS: %04x EFLAGS: %08lx\n" , - (u16 )regs->ds, (u16 )regs->es, (u16 )regs->fs, gs, ss, regs->flags); + (u16 )regs->ds, (u16 )regs->es, (u16 )regs->fs, gs, regs->ss, regs->flags);if (mode !=SHOW_REGS_ALL )return ;
arch/x86/kernel/ptrace.c +0 -29
@@ -153,35 +153,6 @@ static inline bool invalid_selector(u16#define FLAG_MASK FLAG_MASK_32 -/* -* X86_32 CPUs don't save ss and esp if the CPU is already in kernel mode -* when it traps. The previous stack will be directly underneath the saved -* registers, and 'sp/ss' won't even have been saved. Thus the '®s->sp'. -* -* Now, if the stack is empty, '®s->sp' is out of range. In this -* case we try to take the previous stack. To always return a non-null -* stack pointer we fall back to regs as stack if no previous stack -* exists. -* -* This is valid only for kernel mode traps. -*/ -unsigned long kernel_stack_pointer (struct pt_regs *regs) -{ -unsigned long context = (unsigned long )regs & ~(THREAD_SIZE -1 ); -unsigned long sp = (unsigned long )®s->sp; -u32 *prev_esp; - -if (context == (sp & ~(THREAD_SIZE -1 ))) -return sp; - - prev_esp = (u32 *)(context); -if (*prev_esp) -return (unsigned long )*prev_esp; - -return (unsigned long )regs; -} -EXPORT_SYMBOL_GPL (kernel_stack_pointer); -static unsigned long *pt_regs_access (struct pt_regs *regs,unsigned long regno) {BUILD_BUG_ON (offsetof(struct pt_regs , bx) !=0 );
arch/x86/kernel/time.c +1 -2
@@ -37,8 +37,7 @@ unsigned long profile_pc(struct pt_regs#ifdef CONFIG_FRAME_POINTER return *(unsigned long *)(regs->bp +sizeof (long ));#else -unsigned long *sp = - (unsigned long *)kernel_stack_pointer (regs); +unsigned long *sp = (unsigned long *)regs->sp;/* * Return address is either directly at stack pointer * or above a saved flags. Eflags has bits 22-31 zero,
arch/x86/kernel/unwind_frame.c +4 -28
@@ -69,15 +69,6 @@ static void unwind_dump(struct unwind_st } } -static size_t regs_size (struct pt_regs *regs) -{ -/* x86_32 regs from kernel mode are two words shorter: */ -if (IS_ENABLED (CONFIG_X86_32 ) && !user_mode (regs)) -return sizeof (*regs) -2 *sizeof (long ); - -return sizeof (*regs); -} -static bool in_entry_code (unsigned long ip) {char *addr = (char *)ip;
@@ -197,12 +188,6 @@ static struct pt_regs *decode_frame_poin }#endif -#ifdef CONFIG_X86_32 -#define KERNEL_REGS_SIZE (sizeof(struct pt_regs) - 2*sizeof(long)) -#else -#define KERNEL_REGS_SIZE (sizeof(struct pt_regs)) -#endif -static bool update_stack_state (struct unwind_state *state,unsigned long *next_bp) {
@@ -213,7 +198,7 @@ static bool update_stack_state(struct unsize_t len;if (state->regs) - prev_frame_end = (void *)state->regs +regs_size (state->regs); + prev_frame_end = (void *)state->regs +sizeof (*state->regs);else prev_frame_end = (void *)state->bp +FRAME_HEADER_SIZE ;
@@ -221,7 +206,7 @@ static bool update_stack_state(struct un regs =decode_frame_pointer (next_bp);if (regs) { frame = (unsigned long *)regs; - len =KERNEL_REGS_SIZE ; + len =sizeof (*regs); state->got_irq = true; }else { frame = next_bp;
@@ -245,14 +230,6 @@ static bool update_stack_state(struct un frame < prev_frame_end)return false; -/* -* On 32-bit with user mode regs, make sure the last two regs are safe -* to access: -*/ -if (IS_ENABLED (CONFIG_X86_32 ) && regs &&user_mode (regs) && - !on_stack (info, frame, len +2 *sizeof (long ))) -return false; -/* Move state to the next frame: */ if (regs) { state->regs = regs;
@@ -411,10 +388,9 @@ void __unwind_start(struct unwind_state* Pretend that the frame is complete and that BP points to it, but save * the real BP so that we can use it when looking for the next frame. */ -if (regs && regs->ip ==0 && - (unsigned long *)kernel_stack_pointer (regs) >= first_frame) { +if (regs && regs->ip ==0 && (unsigned long *)regs->sp >= first_frame) { state->next_bp = bp; - bp = ((unsigned long *)kernel_stack_pointer (regs)) -1 ; + bp = ((unsigned long *)regs->sp) -1 ; }/* Initialize stack info and make sure the frame data is accessible: */
arch/x86/kernel/unwind_orc.c +1 -1
@@ -579,7 +579,7 @@ void __unwind_start(struct unwind_state goto done; state->ip = regs->ip; - state->sp =kernel_stack_pointer (regs); + state->sp = regs->sp; state->bp = regs->bp; state->regs = regs; state->full_regs = true;
Re: [PATCH 05/15] x86_32: Provide consistent pt_regs
Quoted 9 lines · On Wed, 05 Jun 2019 15:07:58 +0200 Peter Zijlstra <[email protected]> wrote:
Currently
pt_regsonx86_32has an oddity in that kernel regs (!user_mode(regs)) are short two entries (esp/ss). This means that any code trying to use them (typically:regs->sp) needs to jump through some unfortunate hoops.Change the entry code to fix this up and create a full
pt_regsframe.This then simplifies various trampolines in ftrace and kprobes, the stack unwinder, ptrace, kdump and kgdb.
The kprobes parts are looks good to me.
Thank you!
Quoted 720 lines
Signed-off-by Peter Zijlstra (Intel) <[email protected]>--- arch/x86/entry/entry_32.S | 105 ++++++++++++++++++++++++++++++++++---- arch/x86/include/asm/kexec.h | 17 ------ arch/x86/include/asm/ptrace.h | 17 ------ arch/x86/include/asm/stacktrace.h | 2 arch/x86/kernel/crash.c | 8 -- arch/x86/kernel/ftrace_32.S | 77 +++++++++++++++------------ arch/x86/kernel/kgdb.c | 8 -- arch/x86/kernel/kprobes/common.h | 4 - arch/x86/kernel/kprobes/core.c | 29 ++++------ arch/x86/kernel/kprobes/opt.c | 20 ++++--- arch/x86/kernel/process_32.c | 16 +---- arch/x86/kernel/ptrace.c | 29 ---------- arch/x86/kernel/time.c | 3 - arch/x86/kernel/unwind_frame.c | 32 +---------- arch/x86/kernel/unwind_orc.c | 2 15 files changed, 178 insertions(+), 191 deletions(-)arch/x86/entry/entry_32.S +95
-10@@ -202,9 +202,102 @@ .Lend_\@: .endm +#define CS_FROM_ENTRY_STACK (1 << 31) +#define CS_FROM_USER_CR3 (1 << 30) +#define CS_FROM_KERNEL (1 << 29) + +.macro FIXUP_FRAME + /* + * The high bits of the CS dword (__csh) are used for CS_FROM_*. + * Clear them in case hardware didn't do this for us. + */ + andl $0x0000ffff, 3*4(%esp) + +#ifdef CONFIG_VM86 + testl $X86_EFLAGS_VM, 4*4(%esp) + jnz .Lfrom_usermode_no_fixup_\@ +#endif + testl $SEGMENT_RPL_MASK, 3*4(%esp) + jnz .Lfrom_usermode_no_fixup_\@ + + orl $CS_FROM_KERNEL, 3*4(%esp) + + /* + * When we're here from kernel mode; the (exception) stack looks like: + * + * 5*4(%esp) - <previous context> + * 4*4(%esp) - flags + * 3*4(%esp) - cs + * 2*4(%esp) - ip + * 1*4(%esp) - orig_eax + * 0*4(%esp) - gs / function + * + * Lets build a 5 entry IRET frame after that, such that struct pt_regs + * is complete and in particular regs->sp is correct. This gives us + * the original 5 enties as gap: + * + * 12*4(%esp) - <previous context> + * 11*4(%esp) - gap / flags + * 10*4(%esp) - gap / cs + * 9*4(%esp) - gap / ip + * 8*4(%esp) - gap / orig_eax + * 7*4(%esp) - gap / gs / function + * 6*4(%esp) - ss + * 5*4(%esp) - sp + * 4*4(%esp) - flags + * 3*4(%esp) - cs + * 2*4(%esp) - ip + * 1*4(%esp) - orig_eax + * 0*4(%esp) - gs / function + */ + + pushl %ss # ss + pushl %esp # sp (points at ss) + addl $6*4, (%esp) # point sp back at the previous context + pushl 6*4(%esp) # flags + pushl 6*4(%esp) # cs + pushl 6*4(%esp) # ip + pushl 6*4(%esp) # orig_eax + pushl 6*4(%esp) # gs / function +.Lfrom_usermode_no_fixup_\@: +.endm + +.macro IRET_FRAME + testl $CS_FROM_KERNEL, 1*4(%esp) + jz .Lfinished_frame_\@ + + /* + * Reconstruct the 3 entry IRET frame right after the (modified) + * regs->sp without lowering %esp in between, such that an NMI in the + * middle doesn't scribble our stack. + */ + pushl %eax + pushl %ecx + movl 5*4(%esp), %eax # (modified) regs->sp + + movl 4*4(%esp), %ecx # flags + movl %ecx, -4(%eax) + + movl 3*4(%esp), %ecx # cs + andl $0x0000ffff, %ecx + movl %ecx, -8(%eax) + + movl 2*4(%esp), %ecx # ip + movl %ecx, -12(%eax) + + movl 1*4(%esp), %ecx # eax + movl %ecx, -16(%eax) + + popl %ecx + lea -16(%eax), %esp + popl %eax +.Lfinished_frame_\@: +.endm + .macro SAVE_ALL pt_regs_ax=%eax switch_stacks=0 cld PUSH_GS + FIXUP_FRAME pushl %fs pushl %es pushl %ds@@ -358,9 +451,6 @@ * switch to it before we do any copying. */ -#define CS_FROM_ENTRY_STACK (1 << 31) -#define CS_FROM_USER_CR3 (1 << 30) - .macro SWITCH_TO_KERNEL_STACK ALTERNATIVE "", "jmp .Lend_\@", X86_FEATURE_XENPV@@ -374,13 +464,6 @@ * that register for the time this macro runs */ - /* - * The high bits of the CS dword (__csh) are used for - * CS_FROM_ENTRY_STACK and CS_FROM_USER_CR3. Clear them in case - * hardware didn't do this for us. - */ - andl $(0x0000ffff), PT_CS(%esp) - /* Are we on the entry stack? Bail out if not! */ movl PER_CPU_VAR(cpu_entry_area), %ecx addl $CPU_ENTRY_AREA_entry_stack + SIZEOF_entry_stack, %ecx@@ -990,6 +1073,7 @@ ENTRY(entry_INT80_32) /* Restore user state */ RESTORE_REGS pop=4 # skip orig_eax/error_code .Lirq_return: + IRET_FRAME /* * ARCH_HAS_MEMBARRIER_SYNC_CORE rely on IRET core serialization * when returning from IPI handler and when returning from@@ -1340,6 +1424,7 @@ END(page_fault) common_exception: /* the function address is in %gs's slot on the stack */ + FIXUP_FRAME pushl %fs pushl %es pushl %dsarch/x86/include/asm/kexec.h +0
-17@@ -71,22 +71,6 @@ struct kimage;#define KEXEC_BACKUP_SRC_END (640 * 1024UL - 1)/* 640K */ /* -* CPU does not save ss and sp on stack if execution is already -* running in kernel mode at the time of NMI occurrence. This code -* fixes it. -*/ -static inline void crash_fixup_ss_esp (struct pt_regs *newregs, -struct pt_regs *oldregs) -{ -#ifdef CONFIG_X86_32 - newregs->sp = (unsigned long )&(oldregs->sp); - asmvolatile ("xorl %%eax, %%eax\n\t" -"movw %%ss, %%ax\n\t" - :"=a" (newregs->ss)); -#endif -} - -/* * This function is responsible for capturing register states if coming * via panic otherwise just fix up the ss and sp if coming via kernel * mode exception. @@ -96,7 +80,6 @@ static inline void crash_setup_regs(stru {if (oldregs) {memcpy (newregs, oldregs,sizeof (*newregs)); -crash_fixup_ss_esp (newregs, oldregs); }else {#ifdef CONFIG_X86_32 asmvolatile ("movl %%ebx,%0" :"=m" (newregs->bx));arch/x86/include/asm/ptrace.h +2
-15@@ -166,14 +166,10 @@ static inline bool user_64bit_mode(struc#define compat_user_stack_pointer () current_pt_regs()->sp#endif -#ifdef CONFIG_X86_32 -extern unsigned long kernel_stack_pointer (struct pt_regs *regs); -#else static inline unsigned long kernel_stack_pointer (struct pt_regs *regs) {return regs->sp; } -#endif #define GET_IP (regs) ((regs)->ip)#define GET_FP (regs) ((regs)->bp)@@ -201,14 +197,6 @@ static inline unsigned long regs_get_regif (unlikely (offset >MAX_REG_OFFSET ))return 0 ;#ifdef CONFIG_X86_32 -/* -* Traps from the kernel do not save sp and ss. -* Use the helper function to retrieve sp. -*/ -if (offset == offsetof(struct pt_regs , sp) && - regs->cs == __KERNEL_CS) -return kernel_stack_pointer (regs); -/* The selector fields are 16-bit. */ if (offset == offsetof(struct pt_regs, cs) || offset == offsetof(struct pt_regs, ss) ||@@ -234,8 +222,7 @@ static inline unsigned long regs_get_regstatic inline int regs_within_kernel_stack (struct pt_regs *regs,unsigned long addr) { -return ((addr & ~(THREAD_SIZE -1 )) == - (kernel_stack_pointer (regs) & ~(THREAD_SIZE -1 ))); +return ((addr & ~(THREAD_SIZE -1 )) == (regs->sp & ~(THREAD_SIZE -1 ))); }/** @@ -249,7 +236,7 @@ static inline int regs_within_kernel_sta*/ static inline unsigned long *regs_get_kernel_stack_nth_addr (struct pt_regs *regs,unsigned int n) { -unsigned long *addr = (unsigned long *)kernel_stack_pointer (regs); +unsigned long *addr = (unsigned long *)regs->sp; addr += n;if (regs_within_kernel_stack (regs, (unsigned long )addr))arch/x86/include/asm/stacktrace.h +1
-1@@ -78,7 +78,7 @@ static inline unsigned long * get_stack_pointer(struct task_struct *task, struct pt_regs *regs) { if (regs) - return (unsigned long *)kernel_stack_pointer(regs); + return (unsigned long *)regs->sp; if (task == current) return __builtin_frame_address(0);arch/x86/kernel/crash.c +0
-8@@ -72,14 +72,6 @@ static inline void cpu_crash_vmclear_loastatic void kdump_nmi_callback (int cpu,struct pt_regs *regs) { -#ifdef CONFIG_X86_32 -struct pt_regs fixed_regs; - -if (!user_mode (regs)) { -crash_fixup_ss_esp (&fixed_regs, regs); - regs = &fixed_regs; - } -#endif crash_save_cpu (regs, cpu);/* arch/x86/kernel/ftrace_32.S +43
-34@@ -10,6 +10,7 @@ #include <asm/ftrace.h> #include <asm/nospec-branch.h> #include <asm/frame.h> +#include <asm/asm-offsets.h> # define function_hook __fentry__ EXPORT_SYMBOL(__fentry__)@@ -90,26 +91,38 @@ END(ftrace_caller) ENTRY(ftrace_regs_caller) /* - * i386 does not save SS and ESP when coming from kernel. - * Instead, to get sp, ®s->sp is used (see ptrace.h). - * Unfortunately, that means eflags must be at the same location - * as the current return ip is. We move the return ip into the - * regs->ip location, and move flags into the return ip location. + * We're here from an mcount/fentry CALL, and the stack frame looks like: + * + * <previous context> + * RET-IP + * + * The purpose of this function is to call out in an emulated INT3 + * environment with a stack frame like: + * + * <previous context> + * gap / RET-IP + * gap + * gap + * gap + * pt_regs + * + * We do _NOT_ restore: ss, flags, cs, gs, fs, es, ds */ - pushl $__KERNEL_CS - pushl 4(%esp) /* Save the return ip */ - pushl $0 /* Load 0 into orig_ax */ + subl $3*4, %esp # RET-IP + 3 gaps + pushl %ss # ss + pushl %esp # points at ss + addl $5*4, (%esp) # make it point at <previous context> + pushfl # flags + pushl $__KERNEL_CS # cs + pushl 7*4(%esp) # ip <- RET-IP + pushl $0 # orig_eax + pushl %gs pushl %fs pushl %es pushl %ds - pushl %eax - - /* Get flags and place them into the return ip slot */ - pushf - popl %eax - movl %eax, 8*4(%esp) + pushl %eax pushl %ebp pushl %edi pushl %esi@@ -119,24 +132,25 @@ ENTRY(ftrace_regs_caller) ENCODE_FRAME_POINTER - movl 12*4(%esp), %eax /* Load ip (1st parameter) */ - subl $MCOUNT_INSN_SIZE, %eax /* Adjust ip */ - movl 15*4(%esp), %edx /* Load parent ip (2nd parameter) */ - movl function_trace_op, %ecx /* Save ftrace_pos in 3rd parameter */ - pushl %esp /* Save pt_regs as 4th parameter */ + movl PT_EIP(%esp), %eax # 1st argument: IP + subl $MCOUNT_INSN_SIZE, %eax + movl 21*4(%esp), %edx # 2nd argument: parent ip + movl function_trace_op, %ecx # 3rd argument: ftrace_pos + pushl %esp # 4th argument: pt_regs GLOBAL(ftrace_regs_call) call ftrace_stub - addl $4, %esp /* Skip pt_regs */ + addl $4, %esp # skip 4th argument - /* restore flags */ - push 14*4(%esp) - popf - - /* Move return ip back to its original location */ - movl 12*4(%esp), %eax - movl %eax, 14*4(%esp) + /* place IP below the new SP */ + movl PT_OLDESP(%esp), %eax + movl PT_EIP(%esp), %ecx + movl %ecx, -4(%eax) + + /* place EAX below that */ + movl PT_EAX(%esp), %ecx + movl %ecx, -8(%eax) popl %ebx popl %ecx@@ -144,14 +158,9 @@ GLOBAL(ftrace_regs_call) popl %esi popl %edi popl %ebp - popl %eax - popl %ds - popl %es - popl %fs - popl %gs - /* use lea to not affect flags */ - lea 3*4(%esp), %esp /* Skip orig_ax, ip and cs */ + lea -8(%eax), %esp + popl %eax jmp .Lftrace_retarch/x86/kernel/kgdb.c +0
-8@@ -127,14 +127,6 @@ char *dbg_get_reg(int regno, void *mem,#ifdef CONFIG_X86_32 switch (regno) { -case GDB_SS : -if (!user_mode (regs)) - *(unsigned long *)mem = __KERNEL_DS; -break ; -case GDB_SP : -if (!user_mode (regs)) - *(unsigned long *)mem =kernel_stack_pointer (regs); -break ;case GDB_GS :case GDB_FS : *(unsigned long *)mem =0xFFFF ;arch/x86/kernel/kprobes/common.h +2
-2@@ -72,8 +72,8 @@" popl %edi\n" \" popl %ebp\n" \" popl %eax\n" \ -/* Skip ds, es, fs, gs, orig_ax, and ip. Note: don't pop cs here*/ \ - " addl $24, %esp\n" +/* Skip ds, es, fs, gs, orig_ax, ip, and cs. */ \ + " addl $7*4, %esp\n"#endif /* Ensure if the instruction can be boostable */ arch/x86/kernel/kprobes/core.c +12
-17@@ -69,7 +69,7 @@ DEFINE_PER_CPU(struct kprobe *, current_kprobe) = NULL; DEFINE_PER_CPU(struct kprobe_ctlblk, kprobe_ctlblk); -#define stack_addr (regs) ((unsigned long *)kernel_stack_pointer(regs)) +#define stack_addr (regs) ((unsigned long *)regs->sp)#define W (row, b0, b1, b2, b3, b4, b5, b6, b7, b8, b9, ba, bb, bc, bd, be, bf)\ (((b0##UL << 0x0)|(b1##UL << 0x1)|(b2##UL << 0x2)|(b3##UL << 0x3) | \@@ -731,29 +731,27 @@ asm(".global kretprobe_trampoline\n" ".type kretprobe_trampoline, @function\n" "kretprobe_trampoline:\n" -#ifdef CONFIG_X86_64/* We don't bother saving the ss register */ +#ifdef CONFIG_X86_64" pushq %rsp\n" " pushfq\n" SAVE_REGS_STRING " movq %rsp, %rdi\n" " call trampoline_handler\n" /* Replace saved sp with true return address. */ - " movq %rax, 152(%rsp)\n" +" movq %rax, 19*8(%rsp)\n" RESTORE_REGS_STRING " popfq\n" #else - " pushf\n" + " pushl %esp\n" + " pushfl\n"SAVE_REGS_STRING " movl %esp, %eax\n" " call trampoline_handler\n" - /* Move flags to cs */ - " movl 56(%esp), %edx\n" - " movl %edx, 52(%esp)\n" - /* Replace saved flags with true return address. */ - " movl %eax, 56(%esp)\n" +/* Replace saved sp with true return address. */ +" movl %eax, 15*4(%esp)\n" RESTORE_REGS_STRING - " popf\n" +" popfl\n" #endif " ret\n" ".size kretprobe_trampoline, .-kretprobe_trampoline\n"@@ -794,16 +792,13 @@ __used __visible void *trampoline_handleINIT_HLIST_HEAD (&empty_rp);kretprobe_hash_lock (current, &head, &flags);/* fixup registers */ -#ifdef CONFIG_X86_64 regs->cs = __KERNEL_CS; -/* On x86-64, we use pt_regs->sp for return address holder. */ - frame_pointer = ®s->sp; -#else - regs->cs = __KERNEL_CS |get_kernel_rpl (); +#ifdef CONFIG_X86_32 + regs->cs |=get_kernel_rpl (); regs->gs =0 ; -/* On x86-32, we use pt_regs->flags for return address holder. */ - frame_pointer = ®s->flags;#endif +/* We use pt_regs->sp for return address holder. */ + frame_pointer = ®s->sp; regs->ip = trampoline_address; regs->orig_ax = ~0UL ;arch/x86/kernel/kprobes/opt.c +12
-8@@ -115,14 +115,15 @@ asm ("optprobe_template_call:\n" ASM_NOP5 /* Move flags to rsp */ -" movq 144(%rsp), %rdx\n" -" movq %rdx, 152(%rsp)\n" +" movq 18*8(%rsp), %rdx\n" +" movq %rdx, 19*8(%rsp)\n" RESTORE_REGS_STRING /* Skip flags entry */ " addq $8, %rsp\n" " popfq\n"#else /* CONFIG_X86_32 */ -" pushf\n" +" pushl %esp\n" +" pushfl\n" SAVE_REGS_STRING " movl %esp, %edx\n" ".global optprobe_template_val\n" @@ -131,9 +132,13 @@ asm (".global optprobe_template_call\n" "optprobe_template_call:\n" ASM_NOP5 +/* Move flags into esp */ +" movl 14*4(%esp), %edx\n" +" movl %edx, 15*4(%esp)\n" RESTORE_REGS_STRING -" addl $4, %esp\n" /* skip cs */ - " popf\n" +/* Skip flags entry */ +" addl $4, %esp\n" + " popfl\n"#endif ".global optprobe_template_end\n" "optprobe_template_end:\n" @@ -165,10 +170,9 @@ optimized_callback(struct optimized_kpro }else {struct kprobe_ctlblk *kcb =get_kprobe_ctlblk ();/* Save skipped registers */ -#ifdef CONFIG_X86_64 regs->cs = __KERNEL_CS; -#else - regs->cs = __KERNEL_CS |get_kernel_rpl (); +#ifdef CONFIG_X86_32 + regs->cs |=get_kernel_rpl (); regs->gs =0 ;#endif regs->ip = (unsigned long )op->kp.addr +INT3_SIZE ;arch/x86/kernel/process_32.c +5
-11@@ -62,27 +62,21 @@ void __show_regs(struct pt_regs *regs, e {unsigned long cr0 =0L , cr2 =0L , cr3 =0L , cr4 =0L ;unsigned long d0, d1, d2, d3, d6, d7; -unsigned long sp; -unsigned short ss, gs; +unsigned short gs; -if (user_mode (regs)) { - sp = regs->sp; - ss = regs->ss; +if (user_mode (regs)) gs =get_user_gs (regs); - }else { - sp =kernel_stack_pointer (regs); -savesegment (ss, ss); +else savesegment (gs, gs); - }show_ip (regs,KERN_DEFAULT );printk (KERN_DEFAULT "EAX: %08lx EBX: %08lx ECX: %08lx EDX: %08lx\n" , regs->ax, regs->bx, regs->cx, regs->dx);printk (KERN_DEFAULT "ESI: %08lx EDI: %08lx EBP: %08lx ESP: %08lx\n" , - regs->si, regs->di, regs->bp, sp); + regs->si, regs->di, regs->bp, regs->sp);printk (KERN_DEFAULT "DS: %04x ES: %04x FS: %04x GS: %04x SS: %04x EFLAGS: %08lx\n" , - (u16 )regs->ds, (u16 )regs->es, (u16 )regs->fs, gs, ss, regs->flags); + (u16 )regs->ds, (u16 )regs->es, (u16 )regs->fs, gs, regs->ss, regs->flags);if (mode !=SHOW_REGS_ALL )return ;arch/x86/kernel/ptrace.c +0
-29@@ -153,35 +153,6 @@ static inline bool invalid_selector(u16#define FLAG_MASK FLAG_MASK_32 -/* -* X86_32 CPUs don't save ss and esp if the CPU is already in kernel mode -* when it traps. The previous stack will be directly underneath the saved -* registers, and 'sp/ss' won't even have been saved. Thus the '®s->sp'. -* -* Now, if the stack is empty, '®s->sp' is out of range. In this -* case we try to take the previous stack. To always return a non-null -* stack pointer we fall back to regs as stack if no previous stack -* exists. -* -* This is valid only for kernel mode traps. -*/ -unsigned long kernel_stack_pointer (struct pt_regs *regs) -{ -unsigned long context = (unsigned long )regs & ~(THREAD_SIZE -1 ); -unsigned long sp = (unsigned long )®s->sp; -u32 *prev_esp; - -if (context == (sp & ~(THREAD_SIZE -1 ))) -return sp; - - prev_esp = (u32 *)(context); -if (*prev_esp) -return (unsigned long )*prev_esp; - -return (unsigned long )regs; -} -EXPORT_SYMBOL_GPL (kernel_stack_pointer); -static unsigned long *pt_regs_access (struct pt_regs *regs,unsigned long regno) {BUILD_BUG_ON (offsetof(struct pt_regs , bx) !=0 );arch/x86/kernel/time.c +1
-2@@ -37,8 +37,7 @@ unsigned long profile_pc(struct pt_regs#ifdef CONFIG_FRAME_POINTER return *(unsigned long *)(regs->bp +sizeof (long ));#else -unsigned long *sp = - (unsigned long *)kernel_stack_pointer (regs); +unsigned long *sp = (unsigned long *)regs->sp;/* * Return address is either directly at stack pointer * or above a saved flags. Eflags has bits 22-31 zero, arch/x86/kernel/unwind_frame.c +4
-28@@ -69,15 +69,6 @@ static void unwind_dump(struct unwind_st } } -static size_t regs_size (struct pt_regs *regs) -{ -/* x86_32 regs from kernel mode are two words shorter: */ -if (IS_ENABLED (CONFIG_X86_32 ) && !user_mode (regs)) -return sizeof (*regs) -2 *sizeof (long ); - -return sizeof (*regs); -} -static bool in_entry_code (unsigned long ip) {char *addr = (char *)ip;@@ -197,12 +188,6 @@ static struct pt_regs *decode_frame_poin }#endif -#ifdef CONFIG_X86_32 -#define KERNEL_REGS_SIZE (sizeof(struct pt_regs) - 2*sizeof(long)) -#else -#define KERNEL_REGS_SIZE (sizeof(struct pt_regs)) -#endif -static bool update_stack_state (struct unwind_state *state,unsigned long *next_bp) {@@ -213,7 +198,7 @@ static bool update_stack_state(struct unsize_t len;if (state->regs) - prev_frame_end = (void *)state->regs +regs_size (state->regs); + prev_frame_end = (void *)state->regs +sizeof (*state->regs);else prev_frame_end = (void *)state->bp +FRAME_HEADER_SIZE ;@@ -221,7 +206,7 @@ static bool update_stack_state(struct un regs =decode_frame_pointer (next_bp);if (regs) { frame = (unsigned long *)regs; - len =KERNEL_REGS_SIZE ; + len =sizeof (*regs); state->got_irq = true; }else { frame = next_bp;@@ -245,14 +230,6 @@ static bool update_stack_state(struct un frame < prev_frame_end)return false; -/* -* On 32-bit with user mode regs, make sure the last two regs are safe -* to access: -*/ -if (IS_ENABLED (CONFIG_X86_32 ) && regs &&user_mode (regs) && - !on_stack (info, frame, len +2 *sizeof (long ))) -return false; -/* Move state to the next frame: */ if (regs) { state->regs = regs;@@ -411,10 +388,9 @@ void __unwind_start(struct unwind_state* Pretend that the frame is complete and that BP points to it, but save * the real BP so that we can use it when looking for the next frame. */ -if (regs && regs->ip ==0 && - (unsigned long *)kernel_stack_pointer (regs) >= first_frame) { +if (regs && regs->ip ==0 && (unsigned long *)regs->sp >= first_frame) { state->next_bp = bp; - bp = ((unsigned long *)kernel_stack_pointer (regs)) -1 ; + bp = ((unsigned long *)regs->sp) -1 ; }/* Initialize stack info and make sure the frame data is accessible: */ arch/x86/kernel/unwind_orc.c +1
-1@@ -579,7 +579,7 @@ void __unwind_start(struct unwind_state goto done; state->ip = regs->ip; - state->sp =kernel_stack_pointer (regs); + state->sp = regs->sp; state->bp = regs->bp; state->regs = regs; state->full_regs = true;
Signature · 1 line
Masami Hiramatsu <[email protected]>
Re: [PATCH 05/15] x86_32: Provide consistent pt_regs
In reply to Peter ZijlstraQuoted 21 lines · Peter Zijlstra
Currently
pt_regsonx86_32has an oddity in that kernel regs (!user_mode(regs)) are short two entries (esp/ss). This means that any code trying to use them (typically:regs->sp) needs to jump through some unfortunate hoops.Change the entry code to fix this up and create a full
pt_regsframe.This then simplifies various trampolines in ftrace and kprobes, the stack unwinder, ptrace, kdump and kgdb.
Signed-off-by Peter Zijlstra (Intel) <[email protected]>--- arch/x86/entry/entry_32.S | 105 ++++++++++++++++++++++++++++++++++---- arch/x86/include/asm/kexec.h | 17 ------ arch/x86/include/asm/ptrace.h | 17 ------ arch/x86/include/asm/stacktrace.h | 2 arch/x86/kernel/crash.c | 8 -- arch/x86/kernel/ftrace_32.S | 77 +++++++++++++++------------ arch/x86/kernel/kgdb.c | 8 -- arch/x86/kernel/kprobes/common.h | 4 -
Quoted 8 lines
arch/x86/kernel/kprobes/core.c | 29 ++++------ arch/x86/kernel/kprobes/opt.c | 20 ++++--- arch/x86/kernel/process_32.c | 16 +---- arch/x86/kernel/ptrace.c | 29 ---------- arch/x86/kernel/time.c | 3 - arch/x86/kernel/unwind_frame.c | 32 +---------- arch/x86/kernel/unwind_orc.c | 2 15 files changed, 178 insertions(+), 191 deletions(-)
I recall writing some of this code (some of the kernel_stack_pointer removal stuff) so please give me a shout-out ;-)
Otherwise:
Signature · 1 line
Josh
Peter Zijlstra
maintainerRe: [PATCH 05/15] x86_32: Provide consistent pt_regs
Quoted 31 lines · Josh Poimboeuf
Quoted 21 lines · Peter Zijlstra
Currently
pt_regsonx86_32has an oddity in that kernel regs (!user_mode(regs)) are short two entries (esp/ss). This means that any code trying to use them (typically:regs->sp) needs to jump through some unfortunate hoops.Change the entry code to fix this up and create a full
pt_regsframe.This then simplifies various trampolines in ftrace and kprobes, the stack unwinder, ptrace, kdump and kgdb.
Signed-off-by Peter Zijlstra (Intel) <[email protected]>--- arch/x86/entry/entry_32.S | 105 ++++++++++++++++++++++++++++++++++---- arch/x86/include/asm/kexec.h | 17 ------ arch/x86/include/asm/ptrace.h | 17 ------ arch/x86/include/asm/stacktrace.h | 2 arch/x86/kernel/crash.c | 8 -- arch/x86/kernel/ftrace_32.S | 77 +++++++++++++++------------ arch/x86/kernel/kgdb.c | 8 -- arch/x86/kernel/kprobes/common.h | 4 -Quoted 8 lines
arch/x86/kernel/kprobes/core.c | 29 ++++------ arch/x86/kernel/kprobes/opt.c | 20 ++++--- arch/x86/kernel/process_32.c | 16 +---- arch/x86/kernel/ptrace.c | 29 ---------- arch/x86/kernel/time.c | 3 - arch/x86/kernel/unwind_frame.c | 32 +---------- arch/x86/kernel/unwind_orc.c | 2 15 files changed, 178 insertions(+), 191 deletions(-)
Quoted 2 lines
I recall writing some of this code (some of the
kernel_stack_pointerremoval stuff) so please give me a shout-out ;-)
Absolutely, sorry for not doing that. I've added:
"Much thanks to Josh for help with the cleanups!"
Peter Zijlstra
maintainerPATCH 6/15 x86_32: Allow int3_emulate_push()
In reply to Peter ZijlstraNow that x86_32 has an unconditional gap on the kernel stack frame, the int3_emulate_push() thing will work without further changes.
--- arch/x86/include/asm/text-patching.h | 2 -- arch/x86/kernel/ftrace.c | 7 ------- 2 files changed, 9 deletions(-)
arch/x86/include/asm/text-patching.h +0 -2
@@ -51,7 +51,6 @@ static inline void int3_emulate_jmp(stru#define INT3_INSN_SIZE 1#define CALL_INSN_SIZE 5 -#ifdef CONFIG_X86_64 static inline void int3_emulate_push (struct pt_regs *regs,unsigned long val) {/*
@@ -69,7 +68,6 @@ static inline void int3_emulate_call(strint3_emulate_push (regs, regs->ip -INT3_INSN_SIZE +CALL_INSN_SIZE );int3_emulate_jmp (regs, func); } -#endif /* CONFIG_X86_64 */ #endif /* !CONFIG_UML_X86 */ #endif /* _ASM_X86_TEXT_PATCHING_H */
arch/x86/kernel/ftrace.c +0 -7
@@ -300,7 +300,6 @@ int ftrace_int3_handler(struct pt_regs * ip = regs->ip -INT3_INSN_SIZE ; -#ifdef CONFIG_X86_64 if (ftrace_location (ip)) {int3_emulate_call (regs, (unsigned long )ftrace_regs_caller);return 1 ;
@@ -312,12 +311,6 @@ int ftrace_int3_handler(struct pt_regs *int3_emulate_call (regs, ftrace_update_func_call);return 1 ; } -#else -if (ftrace_location (ip) ||is_ftrace_caller (ip)) { -int3_emulate_jmp (regs, ip +CALL_INSN_SIZE ); -return 1 ; - } -#endif return 0 ; }
Peter Zijlstra
maintainerPATCH 7/15 x86: Add int3_emulate_call() selftest
In reply to Peter ZijlstraGiven that the entry_*.S changes for this functionality are somewhat tricky, make sure the paths are tested every boot, instead of on the rare occasion when we trip an INT3 while rewriting text.
--- arch/x86/kernel/alternative.c | 81 +++++++++++++++++++++++++++++++++++++++--- 1 file changed, 77 insertions(+), 4 deletions(-)
arch/x86/kernel/alternative.c +77 -4
@@ -614,11 +614,83 @@ extern struct paravirt_patch_site __star __stop_parainstructions[];#endif /* CONFIG_PARAVIRT */ +/* +* Self-test for the INT3 based CALL emulation code. +* +* This exercises int3_emulate_call() to make sure INT3 pt_regs are set up +* properly and that there is a stack gap between the INT3 frame and the +* previous context. Without this gap doing a virtual PUSH on the interrupted +* stack would corrupt the INT3 IRET frame. +* +* See entry_{32,64}.S for more details. +*/ +static void __init int3_magic (unsigned int *ptr) +{ + *ptr =1 ; +} + +extern __initdata unsigned long int3_selftest_ip;/* defined in asm below */ + +static int __init +int3_exception_notify (struct notifier_block *self,unsigned long val,void *data) +{ +struct die_args *args = data; +struct pt_regs *regs = args->regs; + +if (!regs ||user_mode (regs)) +return NOTIFY_DONE ; + +if (val !=DIE_INT3 ) +return NOTIFY_DONE ; + +if (regs->ip -INT3_INSN_SIZE != int3_selftest_ip) +return NOTIFY_DONE ; + +int3_emulate_call (regs, (unsigned long )&int3_magic); +return NOTIFY_STOP ; +} + +static void __init int3_selftest (void ) +{ +static __initdata struct notifier_block int3_exception_nb = { + .notifier_call = int3_exception_notify, + .priority =INT_MAX -1 ,/* last */ + }; +unsigned int val =0 ; + +BUG_ON (register_die_notifier (&int3_exception_nb)); + +/* +* Basically: int3_magic(&val); but really complicated :-) +* +* Stick the address of the INT3 instruction into int3_selftest_ip, +* then trigger the INT3, padded with NOPs to match a CALL instruction +* length. +*/ + asmvolatile ("1: int3; nop; nop; nop; nop\n\t" +".pushsection .init.data,\"aw\"\n\t" + ".align " __ASM_SEL(4, 8) "\n\t" +".type int3_selftest_ip, @object\n\t" + ".size int3_selftest_ip, " __ASM_SEL(4, 8) "\n\t" + "int3_selftest_ip:\n\t" + __ASM_SEL(.long, .quad) " 1b\n\t" + ".popsection\n\t" + : : __ASM_SEL_RAW(a, D) (&val) : "memory"); + +BUG_ON (val !=1 ); + +unregister_die_notifier (&int3_exception_nb); +} +void __init alternative_instructions (void ) { -/* The patching is not fully atomic, so try to avoid local interruptions -that might execute the to be patched code. -Other CPUs are not running. */ +int3_selftest (); + +/* +* The patching is not fully atomic, so try to avoid local +* interruptions that might execute the to be patched code. +* Other CPUs are not running. +*/ stop_nmi ();/*
@@ -643,10 +715,11 @@ void __init alternative_instructions(voi _text, _etext); } -if (!uniproc_patched ||num_possible_cpus () ==1 ) +if (!uniproc_patched ||num_possible_cpus () ==1 ) {free_init_pages ("SMP alternatives" , (unsigned long )__smp_locks, (unsigned long )__smp_locks_end); + }#endif apply_paravirt (__parainstructions, __parainstructions_end);
Re: [PATCH 07/15] x86: Add int3_emulate_call() selftest
Quoted 6 lines · Peter Zijlstra
Given that the entry_*.S changes for this functionality are somewhat tricky, make sure the paths are tested every boot, instead of on the rare occasion when we trip an INT3 while rewriting text.
Requested-by Andy Lutomirski <[email protected]>Signed-off-by Peter Zijlstra (Intel) <[email protected]>
Signature · 1 line
Josh
Andy Lutomirski
maintainerRe: [PATCH 07/15] x86: Add int3_emulate_call() selftest
Quoted 11 lines · Josh Poimboeuf
Quoted 6 lines · Peter Zijlstra
Given that the entry_*.S changes for this functionality are somewhat tricky, make sure the paths are tested every boot, instead of on the rare occasion when we trip an INT3 while rewriting text.
Requested-by Andy Lutomirski <[email protected]>Signed-off-by Peter Zijlstra (Intel) <[email protected]>Reviewed-by Josh Poimboeuf <[email protected]>
Looks good to me, too, except that I seriously hate die notifiers that return NOTIFY_STOP, and I eventually want to remove support for them. This can wait, though.
Peter Zijlstra
maintainerRe: [PATCH 07/15] x86: Add int3_emulate_call() selftest
Quoted 7 lines · Andy Lutomirski
Quoted 6 lines · Josh Poimboeuf
Quoted 4 lines · Peter Zijlstra
Given that the entry_*.S changes for this functionality are somewhat tricky, make sure the paths are tested every boot, instead of on the rare occasion when we trip an INT3 while rewriting text.
Quoted 7 lines
Quoted 5 lines
Quoted 2 lines
Requested-by Andy Lutomirski <[email protected]>Signed-off-by Peter Zijlstra (Intel) <[email protected]>Reviewed-by Josh Poimboeuf <[email protected]>Looks good to me, too,
I'll translate that into an Acked-by from you, if you don't mind :-)
Quoted 3 lines
except that I seriously hate die notifiers that
returnNOTIFY_STOP, and I eventually want to remove support for them. This can wait, though.
Yes, I share your hatred for notifiers in general. But since they are still here and I do think it is a waste to have an unconditional function call in do_int3() just for this, I figured I'd use them.
Peter Zijlstra
maintainerPATCH 8/15 x86/alternatives: Teach text_poke_bp() to emulate instructions
In reply to Peter ZijlstraIn preparation for static_call support, teach text_poke_bp() to emulate instructions, including CALL.
The current text_poke_bp() takes a @handler argument which is used as a jump target when the temporary INT3 is hit by a different CPU.
When patching CALL instructions, this doesn't work because we'd miss the PUSH of the return address. Instead, teach poke_int3_handler() to emulate an instruction, typically the instruction we're patching in.
This fits almost all text_poke_bp() users, except arch_unoptimize_kprobe() which restores random text, and for that site we have to build an explicit emulate instruction.
--- arch/x86/include/asm/text-patching.h | 2 - arch/x86/kernel/alternative.c | 47 ++++++++++++++++++++++++++--------- arch/x86/kernel/jump_label.c | 3 -- arch/x86/kernel/kprobes/opt.c | 11 +++++--- 4 files changed, 46 insertions(+), 17 deletions(-)
arch/x86/include/asm/text-patching.h +1 -1
@@ -37,7 +37,7 @@ extern void text_poke_early(void *addr,extern void *text_poke (void *addr,const void *opcode,size_t len);extern void *text_poke_kgdb (void *addr,const void *opcode,size_t len);extern int poke_int3_handler (struct pt_regs *regs); -extern void text_poke_bp (void *addr,const void *opcode,size_t len,void *handler); +extern void text_poke_bp (void *addr,const void *opcode,size_t len,const void *emulate);extern int after_bootmem;extern __ro_after_init struct mm_struct *poking_mm;extern __ro_after_init unsigned long poking_addr;
arch/x86/kernel/alternative.c +36 -11
89 lines not on this page · raw message
arch/x86/kernel/jump_label.c +1 -2
@@ -87,8 +87,7 @@ static void __ref __jump_label_transformreturn ; } -text_poke_bp ((void *)jump_entry_code (entry), code,JUMP_LABEL_NOP_SIZE , - (void *)jump_entry_code (entry) +JUMP_LABEL_NOP_SIZE ); +text_poke_bp ((void *)jump_entry_code (entry), code,JUMP_LABEL_NOP_SIZE ,NULL ); } void arch_jump_label_transform(struct jump_entry *entry,
arch/x86/kernel/kprobes/opt.c +8 -3
@@ -437,8 +437,7 @@ void arch_optimize_kprobes(struct list_h insn_buff[0 ] =RELATIVEJUMP_OPCODE ; *(s32 *)(&insn_buff[1 ]) = rel; -text_poke_bp (op->kp.addr, insn_buff,RELATIVEJUMP_SIZE , - op->optinsn.insn); +text_poke_bp (op->kp.addr, insn_buff,RELATIVEJUMP_SIZE ,NULL );list_del_init (&op->list); }
@@ -448,12 +447,18 @@ void arch_optimize_kprobes(struct list_hvoid arch_unoptimize_kprobe (struct optimized_kprobe *op) {u8 insn_buff[RELATIVEJUMP_SIZE ]; +u8 emulate_buff[RELATIVEJUMP_SIZE ];/* Set int3 to first byte for kprobes */ insn_buff[0 ] =BREAKPOINT_INSTRUCTION ;memcpy (insn_buff +1 , op->optinsn.copied_insn,RELATIVE_ADDR_SIZE ); + + emulate_buff[0 ] =RELATIVEJUMP_OPCODE ; + *(s32 *)(&emulate_buff[1 ]) = (s32)((long )op->optinsn.insn - + ((long )op->kp.addr +RELATIVEJUMP_SIZE )); +text_poke_bp (op->kp.addr, insn_buff,RELATIVEJUMP_SIZE , - op->optinsn.insn); + emulate_buff); }/*
Re: [PATCH 08/15] x86/alternatives: Teach text_poke_bp() to emulate instructions
Quoted 54 lines · Peter Zijlstra
In preparation for
static_callsupport, teachtext_poke_bp()to emulate instructions, including CALL.The current
text_poke_bp()takes a @handler argument which is used as a jump target when the temporary INT3 is hit by a different CPU.When patching CALL instructions, this doesn't work because we'd miss the PUSH of the return address. Instead, teach
poke_int3_handler()to emulate an instruction, typically the instruction we're patching in.This fits almost all
text_poke_bp()users, exceptarch_unoptimize_kprobe()which restores random text, and for that site we have to build an explicit emulate instruction.Cc Daniel Bristot de Oliveira <[email protected]>Cc Nadav Amit <[email protected]>Signed-off-by Peter Zijlstra (Intel) <[email protected]>---
arch/x86/include/asm/text-patching.h | 2 - arch/x86/kernel/alternative.c | 47 ++++++++++++++++++++++++++--------- arch/x86/kernel/jump_label.c | 3 -- arch/x86/kernel/kprobes/opt.c | 11 +++++--- 4 files changed, 46 insertions(+), 17 deletions(-)arch/x86/include/asm/text-patching.h +1
-1@@ -37,7 +37,7 @@ extern void text_poke_early(void *addr,extern void *text_poke (void *addr,const void *opcode,size_t len);extern void *text_poke_kgdb (void *addr,const void *opcode,size_t len);extern int poke_int3_handler (struct pt_regs *regs); -extern void text_poke_bp (void *addr,const void *opcode,size_t len,void *handler); +extern void text_poke_bp (void *addr,const void *opcode,size_t len,const void *emulate);extern int after_bootmem;extern __ro_after_init struct mm_struct *poking_mm;extern __ro_after_init unsigned long poking_addr;arch/x86/kernel/alternative.c +7
-1@@ -921,19 +921,25 @@ static void do_sync_core(void *info) }static bool bp_patching_in_progress; -static void *bp_int3_handler, *bp_int3_addr; +static const void *bp_int3_opcode, *bp_int3_addr;int poke_int3_handler (struct pt_regs *regs) { +long ip = regs->ip -INT3_INSN_SIZE +CALL_INSN_SIZE ; +struct opcode { +u8 insn; +s32 rel; + } __packed opcode; +
arch/x86/kernel/alternative.c static void do_sync_core(void *info) · line 940
Quoted 10 lines
arch/x86/kernel/alternative.c +3
-3/* * Having observed our INT3 instruction, we now must observe * bp_patching_in_progress. * -* in_progress = TRUE INT3 -* WMB RMB -* write INT3 if (in_progress) +* in_progress = TRUE INT3 +* WMB RMB +* write INT3 if (in_progress)
I don’t see what has changed in this chunk… Whitespaces?
Quoted 12 lines
arch/x86/kernel/alternative.c +1
-3* -* Idem for bp_int3_handler. +* Idem for bp_int3_opcode. */ smp_rmb ();@@ -943,8 +949,21 @@ int poke_int3_handler(struct pt_regs *reif (user_mode (regs) || regs->ip != (unsigned long )bp_int3_addr)return 0 ; -/* set up the specified breakpoint handler */ - regs->ip = (unsigned long ) bp_int3_handler;
arch/x86/kernel/alternative.c int poke_int3_handler(struct pt_regs *re · line 961
Quoted 10 lines
arch/x86/kernel/alternative.c +10
-0+ opcode = *(struct opcode *)bp_int3_opcode; + +switch (opcode.insn) { +case 0xE8 :/* CALL */ +int3_emulate_call (regs, ip + opcode.rel); +break ; + +case 0xE9 :/* JMP */ +int3_emulate_jmp (regs, ip + opcode.rel); +break ;
Consider using RELATIVECALL_OPCODE and RELATIVEJUMP_OPCODE instead of the constants (0xE8, 0xE9), just as you do later in the patch.
Peter Zijlstra
maintainerRe: [PATCH 08/15] x86/alternatives: Teach text_poke_bp() to emulate instructions
Quoted 12 lines · Nadav Amit
Quoted 12 lines
int
poke_int3_handler(struct pt_regs *regs)
{(quoted diff) +6
-0+ long ip = regs->ip - INT3_INSN_SIZE + CALL_INSN_SIZE; + struct opcode { + u8 insn; + s32 rel; + } __packed opcode; +/* * Having observed our INT3 instruction, we now must observe * bp_patching_in_progress. *
Quoted 8 lines
Review
Quoted 6 lines
(quoted diff) +3
-3- * in_progress = TRUE INT3 - * WMB RMB - * write INT3 if (in_progress) + * in_progress = TRUE INT3 + * WMB RMB + * write INT3 if (in_progress)I don’t see what has changed in this chunk… Whitespaces?
Yep, my editor kept marking that stuff red (space before tab), which annoyed me enough to fix it.
Quoted 10 lines
Quoted 10 lines
*(quoted diff) +1
-1- * Idem for bp_int3_handler. + * Idem for bp_int3_opcode.*/ smp_rmb();(quoted diff) +0
-0@@ -943,8 +949,21 @@ int poke_int3_handler(struct pt_regs *re if (user_mode(regs) || regs->ip != (unsigned long)bp_int3_addr) return 0;
Quoted 15 lines
Review int poke_int3_handler(struct pt_regs *re · line 961
Quoted 12 lines
(quoted diff) +10
-2- /* set up the specified breakpoint handler */ - regs->ip = (unsigned long) bp_int3_handler; + opcode = *(struct opcode *)bp_int3_opcode; + + switch (opcode.insn) { + case 0xE8: /* CALL */ + int3_emulate_call(regs, ip + opcode.rel); + break; + + case 0xE9: /* JMP */ + int3_emulate_jmp(regs, ip + opcode.rel); + break;Consider using
RELATIVECALL_OPCODEandRELATIVEJUMP_OPCODEinstead of the constants (0xE8,0xE9), just as you do later in the patch.
Those are private to kprobes..
but I can do something like so:
arch/x86/include/asm/text-patching.h +8 -2
@@ -48,8 +48,14 @@ static inline void int3_emulate_jmp(stru regs->ip = ip; } -#define INT3_INSN_SIZE 1 -#define CALL_INSN_SIZE 5 +#define INT3_INSN_SIZE 1 +#define INT3_INSN_OPCODE 0xCC + +#define CALL_INSN_SIZE 5 +#define CALL_INSN_OPCODE 0xE8 + +#define JMP_INSN_SIZE 5 +#define JMP_INSN_OPCODE 0xE9static inline void int3_emulate_push (struct pt_regs *regs,unsigned long val) {
arch/x86/kernel/alternative.c +2 -2
@@ -952,11 +952,11 @@ int poke_int3_handler(struct pt_regs *re opcode = *(struct opcode *)bp_int3_opcode;switch (opcode.insn) { -case 0xE8 :/* CALL */ +case CALL_INSN_OPCODE :int3_emulate_call (regs, ip + opcode.rel);break ; -case 0xE9 :/* JMP */ +case JMP_INSN_OPCODE :int3_emulate_jmp (regs, ip + opcode.rel);break ;
Masami Hiramatsu
maintainerRe: [PATCH 08/15] x86/alternatives: Teach text_poke_bp() to emulate instructions
Quoted 81 lines · On Fri, 7 Jun 2019 10:20:13 +0200 Peter Zijlstra <[email protected]> wrote:
Quoted 12 lines · Nadav Amit
Quoted 12 lines
int
poke_int3_handler(struct pt_regs *regs)
{(quoted diff) +6
-0+ long ip = regs->ip - INT3_INSN_SIZE + CALL_INSN_SIZE; + struct opcode { + u8 insn; + s32 rel; + } __packed opcode; +/* * Having observed our INT3 instruction, we now must observe * bp_patching_in_progress. *Quoted 8 lines
Review
Quoted 6 lines
(quoted diff) +3
-3- * in_progress = TRUE INT3 - * WMB RMB - * write INT3 if (in_progress) + * in_progress = TRUE INT3 + * WMB RMB + * write INT3 if (in_progress)I don’t see what has changed in this chunk… Whitespaces?
Yep, my editor kept marking that stuff red (space before tab), which annoyed me enough to fix it.
Quoted 10 lines
Quoted 10 lines
*(quoted diff) +1
-1- * Idem for bp_int3_handler. + * Idem for bp_int3_opcode.*/ smp_rmb();(quoted diff) +0
-0@@ -943,8 +949,21 @@ int poke_int3_handler(struct pt_regs *re if (user_mode(regs) || regs->ip != (unsigned long)bp_int3_addr) return 0;Quoted 15 lines
Review int poke_int3_handler(struct pt_regs *re · line 961
Quoted 12 lines
(quoted diff) +10
-2- /* set up the specified breakpoint handler */ - regs->ip = (unsigned long) bp_int3_handler; + opcode = *(struct opcode *)bp_int3_opcode; + + switch (opcode.insn) { + case 0xE8: /* CALL */ + int3_emulate_call(regs, ip + opcode.rel); + break; + + case 0xE9: /* JMP */ + int3_emulate_jmp(regs, ip + opcode.rel); + break;Consider using
RELATIVECALL_OPCODEandRELATIVEJUMP_OPCODEinstead of the constants (0xE8,0xE9), just as you do later in the patch.Those are private to kprobes..
but I can do something like so:
arch/x86/include/asm/text-patching.h +8
-2@@ -48,8 +48,14 @@ static inline void int3_emulate_jmp(stru regs->ip = ip; } -#define INT3_INSN_SIZE 1 -#define CALL_INSN_SIZE 5 +#define INT3_INSN_SIZE 1 +#define INT3_INSN_OPCODE 0xCC + +#define CALL_INSN_SIZE 5 +#define CALL_INSN_OPCODE 0xE8 + +#define JMP_INSN_SIZE 5 +#define JMP_INSN_OPCODE 0xE9static inline void int3_emulate_push (struct pt_regs *regs,unsigned long val) {arch/x86/kernel/alternative.c +0
-0@@ -952,11 +952,11 @@ int poke_int3_handler(struct pt_regs *re opcode = *(struct opcode *)bp_int3_opcode;switch (opcode.insn) {
arch/x86/kernel/alternative.c int poke_int3_handler(struct pt_regs *re · line 962
Quoted 10 lines
arch/x86/kernel/alternative.c +2
-2-case 0xE8 :/* CALL */ +case CALL_INSN_OPCODE :int3_emulate_call (regs, ip + opcode.rel);break ; -case 0xE9 :/* JMP */ +case JMP_INSN_OPCODE :int3_emulate_jmp (regs, ip + opcode.rel);break ;
This looks good. I don't want to make those opcode as private. I would like to share it.
Thank you,
Signature · 1 line
Masami Hiramatsu <[email protected]>
Masami Hiramatsu
maintainerRe: [PATCH 08/15] x86/alternatives: Teach text_poke_bp() to emulate instructions
In reply to Peter ZijlstraQuoted 6 lines · On Wed, 05 Jun 2019 15:08:01 +0200 Peter Zijlstra <[email protected]> wrote:
In preparation for
static_callsupport, teachtext_poke_bp()to emulate instructions, including CALL.The current
text_poke_bp()takes a @handler argument which is used as a jump target when the temporary INT3 is hit by a different CPU.
Quoted 7 lines
When patching CALL instructions, this doesn't work because we'd miss the PUSH of the return address. Instead, teach
poke_int3_handler()to emulate an instruction, typically the instruction we're patching in.This fits almost all
text_poke_bp()users, exceptarch_unoptimize_kprobe()which restores random text, and for that site we have to build an explicit emulate instruction.
Hm, actually it doesn't restores randome text, since the first byte must always be int3. As the function name means, it just unoptimizes (jump based optprobe -> int3 based kprobe).
Anyway, that is not an issue. With this patch, optprobe must still work.
Quoted 63 lines
Cc Daniel Bristot de Oliveira <[email protected]>Cc Nadav Amit <[email protected]>Signed-off-by Peter Zijlstra (Intel) <[email protected]>--- arch/x86/include/asm/text-patching.h | 2 - arch/x86/kernel/alternative.c | 47 ++++++++++++++++++++++++++--------- arch/x86/kernel/jump_label.c | 3 -- arch/x86/kernel/kprobes/opt.c | 11 +++++--- 4 files changed, 46 insertions(+), 17 deletions(-)arch/x86/include/asm/text-patching.h +1
-1@@ -37,7 +37,7 @@ extern void text_poke_early(void *addr,extern void *text_poke (void *addr,const void *opcode,size_t len);extern void *text_poke_kgdb (void *addr,const void *opcode,size_t len);extern int poke_int3_handler (struct pt_regs *regs); -extern void text_poke_bp (void *addr,const void *opcode,size_t len,void *handler); +extern void text_poke_bp (void *addr,const void *opcode,size_t len,const void *emulate);extern int after_bootmem;extern __ro_after_init struct mm_struct *poking_mm;extern __ro_after_init unsigned long poking_addr;arch/x86/kernel/alternative.c +13
-7@@ -921,19 +921,25 @@ static void do_sync_core(void *info) }static bool bp_patching_in_progress; -static void *bp_int3_handler, *bp_int3_addr; +static const void *bp_int3_opcode, *bp_int3_addr;int poke_int3_handler (struct pt_regs *regs) { +long ip = regs->ip -INT3_INSN_SIZE +CALL_INSN_SIZE ; +struct opcode { +u8 insn; +s32 rel; + } __packed opcode; +/* * Having observed our INT3 instruction, we now must observe * bp_patching_in_progress. * -* in_progress = TRUE INT3 -* WMB RMB -* write INT3 if (in_progress) +* in_progress = TRUE INT3 +* WMB RMB +* write INT3 if (in_progress) * -* Idem for bp_int3_handler. +* Idem for bp_int3_opcode. */ smp_rmb ();@@ -943,8 +949,21 @@ int poke_int3_handler(struct pt_regs *reif (user_mode (regs) || regs->ip != (unsigned long )bp_int3_addr)return 0 ; -/* set up the specified breakpoint handler */ - regs->ip = (unsigned long ) bp_int3_handler; + opcode = *(struct opcode *)bp_int3_opcode; +
arch/x86/kernel/alternative.c int poke_int3_handler(struct pt_regs *re · line 963
Quoted 10 lines
arch/x86/kernel/alternative.c +10
-0+switch (opcode.insn) { +case 0xE8 :/* CALL */ +int3_emulate_call (regs, ip + opcode.rel); +break ; + +case 0xE9 :/* JMP */ +int3_emulate_jmp (regs, ip + opcode.rel); +break ; + +default :/* assume NOP */
Shouldn't we check whether it is actually NOP here?
arch/x86/kernel/alternative.c int poke_int3_handler(struct pt_regs *re · line 966
Quoted 3 lines
arch/x86/kernel/alternative.c +3
-0+int3_emulate_jmp (regs, ip); +break ; + }
BTW, if we fix the length of patching always 5 bytes and allow user to apply it only from/to jump/call/nop, we may be better to remove "len" and rename it, something like "text_poke_branch" etc.
Thank you,
Quoted 87 lines
arch/x86/kernel/alternative.c +10
-4return 1 ; }@@ -955,7 +974,7 @@ NOKPROBE_SYMBOL(poke_int3_handler);* @addr: address to patch * @opcode: opcode of new instruction * @len: length to copy -* @handler: address to jump to when the temporary breakpoint is hit +* @emulate: opcode to emulate, when NULL use @opcode * * Modify multi-byte instruction by using int3 breakpoint on SMP. * We completely avoid stop_machine() here, and achieve the @@ -970,19 +989,25 @@ NOKPROBE_SYMBOL(poke_int3_handler);* replacing opcode * - sync cores */ -void text_poke_bp (void *addr,const void *opcode,size_t len,void *handler) +void text_poke_bp (void *addr,const void *opcode,size_t len,const void *emulate) {unsigned char int3 =0xcc ; - bp_int3_handler = handler; + bp_int3_opcode = emulate ?: opcode; bp_int3_addr = (u8 *)addr +sizeof (int3); bp_patching_in_progress = true;lockdep_assert_held (&text_mutex);/* +* poke_int3_handler() relies on @opcode being a 5 byte instruction; +* notably a JMP, CALL or NOP5_ATOMIC. +*/ +BUG_ON (len !=5 ); + +/* * Corresponding read barrier in int3 notifier for making sure the -* in_progress and handler are correctly ordered wrt. patching. +* in_progress and opcode are correctly ordered wrt. patching. */ smp_wmb ();arch/x86/kernel/jump_label.c +1
-2@@ -87,8 +87,7 @@ static void __ref __jump_label_transformreturn ; } -text_poke_bp ((void *)jump_entry_code (entry), code,JUMP_LABEL_NOP_SIZE , - (void *)jump_entry_code (entry) +JUMP_LABEL_NOP_SIZE ); +text_poke_bp ((void *)jump_entry_code (entry), code,JUMP_LABEL_NOP_SIZE ,NULL ); } void arch_jump_label_transform(struct jump_entry *entry,arch/x86/kernel/kprobes/opt.c +8
-3@@ -437,8 +437,7 @@ void arch_optimize_kprobes(struct list_h insn_buff[0 ] =RELATIVEJUMP_OPCODE ; *(s32 *)(&insn_buff[1 ]) = rel; -text_poke_bp (op->kp.addr, insn_buff,RELATIVEJUMP_SIZE , - op->optinsn.insn); +text_poke_bp (op->kp.addr, insn_buff,RELATIVEJUMP_SIZE ,NULL );list_del_init (&op->list); }@@ -448,12 +447,18 @@ void arch_optimize_kprobes(struct list_hvoid arch_unoptimize_kprobe (struct optimized_kprobe *op) {u8 insn_buff[RELATIVEJUMP_SIZE ]; +u8 emulate_buff[RELATIVEJUMP_SIZE ];/* Set int3 to first byte for kprobes */ insn_buff[0 ] =BREAKPOINT_INSTRUCTION ;memcpy (insn_buff +1 , op->optinsn.copied_insn,RELATIVE_ADDR_SIZE ); + + emulate_buff[0 ] =RELATIVEJUMP_OPCODE ; + *(s32 *)(&emulate_buff[1 ]) = (s32)((long )op->optinsn.insn - + ((long )op->kp.addr +RELATIVEJUMP_SIZE )); +text_poke_bp (op->kp.addr, insn_buff,RELATIVEJUMP_SIZE , - op->optinsn.insn); + emulate_buff); }/*
Signature · 1 line
Masami Hiramatsu <[email protected]>
Peter Zijlstra
maintainerRe: [PATCH 08/15] x86/alternatives: Teach text_poke_bp() to emulate instructions
Quoted 8 lines · Masami Hiramatsu
Quoted 3 lines
This fits almost all
text_poke_bp()users, exceptarch_unoptimize_kprobe()which restores random text, and for that site we have to build an explicit emulate instruction.Hm, actually it doesn't restores randome text, since the first byte must always be int3. As the function name means, it just unoptimizes (jump based optprobe -> int3 based kprobe).
Anyway, that is not an issue. With this patch, optprobe must still work.
I thought it basically restored 5 bytes of original text (with no guarantee it is a single instruction, or even a complete instruction), with the first byte replaced with INT3.
Quoted 4 lines
Quoted 4 lines
arch/x86/kernel/alternative.c +0
-0@@ -943,8 +949,21 @@ int poke_int3_handler(struct pt_regs *reif (user_mode (regs) || regs->ip != (unsigned long )bp_int3_addr)return 0 ;
Quoted 16 lines
Quoted 4 lines
arch/x86/kernel/alternative.c +2
-2-/* set up the specified breakpoint handler */ - regs->ip = (unsigned long ) bp_int3_handler; + opcode = *(struct opcode *)bp_int3_opcode; +arch/x86/kernel/alternative.c int poke_int3_handler(struct pt_regs *re · line 963
Quoted 10 lines
arch/x86/kernel/alternative.c +10
-0+switch (opcode.insn) { +case 0xE8 :/* CALL */ +int3_emulate_call (regs, ip + opcode.rel); +break ; + +case 0xE9 :/* JMP */ +int3_emulate_jmp (regs, ip + opcode.rel); +break ; + +default :/* assume NOP */ Shouldn't we check whether it is actually NOP here?
I was/am lazy and didn't want to deal with:
arch/x86/include/asm/nops.h:#define GENERIC_NOP5_ATOMIC NOP_DS_PREFIX,GENERIC_NOP4 arch/x86/include/asm/nops.h:#define K8_NOP5_ATOMIC 0x66,K8_NOP4 arch/x86/include/asm/nops.h:#define K7_NOP5_ATOMIC NOP_DS_PREFIX,K7_NOP4 arch/x86/include/asm/nops.h:#define P6_NOP5_ATOMIC P6_NOP5
But maybe we should check for all the various NOP5 variants and BUG() on anything unexpected.
Quoted 7 lines
arch/x86/kernel/alternative.c int poke_int3_handler(struct pt_regs *re · line 966
Quoted 3 lines
arch/x86/kernel/alternative.c +3
-0+int3_emulate_jmp (regs, ip); +break ; + }BTW, if we fix the length of patching always 5 bytes and allow user to apply it only from/to jump/call/nop, we may be better to remove "len" and rename it, something like "
text_poke_branch" etc.
I considered it; but was thinking we could still allow patching other instructions, we'd just have to extend the emulation in poke_int3_handler().
Then again, if/when we want to do that, we can also restore the @len argument again.
Re: [PATCH 08/15] x86/alternatives: Teach text_poke_bp() to emulate instructions
Quoted 7 lines · Peter Zijlstra
I was/am lazy and didn't want to deal with:
arch/x86/include/asm/nops.h:#define
GENERIC_NOP5_ATOMICNOP_DS_PREFIX,GENERIC_NOP4 arch/x86/include/asm/nops.h:#defineK8_NOP5_ATOMIC0x66,K8_NOP4 arch/x86/include/asm/nops.h:#defineK7_NOP5_ATOMICNOP_DS_PREFIX,K7_NOP4 arch/x86/include/asm/nops.h:#defineP6_NOP5_ATOMICP6_NOP5
Ugh. Maybe we could just pick one atomic sequence, and not have the magic atomic nops be dynamic.
It's essentially what STATIC_KEY_INIT_NOP #define seems to do anyway.
NOP5_ATOMIC is already special, and not used for the normal nop rewriting, only for kprobe/jump_label/ftrace.
So I suspect we could just replace all cases of
ideal_nops[NOP_ATOMIC5]
with
STATIC_KEY_INIT_NOP
and get rid of the whole "let's optimize the atomic 5-byte nop" entirely.
Hmm?
By definition, NOP_ATOMIC5 is just a single nop anyway, it's not used for the potentially more complex alternative rewriting cases.
Linus
Peter Zijlstra
maintainerRe: [PATCH 08/15] x86/alternatives: Teach text_poke_bp() to emulate instructions
Quoted 11 lines · Linus Torvalds
Quoted 7 lines · Peter Zijlstra
I was/am lazy and didn't want to deal with:
arch/x86/include/asm/nops.h:#define
GENERIC_NOP5_ATOMICNOP_DS_PREFIX,GENERIC_NOP4 arch/x86/include/asm/nops.h:#defineK8_NOP5_ATOMIC0x66,K8_NOP4 arch/x86/include/asm/nops.h:#defineK7_NOP5_ATOMICNOP_DS_PREFIX,K7_NOP4 arch/x86/include/asm/nops.h:#defineP6_NOP5_ATOMICP6_NOP5Ugh. Maybe we could just pick one atomic sequence, and not have the magic atomic nops be dynamic.
That'd be nice..
Quoted 1 line
It's essentially what
STATIC_KEY_INIT_NOP#define seems to do anyway.
Well, that picks something, we'll overwrite it with the ideal nop later, once we've figured out what it should be.
Quoted 2 lines
NOP5_ATOMICis already special, and not used for the normal nop rewriting, only for kprobe/jump_label/ftrace.
Right, but esp ftrace means there's a _lot_ of them around.
Quoted 11 lines
So I suspect we could just replace all cases of
ideal_nops[NOP_ATOMIC5]
with
STATIC_KEY_INIT_NOPand get rid of the whole "let's optimize the atomic 5-byte nop" entirely.
Hmm?
So we have:
GENERIC (x86_32): leal ds:0x00(,%esi,1),%esi
K8 (x86_64): osp osp osp osp nop
K7 (x86_32): leal ds:0x00(,%eax,1),%eax
P6 (x86_64): nopl 0x00(%eax,%eax,1)
And I guess the $64k question is if there's any actual performance difference between the k8 and p6 variants on chips we still care about.
Most modern chips seem to end up selecting p6.
Anyway, the proposed patch looks like so:
---
Subject: x86: Remove ideal_nops[NOP_ATOMIC5]
By picking a single/fixed NOP5_ATOMIC instruction things become simpler.
--- arch/x86/include/asm/jump_label.h | 12 +++--------- arch/x86/include/asm/nops.h | 17 ++++++++--------- arch/x86/kernel/alternative.c | 8 -------- arch/x86/kernel/ftrace.c | 3 ++- arch/x86/kernel/jump_label.c | 37 ++++++------------------------------- arch/x86/kernel/kprobes/core.c | 3 ++- 6 files changed, 21 insertions(+), 59 deletions(-)
arch/x86/include/asm/jump_label.h +3 -9
@@ -4,12 +4,6 @@#define JUMP_LABEL_NOP_SIZE 5 -#ifdef CONFIG_X86_64 -# define STATIC_KEY_INIT_NOP P6_NOP5_ATOMIC -#else -# define STATIC_KEY_INIT_NOP GENERIC_NOP5_ATOMIC -#endif -#include <asm/asm.h> #include <asm/nops.h>
@@ -21,7 +15,7 @@ static __always_inline bool arch_static_branch(struct static_key *key, bool branch) { asm_volatile_goto("1:" - ".byte " __stringify(STATIC_KEY_INIT_NOP) "\n\t" + ".byte " __stringify(NOP5_ATOMIC) "\n\t"".pushsection __jump_table, \"aw\" \n\t" _ASM_ALIGN"\n\t" ".long 1b - ., %l[l_yes] - . \n\t"
@@ -61,7 +55,7 @@ static __always_inline bool arch_static_branch_jump(struct static_key *key, bool .long \target - .Lstatic_jump_after_\@ .Lstatic_jump_after_\@: .else - .byte STATIC_KEY_INIT_NOP + .byte NOP5_ATOMIC .endif .pushsection __jump_table, "aw" _ASM_ALIGN
@@ -73,7 +67,7 @@ static __always_inline bool arch_static_branch_jump(struct static_key *key, bool .macro STATIC_JUMP_IF_FALSE target, key, def .Lstatic_jump_\@: .if \def - .byte STATIC_KEY_INIT_NOP + .byte NOP5_ATOMIC .else/* Equivalent to "jmp.d32 \target" */ .byte 0xe9
arch/x86/include/asm/nops.h +8 -9
@@ -28,7 +28,6 @@#define GENERIC_NOP6 0x8d,0xb6,0x00,0x00,0x00,0x00#define GENERIC_NOP7 0x8d,0xb4,0x26,0x00,0x00,0x00,0x00#define GENERIC_NOP8 GENERIC_NOP1,GENERIC_NOP7 -#define GENERIC_NOP5_ATOMIC NOP_DS_PREFIX,GENERIC_NOP4/* Opteron 64bit nops 1: nop
@@ -44,7 +43,6 @@#define K8_NOP6 K8_NOP3,K8_NOP3#define K8_NOP7 K8_NOP4,K8_NOP3#define K8_NOP8 K8_NOP4,K8_NOP4 -#define K8_NOP5_ATOMIC 0x66,K8_NOP4/* K7 nops uses eax dependencies (arbitrary choice)
@@ -63,7 +61,6 @@#define K7_NOP6 0x8d,0x80,0,0,0,0#define K7_NOP7 0x8D,0x04,0x05,0,0,0,0#define K7_NOP8 K7_NOP7,K7_NOP1 -#define K7_NOP5_ATOMIC NOP_DS_PREFIX,K7_NOP4/* P6 nops uses eax dependencies (Intel-recommended choice)
@@ -86,7 +83,12 @@#define P6_NOP6 0x66,0x0f,0x1f,0x44,0x00,0#define P6_NOP7 0x0f,0x1f,0x80,0,0,0,0#define P6_NOP8 0x0f,0x1f,0x84,0x00,0,0,0,0 -#define P6_NOP5_ATOMIC P6_NOP5 + +#ifdef CONFIG_X86_64 +#define NOP5_ATOMIC P6_NOP5 +#else +#define NOP5_ATOMIC NOP_DS_PREFIX,GENERIC_NOP4 +#endif #ifdef __ASSEMBLY__#define _ASM_MK_NOP (x) .byte x
@@ -103,7 +105,6 @@#define ASM_NOP6 _ASM_MK_NOP(K7_NOP6)#define ASM_NOP7 _ASM_MK_NOP(K7_NOP7)#define ASM_NOP8 _ASM_MK_NOP(K7_NOP8) -#define ASM_NOP5_ATOMIC _ASM_MK_NOP(K7_NOP5_ATOMIC)#elif defined(CONFIG_X86_P6_NOP )#define ASM_NOP1 _ASM_MK_NOP(P6_NOP1)#define ASM_NOP2 _ASM_MK_NOP(P6_NOP2)
@@ -113,7 +114,6 @@#define ASM_NOP6 _ASM_MK_NOP(P6_NOP6)#define ASM_NOP7 _ASM_MK_NOP(P6_NOP7)#define ASM_NOP8 _ASM_MK_NOP(P6_NOP8) -#define ASM_NOP5_ATOMIC _ASM_MK_NOP(P6_NOP5_ATOMIC)#elif defined(CONFIG_X86_64 )#define ASM_NOP1 _ASM_MK_NOP(K8_NOP1)#define ASM_NOP2 _ASM_MK_NOP(K8_NOP2)
@@ -123,7 +123,6 @@#define ASM_NOP6 _ASM_MK_NOP(K8_NOP6)#define ASM_NOP7 _ASM_MK_NOP(K8_NOP7)#define ASM_NOP8 _ASM_MK_NOP(K8_NOP8) -#define ASM_NOP5_ATOMIC _ASM_MK_NOP(K8_NOP5_ATOMIC)#else #define ASM_NOP1 _ASM_MK_NOP(GENERIC_NOP1)#define ASM_NOP2 _ASM_MK_NOP(GENERIC_NOP2)
@@ -133,11 +132,11 @@#define ASM_NOP6 _ASM_MK_NOP(GENERIC_NOP6)#define ASM_NOP7 _ASM_MK_NOP(GENERIC_NOP7)#define ASM_NOP8 _ASM_MK_NOP(GENERIC_NOP8) -#define ASM_NOP5_ATOMIC _ASM_MK_NOP(GENERIC_NOP5_ATOMIC)#endif +#define ASM_NOP5_ATOMIC _ASM_MK_NOP(NOP5_ATOMIC) +#define ASM_NOP_MAX 8 -#define NOP_ATOMIC5 (ASM_NOP_MAX+1)/* Entry for the 5-byte atomic NOP */ #ifndef __ASSEMBLY__extern const unsigned char *const *ideal_nops;
arch/x86/kernel/alternative.c +0 -8
@@ -90,7 +90,6 @@ static const unsigned char intelnops[] =GENERIC_NOP6 ,GENERIC_NOP7 , GENERIC_NOP8, -GENERIC_NOP5_ATOMIC };static const unsigned char *const intel_nops[ASM_NOP_MAX +2 ] = {
@@ -103,7 +102,6 @@ static const unsigned char * const intel_nops[ASM_NOP_MAX+2] = intelnops +1 +2 +3 +4 +5 , intelnops +1 +2 +3 +4 +5 +6 , intelnops + 1 + 2 + 3 + 4 + 5 + 6 + 7, - intelnops + 1 + 2 + 3 + 4 + 5 + 6 + 7 + 8, };#endif
@@ -118,7 +116,6 @@ static const unsigned char k8nops[] =K8_NOP6 ,K8_NOP7 , K8_NOP8, -K8_NOP5_ATOMIC };static const unsigned char *const k8_nops[ASM_NOP_MAX +2 ] = {
@@ -131,7 +128,6 @@ static const unsigned char * const k8_nops[ASM_NOP_MAX+2] = k8nops +1 +2 +3 +4 +5 , k8nops +1 +2 +3 +4 +5 +6 , k8nops + 1 + 2 + 3 + 4 + 5 + 6 + 7, - k8nops + 1 + 2 + 3 + 4 + 5 + 6 + 7 + 8, };#endif
@@ -146,7 +142,6 @@ static const unsigned char k7nops[] =K7_NOP6 ,K7_NOP7 , K7_NOP8, -K7_NOP5_ATOMIC };static const unsigned char *const k7_nops[ASM_NOP_MAX +2 ] = {
@@ -159,7 +154,6 @@ static const unsigned char * const k7_nops[ASM_NOP_MAX+2] = k7nops +1 +2 +3 +4 +5 , k7nops +1 +2 +3 +4 +5 +6 , k7nops + 1 + 2 + 3 + 4 + 5 + 6 + 7, - k7nops + 1 + 2 + 3 + 4 + 5 + 6 + 7 + 8, };#endif
@@ -174,7 +168,6 @@ static const unsigned char p6nops[] =P6_NOP6 ,P6_NOP7 , P6_NOP8, -P6_NOP5_ATOMIC };static const unsigned char *const p6_nops[ASM_NOP_MAX +2 ] = {
@@ -187,7 +180,6 @@ static const unsigned char * const p6_nops[ASM_NOP_MAX+2] = p6nops +1 +2 +3 +4 +5 , p6nops +1 +2 +3 +4 +5 +6 , p6nops + 1 + 2 + 3 + 4 + 5 + 6 + 7, - p6nops + 1 + 2 + 3 + 4 + 5 + 6 + 7 + 8, };#endif
arch/x86/kernel/ftrace.c +2 -1
@@ -101,7 +101,8 @@ static unsigned long text_ip_addr(unsigned long ip)static const unsigned char *ftrace_nop_replace (void ) { -return ideal_nops[NOP_ATOMIC5 ]; +static const unsigned char nop[] = {NOP5_ATOMIC }; +return nop; } static int
arch/x86/kernel/jump_label.c +6 -31
@@ -40,8 +40,7 @@ static void __ref __jump_label_transform(struct jump_entry *entry, int init) {union jump_code_union jmp; -const unsigned char default_nop[] = {STATIC_KEY_INIT_NOP }; -const unsigned char *ideal_nop = ideal_nops[NOP_ATOMIC5 ]; +const unsigned char nop[] = {NOP5_ATOMIC };const void *expect, *code;int line;
@@ -50,21 +49,13 @@ static void __ref __jump_label_transform(struct jump_entry *entry, (jump_entry_code (entry) +JUMP_LABEL_NOP_SIZE );if (type ==JUMP_LABEL_JMP ) { -if (init) { - expect = default_nop; line = __LINE__; - }else { - expect = ideal_nop; line = __LINE__; - } - + expect = nop; + line = __LINE__; code = &jmp.code; }else { -if (init) { - expect = default_nop; line = __LINE__; - }else { - expect = &jmp.code; line = __LINE__; - } - - code = ideal_nop; + expect = &jmp.code; + line = __LINE__; + code = nop; }if (memcmp ((void *)jump_entry_code (entry), expect,JUMP_LABEL_NOP_SIZE ))
@@ -108,22 +99,6 @@ static enum { __init_or_module void arch_jump_label_transform_static(struct jump_entry *entry,enum jump_label_type type) { -/* -* This function is called at boot up and when modules are -* first loaded. Check if the default nop, the one that is -* inserted at compile time, is the ideal nop. If it is, then -* we do not need to update the nop, and we can leave it as is. -* If it is not, then we need to update the nop to the ideal nop. -*/ -if (jlstate ==JL_STATE_START ) { -const unsigned char default_nop[] = {STATIC_KEY_INIT_NOP }; -const unsigned char *ideal_nop = ideal_nops[NOP_ATOMIC5 ]; - -if (memcmp (ideal_nop, default_nop,5 ) !=0 ) - jlstate =JL_STATE_UPDATE ; -else - jlstate =JL_STATE_NO_UPDATE ; - }if (jlstate ==JL_STATE_UPDATE )__jump_label_transform (entry, type,1 ); }
arch/x86/kernel/kprobes/core.c +2 -1
@@ -204,6 +204,7 @@ int can_boost(struct insn *insn, void *addr)static unsigned long __recover_probed_insn (kprobe_opcode_t *buf,unsigned long addr) { +static const unsigned char nop[] = {NOP5_ATOMIC };struct kprobe *kp;unsigned long faddr;
@@ -247,7 +248,7 @@ __recover_probed_insn(kprobe_opcode_t *buf, unsigned long addr)return 0UL ;if (faddr) -memcpy (buf, ideal_nops[NOP_ATOMIC5 ],5 ); +memcpy (buf, nop,5 );else buf[0 ] = kp->opcode;return (unsigned long )buf;
Re: [PATCH 08/15] x86/alternatives: Teach text_poke_bp() to emulate instructions
In reply to Peter ZijlstraQuoted 10 lines · Peter Zijlstra
Quoted 4 lines · Masami Hiramatsu
Quoted 3 lines
This fits almost all
text_poke_bp()users, exceptarch_unoptimize_kprobe()which restores random text, and for that site we have to build an explicit emulate instruction.
Quoted 9 lines
Quoted 4 lines
Hm, actually it doesn't restores randome text, since the first byte must always be int3. As the function name means, it just unoptimizes (jump based optprobe -> int3 based kprobe).
Anyway, that is not an issue. With this patch, optprobe must still work.I thought it basically restored 5 bytes of original text (with no guarantee it is a single instruction, or even a complete instruction), with the first byte replaced with INT3.
I am surely missing some kprobe context, but is it really safe to use this mechanism to replace more than one instruction?
[email protected]
reviewerRe: [PATCH 08/15] x86/alternatives: Teach text_poke_bp() to emulate instructions
Quoted 18 lines · Andy Lutomirski
Quoted 3 lines
On Jun 7, 2019, at 10:34 AM, Peter Zijlstra <[email protected]>
Quoted 5 lines · wrote:
Quoted 2 lines · Masami Hiramatsu
Quoted 2 lines
This fits almost all
text_poke_bp()users, exceptarch_unoptimize_kprobe()which restores random text, and for thatsite
Quoted 6 lines
Quoted 6 lines
Quoted 1 line
we have to build an explicit emulate instruction.
Hm, actually it doesn't restores randome text, since the first byte must always be int3. As the function name means, it just unoptimizes (jump based optprobe -> int3 based kprobe).
Anyway, that is not an issue. With this patch, optprobe must stillwork.
Quoted 1 line
Quoted 8 lines
Quoted 2 lines
I thought it basically restored 5 bytes of original text (with no guarantee it is a single instruction, or even a complete
instruction),
Quoted 2 lines
with the first byte replaced with INT3.
I am surely missing some kprobe context, but is it really safe to use this mechanism to replace more than one instruction?
I don't see how it could be, except *perhaps* inside an NMI have, because you could have a preempted or interrupted now having an in-memory IP pointing inside the middle of the region you are intending to patch.
Signature · 1 line
Sent from my Android device with K-9 Mail. Please excuse my brevity.
Peter Zijlstra
maintainerRe: [PATCH 08/15] x86/alternatives: Teach text_poke_bp() to emulate instructions
In reply to Andy LutomirskiQuoted 15 lines · Andy Lutomirski
Quoted 15 lines · Peter Zijlstra
Quoted 8 lines · Masami Hiramatsu
Quoted 3 lines
This fits almost all
text_poke_bp()users, exceptarch_unoptimize_kprobe()which restores random text, and for that site we have to build an explicit emulate instruction.Hm, actually it doesn't restores randome text, since the first byte must always be int3. As the function name means, it just unoptimizes (jump based optprobe -> int3 based kprobe).
Anyway, that is not an issue. With this patch, optprobe must still work.
Quoted 7 lines
Quoted 4 lines
I thought it basically restored 5 bytes of original text (with no guarantee it is a single instruction, or even a complete instruction), with the first byte replaced with INT3.
I am surely missing some kprobe context, but is it really safe to use this mechanism to replace more than one instruction?
I'm not entirely up-to-scratch here, so Masami, please correct me if I'm wrong.
So what happens is that arch_prepare_optimized_kprobe() <- copy_optimized_instructions() copies however much of the instruction stream is required such that we can overwrite the instruction at @addr with a 5 byte jump.
arch_optimize_kprobe() then does the text_poke_bp() that replaces the instruction @addr with int3, copies the rel jump address and overwrites the int3 with jmp.
And I'm thinking the problem is with something like:
@addr: nop nop nop nop nop
We copy out the nops into the trampoline, overwrite the first nop with an INT3, overwrite the remaining nops with the rel addr, but oops, another CPU can still be executing one of those NOPs, right?
I'm thinking we could fix this by first writing INT3 into all relevant instructions, which is going to be messy, given the current code base.
Peter Zijlstra
maintainerRe: [PATCH 08/15] x86/alternatives: Teach text_poke_bp() to emulate instructions
Quoted 19 lines · Peter Zijlstra
Quoted 2 lines · Andy Lutomirski
I am surely missing some kprobe context, but is it really safe to use this mechanism to replace more than one instruction?
I'm not entirely up-to-scratch here, so Masami, please correct me if I'm wrong.
So what happens is that
arch_prepare_optimized_kprobe()<-copy_optimized_instructions()copies however much of the instruction stream is required such that we can overwrite the instruction at @addr with a 5 byte jump.
arch_optimize_kprobe()then does thetext_poke_bp()that replaces the instruction @addr with int3, copies the rel jump address and overwrites the int3 with jmp.And I'm thinking the problem is with something like:
Quoted 8 lines
@addr: nop nop nop nop nop
We copy out the nops into the trampoline, overwrite the first nop with an INT3, overwrite the remaining nops with the rel addr, but oops, another CPU can still be executing one of those NOPs, right?
I'm thinking we could fix this by first writing INT3 into all relevant instructions, which is going to be messy, given the current code base.
Maybe not that bad; how's something like this?
(completely untested)
--- arch/x86/kernel/alternative.c | 20 ++++++++++++++++++-- 1 file changed, 18 insertions(+), 2 deletions(-)
arch/x86/kernel/alternative.c +18 -2
@@ -24,6 +24,7 @@#include <asm/tlbflush.h> #include <asm/io.h> #include <asm/fixmap.h> +#include <asm/insn.h> int __read_mostly alternatives_patched;
@@ -849,6 +850,7 @@ static void do_sync_core(void *info)static bool bp_patching_in_progress;static void *bp_int3_handler, *bp_int3_addr; +static unsigned int bp_int3_length;int poke_int3_handler (struct pt_regs *regs) {
@@ -867,7 +869,11 @@ int poke_int3_handler(struct pt_regs *regs)if (likely (!bp_patching_in_progress))return 0 ; -if (user_mode (regs) || regs->ip != (unsigned long )bp_int3_addr) +if (user_mode (regs)) +return 0 ; + +if (regs->ip < (unsigned long )bp_int3_addr || + regs->ip >= (unsigned long )bp_int3_addr + bp_int3_length)return 0 ;/* set up the specified breakpoint handler */
@@ -900,9 +906,12 @@ NOKPROBE_SYMBOL(poke_int3_handler);void text_poke_bp (void *addr,const void *opcode,size_t len,void *handler) {unsigned char int3 =0xcc ; +void *kaddr = addr; +struct insn insn; bp_int3_handler = handler; bp_int3_addr = (u8 *)addr +sizeof (int3); + bp_int3_length = len -sizeof (int3); bp_patching_in_progress = true;lockdep_assert_held (&text_mutex);
@@ -913,7 +922,14 @@ void text_poke_bp(void *addr, const void *opcode, size_t len, void *handler)*/ smp_wmb (); -text_poke (addr, &int3,sizeof (int3)); +do { +kernel_insn_init (&insn, kaddr,MAX_INSN_SIZE ); +insn_get_length (&insn); + +text_poke (kaddr, &int3,sizeof (int3)); + + kaddr += insn.length; + }while (kaddr < addr + len);on_each_cpu (do_sync_core,NULL ,1 );
Peter Zijlstra
maintainerRe: [PATCH 08/15] x86/alternatives: Teach text_poke_bp() to emulate instructions
Quoted 58 lines · Peter Zijlstra
Quoted 19 lines · Peter Zijlstra
Quoted 2 lines · Andy Lutomirski
I am surely missing some kprobe context, but is it really safe to use this mechanism to replace more than one instruction?
I'm not entirely up-to-scratch here, so Masami, please correct me if I'm wrong.
So what happens is that
arch_prepare_optimized_kprobe()<-copy_optimized_instructions()copies however much of the instruction stream is required such that we can overwrite the instruction at @addr with a 5 byte jump.
arch_optimize_kprobe()then does thetext_poke_bp()that replaces the instruction @addr with int3, copies the rel jump address and overwrites the int3 with jmp.And I'm thinking the problem is with something like:
Quoted 8 lines
@addr: nop nop nop nop nop
We copy out the nops into the trampoline, overwrite the first nop with an INT3, overwrite the remaining nops with the rel addr, but oops, another CPU can still be executing one of those NOPs, right?
I'm thinking we could fix this by first writing INT3 into all relevant instructions, which is going to be messy, given the current code base.
Maybe not that bad; how's something like this?
(completely untested)
--- arch/x86/kernel/alternative.c | 20 ++++++++++++++++++-- 1 file changed, 18 insertions(+), 2 deletions(-)arch/x86/kernel/alternative.c +2
-0@@ -24,6 +24,7 @@#include <asm/tlbflush.h> #include <asm/io.h> #include <asm/fixmap.h> +#include <asm/insn.h> int __read_mostly alternatives_patched;@@ -849,6 +850,7 @@ static void do_sync_core(void *info)static bool bp_patching_in_progress;static void *bp_int3_handler, *bp_int3_addr; +static unsigned int bp_int3_length;int poke_int3_handler (struct pt_regs *regs) {
arch/x86/kernel/alternative.c int poke_int3_handler(struct pt_regs *regs) · line 877
Quoted 10 lines
arch/x86/kernel/alternative.c +5
-1@@ -867,7 +869,11 @@ int poke_int3_handler(struct pt_regs *regs)if (likely (!bp_patching_in_progress))return 0 ; -if (user_mode (regs) || regs->ip != (unsigned long )bp_int3_addr) +if (user_mode (regs)) +return 0 ; + +if (regs->ip < (unsigned long )bp_int3_addr || + regs->ip >= (unsigned long )bp_int3_addr + bp_int3_length)return 0 ;
Bugger, this isn't right. It'll jump to the beginning of the trampoline, even if it is multiple instructions in, which would lead to executing instructions twice, which would be BAD.
_maybe_, depending on what the slot looks like, we could do something like:
offset = regs->ip - (unsigned long)bp_int3_addr; regs->ip = bp_int3_handler + offset;
That is; jump into the slot at the same offset we hit the INT3, but this is quickly getting yuck.
Quoted 30 lines
arch/x86/kernel/alternative.c +11
-1/* set up the specified breakpoint handler */ @@ -900,9 +906,12 @@ NOKPROBE_SYMBOL(poke_int3_handler);void text_poke_bp (void *addr,const void *opcode,size_t len,void *handler) {unsigned char int3 =0xcc ; +void *kaddr = addr; +struct insn insn; bp_int3_handler = handler; bp_int3_addr = (u8 *)addr +sizeof (int3); + bp_int3_length = len -sizeof (int3); bp_patching_in_progress = true;lockdep_assert_held (&text_mutex);@@ -913,7 +922,14 @@ void text_poke_bp(void *addr, const void *opcode, size_t len, void *handler)*/ smp_wmb (); -text_poke (addr, &int3,sizeof (int3)); +do { +kernel_insn_init (&insn, kaddr,MAX_INSN_SIZE ); +insn_get_length (&insn); + +text_poke (kaddr, &int3,sizeof (int3)); + + kaddr += insn.length; + }while (kaddr < addr + len);on_each_cpu (do_sync_core,NULL ,1 );
Peter Zijlstra
maintainerRe: [PATCH 08/15] x86/alternatives: Teach text_poke_bp() to emulate instructions
Quoted 4 lines · Peter Zijlstra
Bugger, this isn't right. It'll jump to the beginning of the trampoline, even if it is multiple instructions in, which would lead to executing instructions twice, which would be BAD.
Quoted 8 lines
_maybe_, depending on what the slot looks like, we could do something like:
offset = regs->ip - (unsigned long)bp_int3_addr; regs->ip = bp_int3_handler + offset;That is; jump into the slot at the same offset we hit the INT3, but this is quickly getting yuck.
Yeah, that won't work either... it needs something far more complex :/
Steven Rostedt
maintainerRe: [PATCH 08/15] x86/alternatives: Teach text_poke_bp() to emulate instructions
In reply to Peter ZijlstraQuoted 5 lines · On Tue, 11 Jun 2019 10:03:07 +0200 Peter Zijlstra <[email protected]> wrote:
So what happens is that
arch_prepare_optimized_kprobe()<-copy_optimized_instructions()copies however much of the instruction stream is required such that we can overwrite the instruction at @addr with a 5 byte jump.
Quoted 7 lines
arch_optimize_kprobe()then does thetext_poke_bp()that replaces the instruction @addr with int3, copies the rel jump address and overwrites the int3 with jmp.And I'm thinking the problem is with something like:
@addr: nop nop nop nop nop
What would work would be to:
add breakpoint to first opcode. call synchronize_tasks(); /* All tasks now hitting breakpoint and jumping over affected code */ update the rest of the instructions. replace breakpoint with jmp.
One caveat is that the replaced instructions must not be a call function. As if the call function calls schedule then it will circumvent the synchronize_tasks(). It would be OK if that call is the last of the instructions. But I doubt we modify anything more then a call size anyway, so this should still work for all current instances.
-- Steve
Quoted 7 lines
We copy out the nops into the trampoline, overwrite the first nop with an INT3, overwrite the remaining nops with the rel addr, but oops, another CPU can still be executing one of those NOPs, right?
I'm thinking we could fix this by first writing INT3 into all relevant instructions, which is going to be messy, given the current code base.
Steven Rostedt
maintainerRe: [PATCH 08/15] x86/alternatives: Teach text_poke_bp() to emulate instructions
Quoted 5 lines · On Tue, 11 Jun 2019 11:22:54 -0400 Steven Rostedt <[email protected]> wrote:
What would work would be to:
add breakpoint to first opcode. call synchronize_tasks();
BTW, that should be "synchronize_rcu_tasks()"
-- Steve
Quoted 7 lines
/* All tasks now hitting breakpoint and jumping over affected code */ update the rest of the instructions. replace breakpoint with jmp.
Peter Zijlstra
maintainerRe: [PATCH 08/15] x86/alternatives: Teach text_poke_bp() to emulate instructions
In reply to Steven RostedtQuoted 30 lines · Steven Rostedt
Quoted 5 lines · On Tue, 11 Jun 2019 10:03:07 +0200 Peter Zijlstra <[email protected]> wrote:
So what happens is that
arch_prepare_optimized_kprobe()<-copy_optimized_instructions()copies however much of the instruction stream is required such that we can overwrite the instruction at @addr with a 5 byte jump.Quoted 7 lines
arch_optimize_kprobe()then does thetext_poke_bp()that replaces the instruction @addr with int3, copies the rel jump address and overwrites the int3 with jmp.And I'm thinking the problem is with something like:
@addr: nop nop nop nop nop
What would work would be to:
add breakpoint to first opcode. call synchronize_tasks(); /* All tasks now hitting breakpoint and jumping over affected code */ update the rest of the instructions. replace breakpoint with jmp.
Quoted 5 lines
One caveat is that the replaced instructions must not be a call function. As if the call function calls schedule then it will circumvent the
synchronize_tasks(). It would be OK if that call is the last of the instructions. But I doubt we modify anything more then a call size anyway, so this should still work for all current instances.
Right, something like this could work (although I cannot currently find synchronize_tasks), but it would make the optprobe stuff fairly slow (iirc this sync_tasks() thing could be pretty horrible).
Re: [PATCH 08/15] x86/alternatives: Teach text_poke_bp() to emulate instructions
Quoted 39 lines · Peter Zijlstra
Quoted 30 lines · Steven Rostedt
Quoted 5 lines · On Tue, 11 Jun 2019 10:03:07 +0200 Peter Zijlstra <[email protected]> wrote:
So what happens is that
arch_prepare_optimized_kprobe()<-copy_optimized_instructions()copies however much of the instruction stream is required such that we can overwrite the instruction at @addr with a 5 byte jump.Quoted 7 lines
arch_optimize_kprobe()then does thetext_poke_bp()that replaces the instruction @addr with int3, copies the rel jump address and overwrites the int3 with jmp.And I'm thinking the problem is with something like:
@addr: nop nop nop nop nop
What would work would be to:
add breakpoint to first opcode. call synchronize_tasks(); /* All tasks now hitting breakpoint and jumping over affected code */ update the rest of the instructions. replace breakpoint with jmp.Quoted 5 lines
One caveat is that the replaced instructions must not be a call function. As if the call function calls schedule then it will circumvent the
synchronize_tasks(). It would be OK if that call is the last of the instructions. But I doubt we modify anything more then a call size anyway, so this should still work for all current instances.
Quoted 3 lines
Right, something like this could work (although I cannot currently find
synchronize_tasks), but it would make the optprobe stuff fairly slow (iirc thissync_tasks()thing could be pretty horrible).
I have run into similar problems before.
I had two problematic scenarios. In the first case, I had a “call” in the middle of the patched code-block, but this call was always followed by a “jump” to the end of the potentially patched code-block, so I did not have the problem.
In the second case, I had an indirect call (which is shorter than a direct call) being patched into a direct call. In this case, I preceded the indirect call with NOPs so indeed the indirect call was at the end of the patched block.
In certain cases, if a shorter instruction should be potentially patched into a longer one, the shorter one can be preceded by some prefixes. If there are multiple REX prefixes, for instance, the CPU only uses the last one, IIRC. This can allow to avoid synchronize_sched() when patching a single instruction into another instruction with a different length.
Not sure how helpful this information is, but sharing - just in case.
Peter Zijlstra
maintainerRe: [PATCH 08/15] x86/alternatives: Teach text_poke_bp() to emulate instructions
Quoted 8 lines · Nadav Amit
I have run into similar problems before.
I had two problematic scenarios. In the first case, I had a “call” in the middle of the patched code-block, but this call was always followed by a “jump” to the end of the potentially patched code-block, so I did not have the problem.
In the second case, I had an indirect call (which is shorter than a direct
Longer, 6 bytes vs 5 if I'm not mistaken.
Quoted 11 lines
call) being patched into a direct call. In this case, I preceded the indirect call with NOPs so indeed the indirect call was at the end of the patched block.
In certain cases, if a shorter instruction should be potentially patched into a longer one, the shorter one can be preceded by some prefixes. If there are multiple REX prefixes, for instance, the CPU only uses the last one, IIRC. This can allow to avoid
synchronize_sched()when patching a single instruction into another instruction with a different length.Not sure how helpful this information is, but sharing - just in case.
I think we can patch multiple instructions provided:
- all but one instruction are a NOP,
- there are no branch targets inside the range.
By poking INT3 at every instruction in the range and then doing the machine wide IPI+SYNC, we'll trap every CPU that is in-side the range.
Because all but one instruction are a NOP, we can emulate only the one instruction (assuming the real instruction is always last), otherwise NOP when we're behind the real instruction.
Then we can write new instructions, leaving the initial INT3 until last.
Something like this might be useful if we want to support immediate instructions (like patch_data_* in paravirt_patch.c) for static_call().
Re: [PATCH 08/15] x86/alternatives: Teach text_poke_bp() to emulate instructions
Quoted 6 lines · Peter Zijlstra
Quoted 2 lines · Nadav Amit
I have run into similar problems before.
Quoted 8 lines
Quoted 6 lines
I had two problematic scenarios. In the first case, I had a “call” in the middle of the patched code-block, but this call was always followed by a “jump” to the end of the potentially patched code-block, so I did not have the problem.
In the second case, I had an indirect call (which is shorter than a direct
Longer, 6 bytes vs 5 if I'm not mistaken.
Shorter (2-3 bytes IIRC), since the target was held in a register.
Quoted 21 lines
Quoted 5 lines
call) being patched into a direct call. In this case, I preceded the indirect call with NOPs so indeed the indirect call was at the end of the patched block.
Quoted 7 lines
In certain cases, if a shorter instruction should be potentially patched into a longer one, the shorter one can be preceded by some prefixes. If there are multiple REX prefixes, for instance, the CPU only uses the last one, IIRC. This can allow to avoid
synchronize_sched()when patching a single instruction into another instruction with a different length.Not sure how helpful this information is, but sharing - just in case.
I think we can patch multiple instructions provided:
- all but one instruction are a NOP,
- there are no branch targets inside the range.By poking INT3 at every instruction in the range and then doing the machine wide IPI+SYNC, we'll trap every CPU that is in-side the range.
Quoted 8 lines
Because all but one instruction are a NOP, we can emulate only the one instruction (assuming the real instruction is always last), otherwise NOP when we're behind the real instruction.
Then we can write new instructions, leaving the initial INT3 until last.
Something like this might be useful if we want to support immediate instructions (like patch_data_* in
paravirt_patch.c) forstatic_call().
I don’t know what you regard when you say SYNC, but if you regard something like sync_core() (in contrast to something like synchronize_sched() ), I am not sure it is sufficient.
Using IPI+sync_core(), I think, would make an assumption that IRQs are never enabled inside IRQ and exception handlers, or that these handlers would not be invoked while the patched code is executed. Otherwise, the IPI might be received inside the IRQ/exception handler, and then return from the handler will be into the middle of a patched instruction.
Andy Lutomirski
maintainerRe: [PATCH 08/15] x86/alternatives: Teach text_poke_bp() to emulate instructions
In reply to Peter ZijlstraQuoted 26 lines · Peter Zijlstra
Quoted 8 lines · Nadav Amit
I have run into similar problems before.
I had two problematic scenarios. In the first case, I had a “call” in the middle of the patched code-block, but this call was always followed by a “jump” to the end of the potentially patched code-block, so I did not have the problem.
In the second case, I had an indirect call (which is shorter than a direct
Longer, 6 bytes vs 5 if I'm not mistaken.
Quoted 11 lines
call) being patched into a direct call. In this case, I preceded the indirect call with NOPs so indeed the indirect call was at the end of the patched block.
In certain cases, if a shorter instruction should be potentially patched into a longer one, the shorter one can be preceded by some prefixes. If there are multiple REX prefixes, for instance, the CPU only uses the last one, IIRC. This can allow to avoid
synchronize_sched()when patching a single instruction into another instruction with a different length.Not sure how helpful this information is, but sharing - just in case.
Quoted 7 lines
I think we can patch multiple instructions provided:
- all but one instruction are a NOP,
- there are no branch targets inside the range.By poking INT3 at every instruction in the range and then doing the machine wide IPI+SYNC, we'll trap every CPU that is in-side the range.
How do you know you'll trap them? You need to IPI, serialize, and get them to execute an instruction. If the CPU is in an interrupt and RIP just happens to be pointed to the INT3, you need them to execute a whole lot more than just one instruction.
Peter Zijlstra
maintainerRe: [PATCH 08/15] x86/alternatives: Teach text_poke_bp() to emulate instructions
Quoted 32 lines · Andy Lutomirski
Quoted 31 lines · Peter Zijlstra
Quoted 8 lines · Nadav Amit
I have run into similar problems before.
I had two problematic scenarios. In the first case, I had a “call” in the middle of the patched code-block, but this call was always followed by a “jump” to the end of the potentially patched code-block, so I did not have the problem.
In the second case, I had an indirect call (which is shorter than a direct
Longer, 6 bytes vs 5 if I'm not mistaken.
Quoted 11 lines
call) being patched into a direct call. In this case, I preceded the indirect call with NOPs so indeed the indirect call was at the end of the patched block.
In certain cases, if a shorter instruction should be potentially patched into a longer one, the shorter one can be preceded by some prefixes. If there are multiple REX prefixes, for instance, the CPU only uses the last one, IIRC. This can allow to avoid
synchronize_sched()when patching a single instruction into another instruction with a different length.Not sure how helpful this information is, but sharing - just in case.
I think we can patch multiple instructions provided:
- all but one instruction are a NOP,
- there are no branch targets inside the range.
Quoted 7 lines
Quoted 2 lines
By poking INT3 at every instruction in the range and then doing the machine wide IPI+SYNC, we'll trap every CPU that is in-side the range.
How do you know you'll trap them? You need to IPI, serialize, and get them to execute an instruction. If the CPU is in an interrupt and RIP just happens to be pointed to the INT3, you need them to execute a whole lot more than just one instruction.
Argh, yes, I'm an idiot.
Re: [PATCH 08/15] x86/alternatives: Teach text_poke_bp() to emulate instructions
In reply to Peter ZijlstraQuoted 42 lines · Peter Zijlstra
Quoted 17 lines · Andy Lutomirski
Quoted 15 lines · Peter Zijlstra
Quoted 8 lines · Masami Hiramatsu
Quoted 3 lines
This fits almost all
text_poke_bp()users, exceptarch_unoptimize_kprobe()which restores random text, and for that site we have to build an explicit emulate instruction.Hm, actually it doesn't restores randome text, since the first byte must always be int3. As the function name means, it just unoptimizes (jump based optprobe -> int3 based kprobe).
Anyway, that is not an issue. With this patch, optprobe must still work.Quoted 7 lines
Quoted 4 lines
I thought it basically restored 5 bytes of original text (with no guarantee it is a single instruction, or even a complete instruction), with the first byte replaced with INT3.
I am surely missing some kprobe context, but is it really safe to use this mechanism to replace more than one instruction?
I'm not entirely up-to-scratch here, so Masami, please correct me if I'm wrong.
So what happens is that
arch_prepare_optimized_kprobe()<-copy_optimized_instructions()copies however much of the instruction stream is required such that we can overwrite the instruction at @addr with a 5 byte jump.
arch_optimize_kprobe()then does thetext_poke_bp()that replaces the instruction @addr with int3, copies the rel jump address and overwrites the int3 with jmp.And I'm thinking the problem is with something like:
Quoted 8 lines
@addr: nop nop nop nop nop
We copy out the nops into the trampoline, overwrite the first nop with an INT3, overwrite the remaining nops with the rel addr, but oops, another CPU can still be executing one of those NOPs, right?
I'm thinking we could fix this by first writing INT3 into all relevant instructions, which is going to be messy, given the current code base.
How does that help? If RIP == x+2 and you want to put a 5-byte jump at address x, no amount of 0xcc is going to change the fact that RIP is in the middle of the jump.
Live patching can handle this by detecting this condition on each CPU, but performance won’t be great. Maybe some synchronize_sched trickery could help.
Steven Rostedt
maintainerRe: [PATCH 08/15] x86/alternatives: Teach text_poke_bp() to emulate instructions
Quoted 7 lines · On Tue, 11 Jun 2019 08:54:23 -0700 Andy Lutomirski <[email protected]> wrote:
How does that help? If RIP == x+2 and you want to put a 5-byte jump at address x, no amount of
0xccis going to change the fact that RIP is in the middle of the jump.Live patching can handle this by detecting this condition on each CPU, but performance won’t be great. Maybe some
synchronize_schedtrickery could help.
We have synchronize_rcu_tasks() which return after all tasks have either entered user space or did a voluntary schedule (was not preempted). Or have not run (still in a sleeping state).
That way we guarantee that all tasks are no longer on any trampoline or code paths that do not call schedule. I use this to free dynamically allocated trampolines used by ftrace. And kprobes uses this too for its own trampolines.
-- Steve
Peter Zijlstra
maintainerRe: [PATCH 08/15] x86/alternatives: Teach text_poke_bp() to emulate instructions
In reply to Andy LutomirskiQuoted 14 lines · Andy Lutomirski
Quoted 14 lines · Peter Zijlstra
arch_optimize_kprobe()then does thetext_poke_bp()that replaces the instruction @addr with int3, copies the rel jump address and overwrites the int3 with jmp.And I'm thinking the problem is with something like:
@addr: nop nop nop nop nop
We copy out the nops into the trampoline, overwrite the first nop with an INT3, overwrite the remaining nops with the rel addr, but oops, another CPU can still be executing one of those NOPs, right?
Quoted 6 lines
Quoted 2 lines
I'm thinking we could fix this by first writing INT3 into all relevant instructions, which is going to be messy, given the current code base.
How does that help? If RIP == x+2 and you want to put a 5-byte jump at address x, no amount of
0xccis going to change the fact that RIP is in the middle of the jump.
What I proposed was doing 0xCC on every instruction that's inside of [x,x+5). After that we do machine wide IPI+SYNC.
So if RIP is at x+2, then it will observe 0xCC and trigger the exception there.
Now, the problem is that my exception cannot recover from anything except NOPs, so while it could work for some corner cases, it doesn't work for the optkprobe case in general.
Only then do we write the JMP offset and again to IPI+SYNC; then we write the 0xE8 and again IPI-SYNC.
But at that point we already have the guarantee nobody is inside [x,x+5). That is, except if we can get to x+2 without first going through x, IOW if x+2 is a branch target we're screwed any which way around and the poke is never valid.
Is that something optkprobes checks? If so, where and how?
Peter Zijlstra
maintainerRe: [PATCH 08/15] x86/alternatives: Teach text_poke_bp() to emulate instructions
In reply to Peter ZijlstraQuoted 25 lines · Peter Zijlstra
Quoted 4 lines · Masami Hiramatsu
Quoted 4 lines
arch/x86/kernel/alternative.c +0
-0@@ -943,8 +949,21 @@ int poke_int3_handler(struct pt_regs *reif (user_mode (regs) || regs->ip != (unsigned long )bp_int3_addr)return 0 ;Quoted 16 lines
Quoted 4 lines
arch/x86/kernel/alternative.c +2
-2-/* set up the specified breakpoint handler */ - regs->ip = (unsigned long ) bp_int3_handler; + opcode = *(struct opcode *)bp_int3_opcode; +arch/x86/kernel/alternative.c int poke_int3_handler(struct pt_regs *re · line 963
Quoted 10 lines
arch/x86/kernel/alternative.c +10
-0+switch (opcode.insn) { +case 0xE8 :/* CALL */ +int3_emulate_call (regs, ip + opcode.rel); +break ; + +case 0xE9 :/* JMP */ +int3_emulate_jmp (regs, ip + opcode.rel); +break ; + +default :/* assume NOP */ Shouldn't we check whether it is actually NOP here?
I was/am lazy and didn't want to deal with:
Quoted 7 lines
arch/x86/include/asm/nops.h:#define
GENERIC_NOP5_ATOMICNOP_DS_PREFIX,GENERIC_NOP4 arch/x86/include/asm/nops.h:#defineK8_NOP5_ATOMIC0x66,K8_NOP4 arch/x86/include/asm/nops.h:#defineK7_NOP5_ATOMICNOP_DS_PREFIX,K7_NOP4 arch/x86/include/asm/nops.h:#defineP6_NOP5_ATOMICP6_NOP5But maybe we should check for all the various NOP5 variants and
BUG()on anything unexpected.
I realized we never actually poke a !ideal nop5_atomic, so I've added that to the latest versions.
Re: [PATCH 08/15] x86/alternatives: Teach text_poke_bp() to emulate instructions
In reply to Peter ZijlstraQuoted 10 lines · Peter Zijlstra
In preparation for
static_callsupport, teachtext_poke_bp()to emulate instructions, including CALL.The current
text_poke_bp()takes a @handler argument which is used as a jump target when the temporary INT3 is hit by a different CPU.When patching CALL instructions, this doesn't work because we'd miss the PUSH of the return address. Instead, teach
poke_int3_handler()to emulate an instruction, typically the instruction we're patching in.
Quoted 7 lines
This fits almost all
text_poke_bp()users, exceptarch_unoptimize_kprobe()which restores random text, and for that site we have to build an explicit emulate instruction.Cc Daniel Bristot de Oliveira <[email protected]>Cc Nadav Amit <[email protected]>Signed-off-by Peter Zijlstra (Intel) <[email protected]>
Signature · 1 line
Josh
Steven Rostedt
maintainerRe: [PATCH 08/15] x86/alternatives: Teach text_poke_bp() to emulate instructions
In reply to Peter ZijlstraQuoted 7 lines · On Wed, 05 Jun 2019 15:08:01 +0200 Peter Zijlstra <[email protected]> wrote:
arch/x86/kernel/alternative.c +2
-2-void text_poke_bp (void *addr,const void *opcode,size_t len,void *handler) +void text_poke_bp (void *addr,const void *opcode,size_t len,const void *emulate) {unsigned char int3 =0xcc ; - bp_int3_handler = handler; + bp_int3_opcode = emulate ?: opcode;
arch/x86/kernel/alternative.c NOKPROBE_SYMBOL(poke_int3_handler); · line 1006
Quoted 10 lines
arch/x86/kernel/alternative.c +4
-0bp_int3_addr = (u8 *)addr +sizeof (int3); bp_patching_in_progress = true;lockdep_assert_held (&text_mutex);/* +* poke_int3_handler() relies on @opcode being a 5 byte instruction; +* notably a JMP, CALL or NOP5_ATOMIC. +*/ +BUG_ON (len !=5 );
If we have a bug on here, why bother with passing in len at all? Just force it to be 5.
We could make it a WARN_ON() and return without doing anything.
This also prevents us from ever changing two byte jmps.
-- Steve
Quoted 9 lines
arch/x86/kernel/alternative.c +3
-2+ +/* * Corresponding read barrier in int3 notifier for making sure the -* in_progress and handler are correctly ordered wrt. patching. +* in_progress and opcode are correctly ordered wrt. patching. */ smp_wmb (); -
Peter Zijlstra
maintainerRe: [PATCH 08/15] x86/alternatives: Teach text_poke_bp() to emulate instructions
Quoted 15 lines · Steven Rostedt
Quoted 12 lines · On Wed, 05 Jun 2019 15:08:01 +0200 Peter Zijlstra <[email protected]> wrote:
arch/x86/kernel/alternative.c +2
-2-void text_poke_bp (void *addr,const void *opcode,size_t len,void *handler) +void text_poke_bp (void *addr,const void *opcode,size_t len,const void *emulate) {unsigned char int3 =0xcc ; - bp_int3_handler = handler; + bp_int3_opcode = emulate ?: opcode; bp_int3_addr = (u8 *)addr +sizeof (int3); bp_patching_in_progress = true;lockdep_assert_held (&text_mutex);
Quoted 8 lines
arch/x86/kernel/alternative.c NOKPROBE_SYMBOL(poke_int3_handler); · line 1006
Quoted 5 lines
arch/x86/kernel/alternative.c +4
-0/* +* poke_int3_handler() relies on @opcode being a 5 byte instruction; +* notably a JMP, CALL or NOP5_ATOMIC. +*/ +BUG_ON (len !=5 );If we have a bug on here, why bother with passing in len at all? Just force it to be 5.
Masami said the same.
Quoted 3 lines
We could make it a
WARN_ON()and return without doing anything.This also prevents us from ever changing two byte jmps.
It doesn't; that is, we'd need to add emulation for the 3 byte jump, but that'd be pretty trivial.
Peter Zijlstra
maintainerRe: [PATCH 08/15] x86/alternatives: Teach text_poke_bp() to emulate instructions
Quoted 25 lines · Peter Zijlstra
Quoted 15 lines · Steven Rostedt
Quoted 12 lines · On Wed, 05 Jun 2019 15:08:01 +0200 Peter Zijlstra <[email protected]> wrote:
(quoted diff) +2
-2-void text_poke_bp(void *addr, const void *opcode, size_t len, void *handler) +void text_poke_bp(void *addr, const void *opcode, size_t len, const void *emulate) { unsigned char int3 = 0xcc; - bp_int3_handler = handler; + bp_int3_opcode = emulate ?: opcode; bp_int3_addr = (u8 *)addr + sizeof(int3); bp_patching_in_progress = true; lockdep_assert_held(&text_mutex);Quoted 8 lines
Review
Quoted 5 lines
(quoted diff) +4
-0/* + * poke_int3_handler() relies on @opcode being a 5 byte instruction; + * notably a JMP, CALL or NOP5_ATOMIC. + */ + BUG_ON(len != 5);If we have a bug on here, why bother with passing in len at all? Just force it to be 5.
Quoted 8 lines
Masami said the same.
Quoted 3 lines
We could make it a
WARN_ON()and return without doing anything.This also prevents us from ever changing two byte jmps.
It doesn't; that is, we'd need to add emulation for the 3 byte jump, but that'd be pretty trivial.
I can't find a 3 byte jump on x86_64, I could only find a 2 byte one. But something like so should work I suppose, although at this point I'm thinking we should just used the instruction decode we have instead of playing iffy games with packed structures.
arch/x86/include/asm/text-patching.h +3 -0
@@ -57,6 +57,9 @@ static inline void int3_emulate_jmp(struct pt_regs *regs, unsigned long ip)#define JMP_INSN_SIZE 5#define JMP_INSN_OPCODE 0xE9 +#define JMP8_INSN_SIZE 2 +#define JMP8_INSN_OPCODE 0xEB +static inline void int3_emulate_push (struct pt_regs *regs,unsigned long val) {/*
arch/x86/kernel/alternative.c +38 -12
83 lines not on this page · raw message
Peter Zijlstra
maintainerRe: [PATCH 08/15] x86/alternatives: Teach text_poke_bp() to emulate instructions
Quoted 3 lines · Peter Zijlstra
although at this point I'm
thinking we should just used the instruction decode we have instead of playing iffy games with packed structures.
How's something like this? It accepts jmp/32 jmp/8 call and nop5_atomic.
---
Subject: x86/alternatives: Teach text_poke_bp() to emulate instructions
From: Peter Zijlstra <[email protected]>
Date: Wed Jun 5 10:48:37 CEST 2019
In preparation for static_call support, teach text_poke_bp() to emulate instructions, including CALL.
The current text_poke_bp() takes a @handler argument which is used as a jump target when the temporary INT3 is hit by a different CPU.
When patching CALL instructions, this doesn't work because we'd miss the PUSH of the return address. Instead, teach poke_int3_handler() to emulate an instruction, typically the instruction we're patching in.
This fits almost all text_poke_bp() users, except arch_unoptimize_kprobe() which restores random text, and for that site we have to build an explicit emulate instruction.
--- arch/x86/include/asm/text-patching.h | 15 +++++-- arch/x86/kernel/alternative.c | 73 +++++++++++++++++++++++++---------- arch/x86/kernel/jump_label.c | 3 - arch/x86/kernel/kprobes/opt.c | 11 +++-- 4 files changed, 75 insertions(+), 27 deletions(-)
arch/x86/include/asm/text-patching.h +12 -3
@@ -37,7 +37,7 @@ extern void text_poke_early(void *addr,extern void *text_poke (void *addr,const void *opcode,size_t len);extern void *text_poke_kgdb (void *addr,const void *opcode,size_t len);extern int poke_int3_handler (struct pt_regs *regs); -extern void text_poke_bp (void *addr,const void *opcode,size_t len,void *handler); +extern void text_poke_bp (void *addr,const void *opcode,size_t len,const void *emulate);extern int after_bootmem;extern __ro_after_init struct mm_struct *poking_mm;extern __ro_after_init unsigned long poking_addr;
@@ -48,8 +48,17 @@ static inline void int3_emulate_jmp(stru regs->ip = ip; } -#define INT3_INSN_SIZE 1 -#define CALL_INSN_SIZE 5 +#define INT3_INSN_SIZE 1 +#define INT3_INSN_OPCODE 0xCC + +#define CALL_INSN_SIZE 5 +#define CALL_INSN_OPCODE 0xE8 + +#define JMP_INSN_SIZE 5 +#define JMP_INSN_OPCODE 0xE9 + +#define JMP8_INSN_SIZE 2 +#define JMP8_INSN_OPCODE 0xEBstatic inline void int3_emulate_push (struct pt_regs *regs,unsigned long val) {
arch/x86/kernel/alternative.c +54 -19
117 lines not on this page · raw message
arch/x86/kernel/jump_label.c +1 -2
@@ -87,8 +87,7 @@ static void __ref __jump_label_transformreturn ; } -text_poke_bp ((void *)jump_entry_code (entry), code,JUMP_LABEL_NOP_SIZE , - (void *)jump_entry_code (entry) +JUMP_LABEL_NOP_SIZE ); +text_poke_bp ((void *)jump_entry_code (entry), code,JUMP_LABEL_NOP_SIZE ,NULL ); } void arch_jump_label_transform(struct jump_entry *entry,
arch/x86/kernel/kprobes/opt.c +8 -3
@@ -437,8 +437,7 @@ void arch_optimize_kprobes(struct list_h insn_buff[0 ] =RELATIVEJUMP_OPCODE ; *(s32 *)(&insn_buff[1 ]) = rel; -text_poke_bp (op->kp.addr, insn_buff,RELATIVEJUMP_SIZE , - op->optinsn.insn); +text_poke_bp (op->kp.addr, insn_buff,RELATIVEJUMP_SIZE ,NULL );list_del_init (&op->list); }
@@ -448,12 +447,18 @@ void arch_optimize_kprobes(struct list_hvoid arch_unoptimize_kprobe (struct optimized_kprobe *op) {u8 insn_buff[RELATIVEJUMP_SIZE ]; +u8 emulate_buff[RELATIVEJUMP_SIZE ];/* Set int3 to first byte for kprobes */ insn_buff[0 ] =BREAKPOINT_INSTRUCTION ;memcpy (insn_buff +1 , op->optinsn.copied_insn,RELATIVE_ADDR_SIZE ); + + emulate_buff[0 ] =RELATIVEJUMP_OPCODE ; + *(s32 *)(&emulate_buff[1 ]) = (s32)((long )op->optinsn.insn - + ((long )op->kp.addr +RELATIVEJUMP_SIZE )); +text_poke_bp (op->kp.addr, insn_buff,RELATIVEJUMP_SIZE , - op->optinsn.insn); + emulate_buff); }/*
Peter Zijlstra
maintainerPATCH 9/15 compiler.h: Make __ADDRESSABLE() symbol truly unique
In reply to Peter ZijlstraFrom: Josh Poimboeuf <[email protected]>
The __ADDRESSABLE() macro uses the __LINE__ macro to create a temporary symbol which has a unique name. However, if the macro is used multiple times from within another macro, the line number will always be the same, resulting in duplicate symbols.
Make the temporary symbols truly unique by using __UNIQUE_ID instead of __LINE__.
--- include/linux/compiler.h | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-)
include/linux/compiler.h +1 -1
@@ -294,7 +294,7 @@ unsigned long read_word_at_a_time(const*/ #define __ADDRESSABLE (sym) \ static void * __section(".discard.addressable") __used \ - __PASTE(__addressable_##sym, __LINE__) = (void *)&sym; + __UNIQUE_ID(__addressable_##sym) = (void *)&sym;/** * offset_to_ptr - convert a relative memory offset to an absolute pointer
Peter Zijlstra
maintainerPATCH 10/15 static_call: Add basic static call infrastructure
In reply to Peter ZijlstraFrom: Josh Poimboeuf <[email protected]>
Static calls are a replacement for global function pointers. They use code patching to allow direct calls to be used instead of indirect calls. They give the flexibility of function pointers, but with improved performance. This is especially important for cases where retpolines would otherwise be used, as retpolines can significantly impact performance.
The concept and code are an extension of previous work done by Ard Biesheuvel and Steven Rostedt:
https://lkml.kernel.org/r/[email protected]
https://lkml.kernel.org/r/[email protected]
There are two implementations, depending on arch support:
1) out-of-line: patched trampolines (CONFIG_HAVE_STATIC_CALL)
2) basic function pointers
For more details, see the comments in include/linux/static_call.h.
--- arch/Kconfig | 3 include/linux/static_call.h | 135 ++++++++++++++++++++++++++++++++++++++ include/linux/static_call_types.h | 13 +++ 3 files changed, 151 insertions(+) create mode 100644 include/linux/static_call.h create mode 100644 include/linux/static_call_types.h
arch/Kconfig +3 -0
@@ -927,6 +927,9 @@ config LOCK_EVENT_COUNTS the chance of application behavior change because of timing differences. The counts are reported via debugfs. +config HAVE_STATIC_CALL + bool + source "kernel/gcov/Kconfig" source "scripts/gcc-plugins/Kconfig"
include/linux/static_call.h new file +135 -0
135 lines not on this page · raw message
include/linux/static_call_types.h new file +13 -0
@@ -0,0 +1,13 @@ +/* SPDX-License-Identifier: GPL-2.0 */ +#ifndef _STATIC_CALL_TYPES_H +#define _STATIC_CALL_TYPES_H + +#include <linux/stringify.h> + +#define STATIC_CALL_TRAMP_PREFIX ____static_call_tramp_ +#define STATIC_CALL_TRAMP_PREFIX_STR __stringify(STATIC_CALL_TRAMP_PREFIX) + +#define STATIC_CALL_TRAMP (key) __PASTE(STATIC_CALL_TRAMP_PREFIX, key) +#define STATIC_CALL_TRAMP_STR (key) __stringify(STATIC_CALL_TRAMP(key)) + +#endif /* _STATIC_CALL_TYPES_H */
Re: [PATCH 10/15] static_call: Add basic static call infrastructure
Quoted 98 lines · Peter Zijlstra
From: Josh Poimboeuf <[email protected]>
Static calls are a replacement for global function pointers. They use code patching to allow direct calls to be used instead of indirect calls. They give the flexibility of function pointers, but with improved performance. This is especially important for cases where retpolines would otherwise be used, as retpolines can significantly impact performance.
The concept and code are an extension of previous work done by Ard Biesheuvel and Steven Rostedt:
https://nam04.safelinks.protection.outlook.com/?url=https%3A%2F%2Flkml.kernel.org%2Fr%2F20181005081333.15018-1-ard.biesheuvel%40linaro.org&data=02%7C01%7Cnamit%40vmware.com%7C3f2ebbeff15e444d2fa008d6e9b9023f%7Cb39138ca3cee4b4aa4d6cd83d9dd62f0%7C0%7C0%7C636953378182765229&sdata=WHceTWVt%2BNu1RFBv8jHp2Tw7VZuI5HxvHt%2FrWnjAmm4%3D&reserved=0
https://nam04.safelinks.protection.outlook.com/?url=https%3A%2F%2Flkml.kernel.org%2Fr%2F20181006015110.653946300%40goodmis.org&data=02%7C01%7Cnamit%40vmware.com%7C3f2ebbeff15e444d2fa008d6e9b9023f%7Cb39138ca3cee4b4aa4d6cd83d9dd62f0%7C0%7C0%7C636953378182765229&sdata=12JcrUsOh7%2FjRwEV9ANHw5SA2A6D4qNJ6z3h5aMHpnE%3D&reserved=0There are two implementations, depending on arch support:
1) out-of-line: patched trampolines (
CONFIG_HAVE_STATIC_CALL)
2) basic function pointersFor more details, see the comments in
include/linux/static_call.h.Cc Steven Rostedt <[email protected]>Cc Julia Cartwright <[email protected]>Cc Ingo Molnar <[email protected]>Cc Ard Biesheuvel <[email protected]>Cc Jason Baron <[email protected]>Cc Rasmus Villemoes <[email protected]>Cc Daniel Bristot de Oliveira <[email protected]>Cc Linus Torvalds <[email protected]>Cc Jiri Kosina <[email protected]>Cc Edward Cree <[email protected]>Cc Thomas Gleixner <[email protected]>Cc Masami Hiramatsu <[email protected]>Cc Borislav Petkov <[email protected]>Cc David Laight <[email protected]>Cc Jessica Yu <[email protected]>Cc Nadav Amit <[email protected]>Cc Andy Lutomirski <[email protected]>Cc "H. Peter Anvin" <[email protected]>Signed-off-by Josh Poimboeuf <[email protected]>Signed-off-by Peter Zijlstra (Intel) <[email protected]>---
arch/Kconfig | 3 include/linux/static_call.h | 135 ++++++++++++++++++++++++++++++++++++++ include/linux/static_call_types.h | 13 +++ 3 files changed, 151 insertions(+)create mode 100644
include/linux/static_call.hcreate mode 100644include/linux/static_call_types.harch/Kconfig +3
-0@@ -927,6 +927,9 @@ config LOCK_EVENT_COUNTS the chance of application behavior change because of timing differences. The counts are reported via debugfs. +config HAVE_STATIC_CALL + bool +source "
kernel/gcov/Kconfig"source "
scripts/gcc-plugins/Kconfig"include/linux/static_call.h new file +29
-0@@ -0,0 +1,135 @@ +/* SPDX-License-Identifier: GPL-2.0 */ +#ifndef _LINUX_STATIC_CALL_H +#define _LINUX_STATIC_CALL_H + +/* +* Static call support +* +* Static calls use code patching to hard-code function pointers into direct +* branch instructions. They give the flexibility of function pointers, but +* with improved performance. This is especially important for cases where +* retpolines would otherwise be used, as retpolines can significantly impact +* performance. +* +* +* API overview: +* +* DECLARE_STATIC_CALL(key, func); +* DEFINE_STATIC_CALL(key, func); +* static_call(key, args...); +* static_call_update(key, func); +* +* +* Usage example: +* +* # Start with the following functions (with identical prototypes): +* int func_a(int arg1, int arg2); +* int func_b(int arg1, int arg2); +* +* # Define a 'my_key' reference, associated with func_a() by default
include/linux/static_call.h line 39
Quoted 10 lines
include/linux/static_call.h new file +10
-0+* DEFINE_STATIC_CALL(my_key, func_a); +* +* # Call func_a() +* static_call(my_key, arg1, arg2); +* +* # Update 'my_key' to point to func_b() +* static_call_update(my_key, func_b); +* +* # Call func_b() +* static_call(my_key, arg1, arg2);
I think that this calling interface is not very intuitive. I understand that the macros/objtool cannot allow the calling interface to be completely transparent (as compiler plugin could). But, can the macros be used to paste the key with the “static_call”? I think that having something like:
static_call__func(arg1, arg2)
Is more readable than
static_call(func, arg1, arg2)
include/linux/static_call.h line 95
Quoted 7 lines
include/linux/static_call.h new file +7
-0+} + +#define static_call_update (key, func) \ +({ \ + BUILD_BUG_ON(!__same_type(func, STATIC_CALL_TRAMP(key))); \ + __static_call_update(&key, func); \ +})
Is this safe against concurrent module removal?
Peter Zijlstra
maintainerRe: [PATCH 10/15] static_call: Add basic static call infrastructure
Quoted 18 lines · Nadav Amit
Quoted 5 lines
+ * Usage example:
+ *+ * # Start with the following functions (with identical prototypes): + * int func_a(int arg1, int arg2); + * int func_b(int arg1, int arg2);Quoted 12 lines
+ *
+ * # Define a 'my_key' reference, associated withfunc_a()by default
+ *DEFINE_STATIC_CALL(my_key, func_a);
+ *
+ * # Callfunc_a()
+ *static_call(my_key, arg1, arg2);
+ *
+ * # Update 'my_key' to point tofunc_b()
+ *static_call_update(my_key, func_b);
+ *
+ * # Callfunc_b()
+ *static_call(my_key, arg1, arg2);
Quoted 1 line
I think that this calling interface is not very intuitive.
Yeah, it is somewhat unfortunate..
Quoted 10 lines
I understand that
the macros/objtool cannot allow the calling interface to be completely transparent (as compiler plugin could). But, can the macros be used to paste the key with the “static_call”? I think that having something like:
static_call__func(arg1, arg2)Is more readable than
static_call(func, arg1, arg2)
Doesn't really make it much better for me; I think I'd prefer to switch to the GCC plugin scheme over this. ISTR there being some propotypes there, but I couldn't quickly locate them.
Quoted 9 lines
include/linux/static_call.h line 95
Quoted 7 lines
include/linux/static_call.h new file +7
-0+} + +#define static_call_update (key, func) \ +({ \ + BUILD_BUG_ON(!__same_type(func, STATIC_CALL_TRAMP(key))); \ + __static_call_update(&key, func); \ +})Is this safe against concurrent module removal?
It is for CONFIG_MODULE=n :-)
Re: [PATCH 10/15] static_call: Add basic static call infrastructure
Quoted 20 lines · Peter Zijlstra
Quoted 18 lines · Nadav Amit
Quoted 17 lines
+ * Usage example:
+ *+ * # Start with the following functions (with identical prototypes): + * int func_a(int arg1, int arg2); + * int func_b(int arg1, int arg2);+ *
+ * # Define a 'my_key' reference, associated withfunc_a()by default
+ *DEFINE_STATIC_CALL(my_key, func_a);
+ *
+ * # Callfunc_a()
+ *static_call(my_key, arg1, arg2);
+ *
+ * # Update 'my_key' to point tofunc_b()
+ *static_call_update(my_key, func_b);
+ *
+ * # Callfunc_b()
+ *static_call(my_key, arg1, arg2);
Quoted 4 lines
Quoted 1 line
I think that this calling interface is not very intuitive.
Yeah, it is somewhat unfortunate..
Another thing I brought up at the time is that it would be useful to have the ability to 'reset' a static call to its default target. E.g., for crypto modules that implement an accelerated version of a library interface, removing the module should revert those call sites back to the original target, without putting a disproportionate burden on the module itself to implement the logic to support this.
Quoted 7 lines
Quoted 7 lines
I understand that
the macros/objtool cannot allow the calling interface to be completely transparent (as compiler plugin could). But, can the macros be used to paste the key with the “static_call”? I think that having something like:
static_call__func(arg1, arg2)
Quoted 8 lines
Quoted 3 lines
Is more readable than
static_call(func, arg1, arg2)Doesn't really make it much better for me; I think I'd prefer to switch to the GCC plugin scheme over this. ISTR there being some propotypes there, but I couldn't quickly locate them.
I implemented the GCC plugin here
https://git.kernel.org/pub/scm/linux/kernel/git/ardb/linux.git/log/?h=static-calls
but IIRC, all it does is annotate call sites exactly how objtool does it.
Quoted 11 lines
Quoted 9 lines
include/linux/static_call.h line 95
Quoted 7 lines
include/linux/static_call.h new file +7
-0+} + +#define static_call_update (key, func) \ +({ \ + BUILD_BUG_ON(!__same_type(func, STATIC_CALL_TRAMP(key))); \ + __static_call_update(&key, func); \ +})Is this safe against concurrent module removal?
It is for
CONFIG_MODULE=n:-)
Re: [PATCH 10/15] static_call: Add basic static call infrastructure
Quoted 30 lines · Ard Biesheuvel
Quoted 22 lines · Peter Zijlstra
Quoted 18 lines · Nadav Amit
Quoted 17 lines
+ * Usage example:
+ *+ * # Start with the following functions (with identical prototypes): + * int func_a(int arg1, int arg2); + * int func_b(int arg1, int arg2);+ *
+ * # Define a 'my_key' reference, associated withfunc_a()by default
+ *DEFINE_STATIC_CALL(my_key, func_a);
+ *
+ * # Callfunc_a()
+ *static_call(my_key, arg1, arg2);
+ *
+ * # Update 'my_key' to point tofunc_b()
+ *static_call_update(my_key, func_b);
+ *
+ * # Callfunc_b()
+ *static_call(my_key, arg1, arg2);Quoted 4 lines
Quoted 1 line
I think that this calling interface is not very intuitive.
Yeah, it is somewhat unfortunate..
Quoted 6 lines
Another thing I brought up at the time is that it would be useful to have the ability to 'reset' a static call to its default target. E.g., for crypto modules that implement an accelerated version of a library interface, removing the module should revert those call sites back to the original target, without putting a disproportionate burden on the module itself to implement the logic to support this.
I was thinking this could be a layer on top. We could have a way to register a static call with the module core so that, when a GPL module with an appropriate symbol is loaded, the static call gets replaced.
KVM could use this too. Or we could just require KVM to be built in some day.
Re: [PATCH 10/15] static_call: Add basic static call infrastructure
In reply to Ard BiesheuvelQuoted 49 lines · Ard Biesheuvel
Quoted 19 lines · Peter Zijlstra
Quoted 18 lines · Nadav Amit
Quoted 17 lines
+ * Usage example:
+ *+ * # Start with the following functions (with identical prototypes): + * int func_a(int arg1, int arg2); + * int func_b(int arg1, int arg2);+ *
+ * # Define a 'my_key' reference, associated withfunc_a()by default
+ *DEFINE_STATIC_CALL(my_key, func_a);
+ *
+ * # Callfunc_a()
+ *static_call(my_key, arg1, arg2);
+ *
+ * # Update 'my_key' to point tofunc_b()
+ *static_call_update(my_key, func_b);
+ *
+ * # Callfunc_b()
+ *static_call(my_key, arg1, arg2);Quoted 3 lines
Quoted 1 line
I think that this calling interface is not very intuitive.
Yeah, it is somewhat unfortunate..
Another thing I brought up at the time is that it would be useful to have the ability to 'reset' a static call to its default target. E.g., for crypto modules that implement an accelerated version of a library interface, removing the module should revert those call sites back to the original target, without putting a disproportionate burden on the module itself to implement the logic to support this.
Quoted 7 lines
Quoted 7 lines
I understand that
the macros/objtool cannot allow the calling interface to be completely transparent (as compiler plugin could). But, can the macros be used to paste the key with the “static_call”? I think that having something like:
static_call__func(arg1, arg2)Quoted 7 lines
Quoted 3 lines
Is more readable than
static_call(func, arg1, arg2)Doesn't really make it much better for me; I think I'd prefer to switch to the GCC plugin scheme over this. ISTR there being some propotypes there, but I couldn't quickly locate them.
Quoted 5 lines
I implemented the GCC plugin here
but IIRC, all it does is annotate call sites exactly how objtool does it.
I did not see your version before I made mine for a similar (but slightly different) purpose:
https://lore.kernel.org/lkml/[email protected]/
My version, I think, is more generic (I don’t think yours consider calls that have a return value). Anyhow, I am sure you know more about GCC plugins than I do.
I do have a version that can take annotations to say which call should be static and to get the symbol it uses.
I also think that this implementation would disallow keys that reside within structs. This would mean that paravirt, for instance, would need to go through many changes to use this infrastructure.
Peter Zijlstra
maintainerRe: [PATCH 10/15] static_call: Add basic static call infrastructure
In reply to Peter ZijlstraQuoted 30 lines · Peter Zijlstra
Quoted 18 lines · Nadav Amit
Quoted 5 lines
+ * Usage example:
+ *+ * # Start with the following functions (with identical prototypes): + * int func_a(int arg1, int arg2); + * int func_b(int arg1, int arg2);Quoted 12 lines
+ *
+ * # Define a 'my_key' reference, associated withfunc_a()by default
+ *DEFINE_STATIC_CALL(my_key, func_a);
+ *
+ * # Callfunc_a()
+ *static_call(my_key, arg1, arg2);
+ *
+ * # Update 'my_key' to point tofunc_b()
+ *static_call_update(my_key, func_b);
+ *
+ * # Callfunc_b()
+ *static_call(my_key, arg1, arg2);Quoted 1 line
I think that this calling interface is not very intuitive.
Yeah, it is somewhat unfortunate..
Quoted 7 lines
I understand that
the macros/objtool cannot allow the calling interface to be completely transparent (as compiler plugin could). But, can the macros be used to paste the key with the “static_call”? I think that having something like:
static_call__func(arg1, arg2)
Quoted 7 lines
Quoted 3 lines
Is more readable than
static_call(func, arg1, arg2)Doesn't really make it much better for me; I think I'd prefer to switch to the GCC plugin scheme over this. ISTR there being some propotypes there, but I couldn't quickly locate them.
How about something like:
static_call(key)(arg1, arg2);
which is very close to the regular indirect call syntax. Furthermore, how about we put the trampolines in .static_call.text instead of relying on prefixes?
Also, I think I can shrink static_call_key by half:
- we can do away with static_call_key::tramp; there are only two usage sites:
o __static_call_update, the static_call() macro can provide the address of STATIC_CALL_TRAMP(key) directly
o static_call_add_module(), has two cases:
* the trampoline is from outside the module; in this case it will already have been updated by a previous call to __static_call_update.
* the trampoline is from inside the module; in this case it will have the default value and it doesn't need an update.
so in no case does static_call_add_module() need to modify a trampoline.
- we can change static_call_key::site_mods into a single next pointer, just like jump_label's static_key.
But so far all the schemes I've come up with require 'key' to be a name, it cannot be an actual 'struct static_call_key *' value. And therefore usage from within structures isn't allowed.
Re: [PATCH 10/15] static_call: Add basic static call infrastructure
Quoted 39 lines · Peter Zijlstra
Quoted 30 lines · Peter Zijlstra
Quoted 18 lines · Nadav Amit
Quoted 5 lines
+ * Usage example:
+ *+ * # Start with the following functions (with identical prototypes): + * int func_a(int arg1, int arg2); + * int func_b(int arg1, int arg2);Quoted 12 lines
+ *
+ * # Define a 'my_key' reference, associated withfunc_a()by default
+ *DEFINE_STATIC_CALL(my_key, func_a);
+ *
+ * # Callfunc_a()
+ *static_call(my_key, arg1, arg2);
+ *
+ * # Update 'my_key' to point tofunc_b()
+ *static_call_update(my_key, func_b);
+ *
+ * # Callfunc_b()
+ *static_call(my_key, arg1, arg2);Quoted 1 line
I think that this calling interface is not very intuitive.
Yeah, it is somewhat unfortunate..
Quoted 7 lines
I understand that
the macros/objtool cannot allow the calling interface to be completely transparent (as compiler plugin could). But, can the macros be used to paste the key with the “static_call”? I think that having something like:
static_call__func(arg1, arg2)Quoted 7 lines
Quoted 3 lines
Is more readable than
static_call(func, arg1, arg2)Doesn't really make it much better for me; I think I'd prefer to switch to the GCC plugin scheme over this. ISTR there being some propotypes there, but I couldn't quickly locate them.
Quoted 5 lines
How about something like:
static_call(key)(arg1, arg2);which is very close to the regular indirect call syntax.
Looks ok to me.
Quoted 2 lines
Furthermore, how about we put the trampolines in .static_call.text instead of relying on prefixes?
Yeah, that would probably be better.
Quoted 17 lines
Also, I think I can shrink
static_call_keyby half:- we can do away with
static_call_key::tramp; there are only two usage sites:o
__static_call_update, thestatic_call()macro can provide the address ofSTATIC_CALL_TRAMP(key)directlyo
static_call_add_module(), has two cases:* the trampoline is from outside the module; in this case it will already have been updated by a previous call to
__static_call_update.
* the trampoline is from inside the module; in this case it will have the default value and it doesn't need an update.
Quoted 2 lines
so in no case does
static_call_add_module()need to modify a trampoline.
Sounds plausible.
Quoted 2 lines
- we can change
static_call_key::site_modsinto a single next pointer, just like jump_label'sstatic_key.
Yep.
Quoted 3 lines
But so far all the schemes I've come up with require 'key' to be a name, it cannot be an actual 'struct
static_call_key*' value. And therefore usage from within structures isn't allowed.
Is that something we need? At least we were able to work around this limitation with tracepoints' usage of static calls. But I could see how it could be useful.
One way to solve that would be a completely different implementation:
have a global trampoline which detects the call site of the caller, associates it with the given key, schedules some work to patch the call site later, and then jumps to key->func. So the first call would trigger the patching.
Then we might not even need objtool :-) But it might be tricky to pull off.
Signature · 1 line
Josh
Peter Zijlstra
maintainerPATCH 11/15 static_call: Add inline static call infrastructure
In reply to Peter ZijlstraFrom: Josh Poimboeuf <[email protected]>
Add infrastructure for an arch-specific CONFIG_HAVE_STATIC_CALL_INLINE option, which is a faster version of CONFIG_HAVE_STATIC_CALL. At runtime, the static call sites are patched directly, rather than using the out-of-line trampolines.
Compared to out-of-line static calls, the performance benefits are more modest, but still measurable. Steven Rostedt did some tracepoint measurements:
https://lkml.kernel.org/r/[email protected]
This code is heavily inspired by the jump label code (aka "static jumps"), as some of the concepts are very similar.
For more details, see the comments in include/linux/static_call.h.
--- arch/Kconfig | 4 include/asm-generic/vmlinux.lds.h | 7 include/linux/module.h | 10 + include/linux/static_call.h | 63 +++++++ include/linux/static_call_types.h | 9 + kernel/Makefile | 1 kernel/module.c | 5 kernel/static_call.c | 316 ++++++++++++++++++++++++++++++++++++++ 8 files changed, 414 insertions(+), 1 deletion(-) create mode 100644 kernel/static_call.c
arch/Kconfig +4 -0
@@ -930,6 +930,10 @@ config LOCK_EVENT_COUNTS config HAVE_STATIC_CALL bool +config HAVE_STATIC_CALL_INLINE + bool + depends on HAVE_STATIC_CALL + source "kernel/gcov/Kconfig" source "scripts/gcc-plugins/Kconfig"
include/asm-generic/vmlinux.lds.h +7 -0
@@ -311,6 +311,12 @@ KEEP(*(__jump_table)) \ __stop___jump_table = .; +#define STATIC_CALL_DATA \ + . = ALIGN(8); \ + __start_static_call_sites = .; \ + KEEP(*(.static_call_sites)) \ + __stop_static_call_sites = .; +/* * Allow architectures to handle ro_after_init data on their * own by defining an empty RO_AFTER_INIT_DATA.
@@ -320,6 +326,7 @@ __start_ro_after_init = .; \ *(.data..ro_after_init) \ JUMP_TABLE_DATA \ + STATIC_CALL_DATA \ __end_ro_after_init = .;#endif
include/linux/module.h +10 -0
@@ -21,6 +21,7 @@#include <linux/rbtree_latch.h> #include <linux/error-injection.h> #include <linux/tracepoint-defs.h> +#include <linux/static_call_types.h> #include <linux/percpu.h> #include <asm/module.h>
@@ -472,6 +473,10 @@ struct module {unsigned int num_ftrace_callsites;unsigned long *ftrace_callsites;#endif +#ifdef CONFIG_HAVE_STATIC_CALL_INLINE +int num_static_call_sites; +struct static_call_site *static_call_sites; +#endif #ifdef CONFIG_LIVEPATCH bool klp;/* Is this a livepatch module? */
@@ -728,6 +733,11 @@ static inline bool within_module(unsigne {return false; } + +static inline bool within_module_init (unsigned long addr,const struct module *mod) +{ +return false; +}/* Get/put a kernel symbol (calls should be symmetric) */ #define symbol_get (x) ({ extern typeof(x) x __attribute__((weak)); &(x); })
include/linux/static_call.h +62 -1
75 lines not on this page · raw message
include/linux/static_call_types.h +9 -0
@@ -10,4 +10,13 @@#define STATIC_CALL_TRAMP (key) __PASTE(STATIC_CALL_TRAMP_PREFIX, key)#define STATIC_CALL_TRAMP_STR (key) __stringify(STATIC_CALL_TRAMP(key)) +/* +* The static call site table needs to be created by external tooling (objtool +* or a compiler plugin). +*/ +struct static_call_site { +s32 addr; +s32 key; +}; +#endif /* _STATIC_CALL_TYPES_H */
kernel/Makefile +1 -0
@@ -100,6 +100,7 @@ obj-$(CONFIG_TRACEPOINTS) += trace/ obj-$(CONFIG_IRQ_WORK) += irq_work.o obj-$(CONFIG_CPU_PM) += cpu_pm.o obj-$(CONFIG_BPF) += bpf/ +obj-$(CONFIG_HAVE_STATIC_CALL_INLINE) += static_call.o obj-$(CONFIG_PERF_EVENTS) += events/
kernel/module.c +5 -0
@@ -3117,6 +3117,11 @@ static int find_module_sections(struct msizeof (*mod->ei_funcs), &mod->num_ei_funcs);#endif +#ifdef CONFIG_HAVE_STATIC_CALL_INLINE + mod->static_call_sites =section_objs (info,".static_call_sites" , +sizeof (*mod->static_call_sites), + &mod->num_static_call_sites); +#endif mod->extable =section_objs (info,"__ex_table" ,sizeof (*mod->extable), &mod->num_exentries);
kernel/static_call.c new file +316 -0
316 lines not on this page · raw message
Re: [PATCH 11/15] static_call: Add inline static call infrastructure
Quoted 12 lines · Peter Zijlstra
From: Josh Poimboeuf <[email protected]>
Add infrastructure for an arch-specific
CONFIG_HAVE_STATIC_CALL_INLINEoption, which is a faster version ofCONFIG_HAVE_STATIC_CALL. At runtime, the static call sites are patched directly, rather than using the out-of-line trampolines.Compared to out-of-line static calls, the performance benefits are more modest, but still measurable. Steven Rostedt did some tracepoint measurements:
[ snip ]
Quoted 13 lines
kernel/static_call.c new file +13
-0+static void static_call_del_module (struct module *mod) +{ +struct static_call_site *start = mod->static_call_sites; +struct static_call_site *stop = mod->static_call_sites + + mod->num_static_call_sites; +struct static_call_site *site; +struct static_call_key *key, *prev_key =NULL ; +struct static_call_mod *site_mod; + +for (site = start; site < stop; site++) { + key =static_call_key (site); +if (key == prev_key) +continue ;
kernel/static_call.c line 227
Quoted 10 lines
kernel/static_call.c new file +10
-0+ prev_key = key; + +list_for_each_entry (site_mod, &key->site_mods, list) { +if (site_mod->mod == mod) { +list_del (&site_mod->list); +kfree (site_mod); +break ; + } + } + }
I think that for safety, when a module is removed, all the static-calls should be traversed to check that none of them calls any function in the removed module. If that happens, perhaps it should be poisoned.
Quoted 6 lines
kernel/static_call.c new file +6
-0+} + +static int static_call_module_notify (struct notifier_block *nb, +unsigned long val,void *data) +{ +struct module *mod = data;
kernel/static_call.c line 243
Quoted 10 lines
kernel/static_call.c new file +10
-0+int ret =0 ; + +cpus_read_lock (); +static_call_lock (); + +switch (val) { +case MODULE_STATE_COMING : +module_disable_ro (mod); + ret =static_call_add_module (mod); +module_enable_ro (mod, false);
Doesn’t it cause some pages to be W+X ? Can it be avoided?
kernel/static_call.c line 246
Quoted 3 lines
kernel/static_call.c new file +3
-0+if (ret) { +WARN (1 ,"Failed to allocate memory for static calls" ); +static_call_del_module (mod);
If static_call_add_module() succeeded in changing some of the calls, but not all, I don’t think that static_call_del_module() will correctly undo static_call_add_module(). The code transformations, I think, will remain.
Peter Zijlstra
maintainerRe: [PATCH 11/15] static_call: Add inline static call infrastructure
Quoted 19 lines · Nadav Amit
Quoted 19 lines
kernel/static_call.c new file +19
-0+static void static_call_del_module (struct module *mod) +{ +struct static_call_site *start = mod->static_call_sites; +struct static_call_site *stop = mod->static_call_sites + + mod->num_static_call_sites; +struct static_call_site *site; +struct static_call_key *key, *prev_key =NULL ; +struct static_call_mod *site_mod; + +for (site = start; site < stop; site++) { + key =static_call_key (site); +if (key == prev_key) +continue ; + prev_key = key; + +list_for_each_entry (site_mod, &key->site_mods, list) { +if (site_mod->mod == mod) { +list_del (&site_mod->list); +kfree (site_mod);
Quoted 8 lines
kernel/static_call.c line 227
Quoted 4 lines
kernel/static_call.c new file +4
-0+break ; + } + } + }I think that for safety, when a module is removed, all the static-calls should be traversed to check that none of them calls any function in the removed module. If that happens, perhaps it should be poisoned.
We don't do that for normal indirect calls either.. I suppose we could here, but meh.
Quoted 10 lines
Quoted 10 lines
kernel/static_call.c new file +10
-0+} + +static int static_call_module_notify (struct notifier_block *nb, +unsigned long val,void *data) +{ +struct module *mod = data; +int ret =0 ; + +cpus_read_lock (); +static_call_lock ();
Quoted 8 lines
kernel/static_call.c line 243
Quoted 6 lines
kernel/static_call.c new file +6
-0+ +switch (val) { +case MODULE_STATE_COMING : +module_disable_ro (mod); + ret =static_call_add_module (mod); +module_enable_ro (mod, false);Doesn’t it cause some pages to be W+X ? Can it be avoided?
I don't know why it does this, jump_labels doesn't seem to need this, and I'm not seeing what static_call needs differently.
Quoted 7 lines
kernel/static_call.c line 246
Quoted 3 lines
kernel/static_call.c new file +3
-0+if (ret) { +WARN (1 ,"Failed to allocate memory for static calls" ); +static_call_del_module (mod);If
static_call_add_module()succeeded in changing some of the calls, but not all, I don’t think thatstatic_call_del_module()will correctly undostatic_call_add_module(). The code transformations, I think, will remain.
Hurm, jump_labels has the same problem.
I wonder why kernel/module.c:prepare_coming_module() doesn't propagate the error from the notifier call. If it were to do that, I think we'll abort the module load and any modifications get lost anyway.
Re: [PATCH 11/15] static_call: Add inline static call infrastructure
Quoted 65 lines · Peter Zijlstra
Quoted 19 lines · Nadav Amit
Quoted 19 lines
kernel/static_call.c new file +19
-0+static void static_call_del_module (struct module *mod) +{ +struct static_call_site *start = mod->static_call_sites; +struct static_call_site *stop = mod->static_call_sites + + mod->num_static_call_sites; +struct static_call_site *site; +struct static_call_key *key, *prev_key =NULL ; +struct static_call_mod *site_mod; + +for (site = start; site < stop; site++) { + key =static_call_key (site); +if (key == prev_key) +continue ; + prev_key = key; + +list_for_each_entry (site_mod, &key->site_mods, list) { +if (site_mod->mod == mod) { +list_del (&site_mod->list); +kfree (site_mod);Quoted 8 lines
kernel/static_call.c line 227
Quoted 4 lines
kernel/static_call.c new file +4
-0+break ; + } + } + }I think that for safety, when a module is removed, all the static-calls should be traversed to check that none of them calls any function in the removed module. If that happens, perhaps it should be poisoned.
We don't do that for normal indirect calls either.. I suppose we could here, but meh.
Quoted 10 lines
Quoted 10 lines
kernel/static_call.c new file +10
-0+} + +static int static_call_module_notify (struct notifier_block *nb, +unsigned long val,void *data) +{ +struct module *mod = data; +int ret =0 ; + +cpus_read_lock (); +static_call_lock ();Quoted 8 lines
kernel/static_call.c line 243
Quoted 6 lines
kernel/static_call.c new file +6
-0+ +switch (val) { +case MODULE_STATE_COMING : +module_disable_ro (mod); + ret =static_call_add_module (mod); +module_enable_ro (mod, false);Doesn’t it cause some pages to be W+X ? Can it be avoided?
I don't know why it does this,
jump_labelsdoesn't seem to need this, and I'm not seeing whatstatic_callneeds differently.Quoted 7 lines
kernel/static_call.c line 246
Quoted 3 lines
kernel/static_call.c new file +3
-0+if (ret) { +WARN (1 ,"Failed to allocate memory for static calls" ); +static_call_del_module (mod);If
static_call_add_module()succeeded in changing some of the calls, but not all, I don’t think thatstatic_call_del_module()will correctly undostatic_call_add_module(). The code transformations, I think, will remain.
Quoted 5 lines
Hurm,
jump_labelshas the same problem.I wonder why kernel/module.c:prepare_coming_module() doesn't propagate the error from the notifier call. If it were to do that, I think we'll abort the module load and any modifications get lost anyway.
This might be a security problem, since it can leave indirect branches, which are susceptible to Spectre v2, in the code.
Peter Zijlstra
maintainerRe: [PATCH 11/15] static_call: Add inline static call infrastructure
Quoted 11 lines · Nadav Amit
Quoted 11 lines · Peter Zijlstra
Quoted 7 lines · Nadav Amit
kernel/static_call.c line 246
Quoted 3 lines
kernel/static_call.c new file +3
-0+if (ret) { +WARN (1 ,"Failed to allocate memory for static calls" ); +static_call_del_module (mod);If
static_call_add_module()succeeded in changing some of the calls, but not all, I don’t think thatstatic_call_del_module()will correctly undostatic_call_add_module(). The code transformations, I think, will remain.
Quoted 8 lines
Quoted 5 lines
Hurm,
jump_labelshas the same problem.I wonder why kernel/module.c:prepare_coming_module() doesn't propagate the error from the notifier call. If it were to do that, I think we'll abort the module load and any modifications get lost anyway.
This might be a security problem, since it can leave indirect branches, which are susceptible to Spectre v2, in the code.
It's a correctness problem too; for both jump_label and static_call, since if we don't patch the call site, we also don't patch the trampoline and who knows what random code it ends up running.
I'll go stare at the module code once my migrane goes again :/
Re: [PATCH 11/15] static_call: Add inline static call infrastructure
In reply to Peter ZijlstraQuoted 10 lines · Peter Zijlstra
Quoted 10 lines
Quoted 10 lines
kernel/static_call.c new file +10
-0+} + +static int static_call_module_notify (struct notifier_block *nb, +unsigned long val,void *data) +{ +struct module *mod = data; +int ret =0 ; + +cpus_read_lock (); +static_call_lock ();
Quoted 8 lines
Quoted 8 lines
kernel/static_call.c line 243
Quoted 6 lines
kernel/static_call.c new file +6
-0+ +switch (val) { +case MODULE_STATE_COMING : +module_disable_ro (mod); + ret =static_call_add_module (mod); +module_enable_ro (mod, false);Doesn’t it cause some pages to be W+X ?
How so?
Quoted 4 lines
Quoted 1 line
Can it be avoided?
I don't know why it does this,
jump_labelsdoesn't seem to need this, and I'm not seeing whatstatic_callneeds differently.
I forgot why I did this, but it's probably for the case where there's a static call site in module init code. It deserves a comment.
Theoretically, jump labels need this to.
BTW, there's a change coming that will require the text_mutex before calling module_{disable,enable}_ro().
Signature · 1 line
Josh
Re: [PATCH 11/15] static_call: Add inline static call infrastructure
Quoted 29 lines · Josh Poimboeuf
Quoted 10 lines · Peter Zijlstra
Quoted 10 lines
Quoted 10 lines
kernel/static_call.c new file +10
-0+} + +static int static_call_module_notify (struct notifier_block *nb, +unsigned long val,void *data) +{ +struct module *mod = data; +int ret =0 ; + +cpus_read_lock (); +static_call_lock ();Quoted 8 lines
Quoted 8 lines
kernel/static_call.c line 243
Quoted 6 lines
kernel/static_call.c new file +6
-0+ +switch (val) { +case MODULE_STATE_COMING : +module_disable_ro (mod); + ret =static_call_add_module (mod); +module_enable_ro (mod, false);Doesn’t it cause some pages to be W+X ?
How so?
Quoted 4 lines
Quoted 1 line
Can it be avoided?
I don't know why it does this,
jump_labelsdoesn't seem to need this, and I'm not seeing whatstatic_callneeds differently.
Quoted 7 lines
I forgot why I did this, but it's probably for the case where there's a static call site in module init code. It deserves a comment.
Theoretically, jump labels need this to.
BTW, there's a change coming that will require the
text_mutexbefore calling module_{disable,enable}_ro().
I think that eventually, the most secure flow is for the module executable to be write-protected immediately after the module signature is checked and then use text_poke() to change the code without removing the write-protection in such manner.
Ideally, these pieces of code (module signature check and static-key/call mechanisms) would somehow be isolated.
I wonder whether static-calls in init-code cannot just be avoided. They would most likely introduce more overhead in patching than they would save in execution time.
Re: [PATCH 11/15] static_call: Add inline static call infrastructure
Quoted 42 lines · Nadav Amit
Quoted 29 lines · Josh Poimboeuf
Quoted 10 lines · Peter Zijlstra
Quoted 10 lines
Quoted 10 lines
kernel/static_call.c new file +10
-0+} + +static int static_call_module_notify (struct notifier_block *nb, +unsigned long val,void *data) +{ +struct module *mod = data; +int ret =0 ; + +cpus_read_lock (); +static_call_lock ();Quoted 8 lines
Quoted 8 lines
kernel/static_call.c line 243
Quoted 6 lines
kernel/static_call.c new file +6
-0+ +switch (val) { +case MODULE_STATE_COMING : +module_disable_ro (mod); + ret =static_call_add_module (mod); +module_enable_ro (mod, false);Doesn’t it cause some pages to be W+X ?
How so?
Quoted 4 lines
Quoted 1 line
Can it be avoided?
I don't know why it does this,
jump_labelsdoesn't seem to need this, and I'm not seeing whatstatic_callneeds differently.Quoted 7 lines
I forgot why I did this, but it's probably for the case where there's a static call site in module init code. It deserves a comment.
Theoretically, jump labels need this to.
BTW, there's a change coming that will require the
text_mutexbefore calling module_{disable,enable}_ro().I think that eventually, the most secure flow is for the module executable to be write-protected immediately after the module signature is checked and then use
text_poke()to change the code without removing the write-protection in such manner.
Quoted 6 lines
Ideally, these pieces of code (module signature check and static-key/call mechanisms) would somehow be isolated.
I wonder whether static-calls in init-code cannot just be avoided. They would most likely introduce more overhead in patching than they would save in execution time.
It's a valid question. Are any tracepoints called from module init? Or -- thinking ahead -- are there any pv calls from module init? That might be plausible.
Signature · 1 line
Josh
Peter Zijlstra
maintainerRe: [PATCH 11/15] static_call: Add inline static call infrastructure
In reply to Josh PoimboeufQuoted 20 lines · Josh Poimboeuf
Quoted 10 lines · Peter Zijlstra
Quoted 10 lines
Quoted 10 lines
kernel/static_call.c new file +10
-0+} + +static int static_call_module_notify (struct notifier_block *nb, +unsigned long val,void *data) +{ +struct module *mod = data; +int ret =0 ; + +cpus_read_lock (); +static_call_lock ();Quoted 8 lines
Quoted 8 lines
kernel/static_call.c line 243
Quoted 6 lines
kernel/static_call.c new file +6
-0+ +switch (val) { +case MODULE_STATE_COMING : +module_disable_ro (mod); + ret =static_call_add_module (mod); +module_enable_ro (mod, false);Doesn’t it cause some pages to be W+X ?
Quoted 1 line
How so?
This is after complete_formation() which does RO,X. If we then disable RO we end up with W+X pages, which is bad.
That said, alternatives, ftrace, dynamic_debug all run before complete_formation() specifically such that they can directly poke text.
Possibly we should add a notifier callback for MODULE_STATE_UNFORMED, but that is for another day.
Quoted 5 lines
Quoted 4 lines
Quoted 1 line
Can it be avoided?
I don't know why it does this,
jump_labelsdoesn't seem to need this, and I'm not seeing whatstatic_callneeds differently.
Quoted 7 lines
I forgot why I did this, but it's probably for the case where there's a static call site in module init code. It deserves a comment.
Theoretically, jump labels need this to.
BTW, there's a change coming that will require the
text_mutexbefore calling module_{disable,enable}_ro().
I can't find why it would need this (and I'm going to remove it). Specifically complete_formation() does enable_ro(.after_init=false), which leaves .ro_after_init writable so {jump_label,static_call}_sort_entries() will work.
But both jump_label and static_call then use the full text_poke(), not text_poke_early(), for modules.
Peter Zijlstra
maintainerPATCH 12/15 x86/static_call: Add out-of-line static call implementation
In reply to Peter ZijlstraFrom: Josh Poimboeuf <[email protected]>
Add the x86 out-of-line static call implementation. For each key, a permanent trampoline is created which is the destination for all static calls for the given key. The trampoline has a direct jump which gets patched by static_call_update() when the destination function changes.
--- arch/x86/Kconfig | 1 arch/x86/include/asm/static_call.h | 28 +++++++++++++++++++++++++++ arch/x86/kernel/Makefile | 1 arch/x86/kernel/static_call.c | 38 +++++++++++++++++++++++++++++++++++++ 4 files changed, 68 insertions(+) create mode 100644 arch/x86/include/asm/static_call.h create mode 100644 arch/x86/kernel/static_call.c
arch/x86/Kconfig +1 -0
@@ -198,6 +198,7 @@ config X86 select HAVE_FUNCTION_ARG_ACCESS_API select HAVE_STACKPROTECTOR if CC_HAS_SANE_STACKPROTECTOR select HAVE_STACK_VALIDATION if X86_64 + select HAVE_STATIC_CALL select HAVE_RSEQ select HAVE_SYSCALL_TRACEPOINTS select HAVE_UNSTABLE_SCHED_CLOCK
arch/x86/include/asm/static_call.h new file +28 -0
@@ -0,0 +1,28 @@ +/* SPDX-License-Identifier: GPL-2.0 */ +#ifndef _ASM_STATIC_CALL_H +#define _ASM_STATIC_CALL_H + +/* +* Manually construct a 5-byte direct JMP to prevent the assembler from +* optimizing it into a 2-byte JMP. +*/ +#define __ARCH_STATIC_CALL_JMP_LABEL (key) ".L" __stringify(key ## _after_jmp) +#define __ARCH_STATIC_CALL_TRAMP_JMP (key, func) \ + ".byte 0xe9 \n" \ + ".long " #func " - " __ARCH_STATIC_CALL_JMP_LABEL(key) "\n" \ + __ARCH_STATIC_CALL_JMP_LABEL(key) ":" + +/* +* This is a permanent trampoline which does a direct jump to the function. +* The direct jump get patched by static_call_update(). +*/ +#define ARCH_DEFINE_STATIC_CALL_TRAMP (key, func) \ + asm(".pushsection .text, \"ax\" \n" \ + ".align 4 \n" \ + ".globl " STATIC_CALL_TRAMP_STR(key) " \n" \ + ".type " STATIC_CALL_TRAMP_STR(key) ", @function \n" \ + STATIC_CALL_TRAMP_STR(key) ": \n" \ + __ARCH_STATIC_CALL_TRAMP_JMP(key, func) " \n" \ + ".popsection \n") + +#endif /* _ASM_STATIC_CALL_H */
arch/x86/kernel/Makefile +1 -0
@@ -63,6 +63,7 @@ obj-y += tsc.o tsc_msr.o io_delay.o rt obj-y += pci-iommu_table.o obj-y += resource.o obj-y += irqflags.o +obj-y += static_call.o obj-y += process.o obj-y += fpu/
arch/x86/kernel/static_call.c new file +38 -0
@@ -0,0 +1,38 @@ +// SPDX-License-Identifier: GPL-2.0 +#include <linux/static_call.h> +#include <linux/memory.h> +#include <linux/bug.h> +#include <asm/text-patching.h> +#include <asm/nospec-branch.h> + +#define CALL_INSN_SIZE 5 + +void arch_static_call_transform (void *site,void *tramp,void *func) +{ +unsigned char opcodes[CALL_INSN_SIZE ]; +unsigned char insn_opcode; +unsigned long insn; +s32 dest_relative; + +mutex_lock (&text_mutex); + + insn = (unsigned long )tramp; + + insn_opcode = *(unsigned char *)insn; +if (insn_opcode !=0xE9 ) { +WARN_ONCE (1 ,"unexpected static call insn opcode 0x%x at %pS" , + insn_opcode, (void *)insn); + goto unlock; + } + + dest_relative = (long )(func) - (long )(insn +CALL_INSN_SIZE ); + + opcodes[0 ] = insn_opcode; +memcpy (&opcodes[1 ], &dest_relative,CALL_INSN_SIZE -1 ); + +text_poke_bp ((void *)insn, opcodes,CALL_INSN_SIZE ,NULL ); + +unlock: +mutex_unlock (&text_mutex); +} +EXPORT_SYMBOL_GPL (arch_static_call_transform);
Re: [PATCH 12/15] x86/static_call: Add out-of-line static call implementation
Quoted 105 lines · Peter Zijlstra
From: Josh Poimboeuf <[email protected]>
Add the x86 out-of-line static call implementation. For each key, a permanent trampoline is created which is the destination for all static calls for the given key. The trampoline has a direct jump which gets patched by
static_call_update()when the destination function changes.Cc Steven Rostedt <[email protected]>Cc Julia Cartwright <[email protected]>Cc Ingo Molnar <[email protected]>Cc Ard Biesheuvel <[email protected]>Cc Jason Baron <[email protected]>Cc Linus Torvalds <[email protected]>Cc Jiri Kosina <[email protected]>Cc Thomas Gleixner <[email protected]>Cc Masami Hiramatsu <[email protected]>Cc Borislav Petkov <[email protected]>Cc David Laight <[email protected]>Cc Jessica Yu <[email protected]>Cc Andy Lutomirski <[email protected]>Cc "H. Peter Anvin" <[email protected]>Signed-off-by Josh Poimboeuf <[email protected]>Signed-off-by Peter Zijlstra (Intel) <[email protected]>---
arch/x86/Kconfig | 1 arch/x86/include/asm/static_call.h | 28 +++++++++++++++++++++++++++ arch/x86/kernel/Makefile | 1 arch/x86/kernel/static_call.c | 38 +++++++++++++++++++++++++++++++++++++ 4 files changed, 68 insertions(+)create mode 100644
arch/x86/include/asm/static_call.hcreate mode 100644arch/x86/kernel/static_call.carch/x86/Kconfig +1
-0@@ -198,6 +198,7 @@ config X86 select HAVE_FUNCTION_ARG_ACCESS_API select HAVE_STACKPROTECTOR if CC_HAS_SANE_STACKPROTECTOR select HAVE_STACK_VALIDATION if X86_64 + select HAVE_STATIC_CALL select HAVE_RSEQ select HAVE_SYSCALL_TRACEPOINTS select HAVE_UNSTABLE_SCHED_CLOCKarch/x86/include/asm/static_call.h new file +28
-0@@ -0,0 +1,28 @@ +/* SPDX-License-Identifier: GPL-2.0 */ +#ifndef _ASM_STATIC_CALL_H +#define _ASM_STATIC_CALL_H + +/* +* Manually construct a 5-byte direct JMP to prevent the assembler from +* optimizing it into a 2-byte JMP. +*/ +#define __ARCH_STATIC_CALL_JMP_LABEL (key) ".L" __stringify(key ## _after_jmp) +#define __ARCH_STATIC_CALL_TRAMP_JMP (key, func) \ + ".byte 0xe9 \n" \ + ".long " #func " - " __ARCH_STATIC_CALL_JMP_LABEL(key) "\n" \ + __ARCH_STATIC_CALL_JMP_LABEL(key) ":" + +/* +* This is a permanent trampoline which does a direct jump to the function. +* The direct jump get patched by static_call_update(). +*/ +#define ARCH_DEFINE_STATIC_CALL_TRAMP (key, func) \ + asm(".pushsection .text, \"ax\" \n" \ + ".align 4 \n" \ + ".globl " STATIC_CALL_TRAMP_STR(key) " \n" \ + ".type " STATIC_CALL_TRAMP_STR(key) ", @function \n" \ + STATIC_CALL_TRAMP_STR(key) ": \n" \ + __ARCH_STATIC_CALL_TRAMP_JMP(key, func) " \n" \ + ".popsection \n") + +#endif /* _ASM_STATIC_CALL_H */ arch/x86/kernel/Makefile +0
-0@@ -63,6 +63,7 @@ obj-y += tsc.o tsc_msr.o io_delay.o rtobj-y += pci-iommu_table.o
obj-y += resource.o
obj-y += irqflags.o
+obj-y += static_call.oobj-y += process.o
obj-y += fpu/arch/x86/kernel/static_call.c new file +15
-0@@ -0,0 +1,38 @@ +// SPDX-License-Identifier: GPL-2.0 +#include <linux/static_call.h> +#include <linux/memory.h> +#include <linux/bug.h> +#include <asm/text-patching.h> +#include <asm/nospec-branch.h> + +#define CALL_INSN_SIZE 5 + +void arch_static_call_transform (void *site,void *tramp,void *func) +{ +unsigned char opcodes[CALL_INSN_SIZE ]; +unsigned char insn_opcode; +unsigned long insn; +s32 dest_relative;
arch/x86/kernel/static_call.c line 25
Quoted 10 lines
arch/x86/kernel/static_call.c new file +10
-0+ +mutex_lock (&text_mutex); + + insn = (unsigned long )tramp; + + insn_opcode = *(unsigned char *)insn; +if (insn_opcode !=0xE9 ) { +WARN_ONCE (1 ,"unexpected static call insn opcode 0x%x at %pS" , + insn_opcode, (void *)insn); + goto unlock;
This might happen if a kprobe is installed on the call, no?
I don’t know if you want to be more gentle handling of this case (or perhaps modify can_probe() to prevent such a case).
Steven Rostedt
maintainerRe: [PATCH 12/15] x86/static_call: Add out-of-line static call implementation
Quoted 118 lines · On Fri, 7 Jun 2019 06:13:58 +0000 Nadav Amit <[email protected]> wrote:
Quoted 105 lines · Peter Zijlstra
From: Josh Poimboeuf <[email protected]>
Add the x86 out-of-line static call implementation. For each key, a permanent trampoline is created which is the destination for all static calls for the given key. The trampoline has a direct jump which gets patched by
static_call_update()when the destination function changes.Cc Steven Rostedt <[email protected]>Cc Julia Cartwright <[email protected]>Cc Ingo Molnar <[email protected]>Cc Ard Biesheuvel <[email protected]>Cc Jason Baron <[email protected]>Cc Linus Torvalds <[email protected]>Cc Jiri Kosina <[email protected]>Cc Thomas Gleixner <[email protected]>Cc Masami Hiramatsu <[email protected]>Cc Borislav Petkov <[email protected]>Cc David Laight <[email protected]>Cc Jessica Yu <[email protected]>Cc Andy Lutomirski <[email protected]>Cc "H. Peter Anvin" <[email protected]>Signed-off-by Josh Poimboeuf <[email protected]>Signed-off-by Peter Zijlstra (Intel) <[email protected]>---
arch/x86/Kconfig | 1 arch/x86/include/asm/static_call.h | 28 +++++++++++++++++++++++++++ arch/x86/kernel/Makefile | 1 arch/x86/kernel/static_call.c | 38 +++++++++++++++++++++++++++++++++++++ 4 files changed, 68 insertions(+)create mode 100644
arch/x86/include/asm/static_call.hcreate mode 100644arch/x86/kernel/static_call.carch/x86/Kconfig +1
-0@@ -198,6 +198,7 @@ config X86 select HAVE_FUNCTION_ARG_ACCESS_API select HAVE_STACKPROTECTOR if CC_HAS_SANE_STACKPROTECTOR select HAVE_STACK_VALIDATION if X86_64 + select HAVE_STATIC_CALL select HAVE_RSEQ select HAVE_SYSCALL_TRACEPOINTS select HAVE_UNSTABLE_SCHED_CLOCKarch/x86/include/asm/static_call.h new file +28
-0@@ -0,0 +1,28 @@ +/* SPDX-License-Identifier: GPL-2.0 */ +#ifndef _ASM_STATIC_CALL_H +#define _ASM_STATIC_CALL_H + +/* +* Manually construct a 5-byte direct JMP to prevent the assembler from +* optimizing it into a 2-byte JMP. +*/ +#define __ARCH_STATIC_CALL_JMP_LABEL (key) ".L" __stringify(key ## _after_jmp) +#define __ARCH_STATIC_CALL_TRAMP_JMP (key, func) \ + ".byte 0xe9 \n" \ + ".long " #func " - " __ARCH_STATIC_CALL_JMP_LABEL(key) "\n" \ + __ARCH_STATIC_CALL_JMP_LABEL(key) ":" + +/* +* This is a permanent trampoline which does a direct jump to the function. +* The direct jump get patched by static_call_update(). +*/ +#define ARCH_DEFINE_STATIC_CALL_TRAMP (key, func) \ + asm(".pushsection .text, \"ax\" \n" \ + ".align 4 \n" \ + ".globl " STATIC_CALL_TRAMP_STR(key) " \n" \ + ".type " STATIC_CALL_TRAMP_STR(key) ", @function \n" \ + STATIC_CALL_TRAMP_STR(key) ": \n" \ + __ARCH_STATIC_CALL_TRAMP_JMP(key, func) " \n" \ + ".popsection \n") + +#endif /* _ASM_STATIC_CALL_H */ arch/x86/kernel/Makefile +0
-0@@ -63,6 +63,7 @@ obj-y += tsc.o tsc_msr.o io_delay.o rtobj-y += pci-iommu_table.o
obj-y += resource.o
obj-y += irqflags.o
+obj-y += static_call.oobj-y += process.o
obj-y += fpu/arch/x86/kernel/static_call.c new file +15
-0@@ -0,0 +1,38 @@ +// SPDX-License-Identifier: GPL-2.0 +#include <linux/static_call.h> +#include <linux/memory.h> +#include <linux/bug.h> +#include <asm/text-patching.h> +#include <asm/nospec-branch.h> + +#define CALL_INSN_SIZE 5 + +void arch_static_call_transform (void *site,void *tramp,void *func) +{ +unsigned char opcodes[CALL_INSN_SIZE ]; +unsigned char insn_opcode; +unsigned long insn; +s32 dest_relative;arch/x86/kernel/static_call.c line 25
Quoted 10 lines
arch/x86/kernel/static_call.c new file +10
-0+ +mutex_lock (&text_mutex); + + insn = (unsigned long )tramp; + + insn_opcode = *(unsigned char *)insn; +if (insn_opcode !=0xE9 ) { +WARN_ONCE (1 ,"unexpected static call insn opcode 0x%x at %pS" , + insn_opcode, (void *)insn); + goto unlock;This might happen if a kprobe is installed on the call, no?
Quoted 3 lines
I don’t know if you want to be more gentle handling of this case (or perhaps modify
can_probe()to prevent such a case).
Perhaps it is better to block kprobes from attaching to a static call. Or have it use the static call directly as it does with ftrace. But that would probably be much more work.
-- Steve
Peter Zijlstra
maintainerRe: [PATCH 12/15] x86/static_call: Add out-of-line static call implementation
In reply to Nadav AmitQuoted 21 lines · Nadav Amit
Quoted 8 lines · Peter Zijlstra
arch/x86/kernel/static_call.c new file +6
-0+void arch_static_call_transform (void *site,void *tramp,void *func) +{ +unsigned char opcodes[CALL_INSN_SIZE ]; +unsigned char insn_opcode; +unsigned long insn; +s32 dest_relative;arch/x86/kernel/static_call.c line 25
Quoted 10 lines
arch/x86/kernel/static_call.c new file +10
-0+ +mutex_lock (&text_mutex); + + insn = (unsigned long )tramp; + + insn_opcode = *(unsigned char *)insn; +if (insn_opcode !=0xE9 ) { +WARN_ONCE (1 ,"unexpected static call insn opcode 0x%x at %pS" , + insn_opcode, (void *)insn); + goto unlock;This might happen if a kprobe is installed on the call, no?
Quoted 3 lines
I don’t know if you want to be more gentle handling of this case (or perhaps modify
can_probe()to prevent such a case).
yuck.. yes, that's something that needs consideration.
Peter Zijlstra
maintainerRe: [PATCH 12/15] x86/static_call: Add out-of-line static call implementation
Quoted 22 lines · Peter Zijlstra
Quoted 21 lines · Nadav Amit
Quoted 8 lines · Peter Zijlstra
arch/x86/kernel/static_call.c new file +6
-0+void arch_static_call_transform (void *site,void *tramp,void *func) +{ +unsigned char opcodes[CALL_INSN_SIZE ]; +unsigned char insn_opcode; +unsigned long insn; +s32 dest_relative;arch/x86/kernel/static_call.c line 25
Quoted 10 lines
arch/x86/kernel/static_call.c new file +10
-0+ +mutex_lock (&text_mutex); + + insn = (unsigned long )tramp; + + insn_opcode = *(unsigned char *)insn; +if (insn_opcode !=0xE9 ) { +WARN_ONCE (1 ,"unexpected static call insn opcode 0x%x at %pS" , + insn_opcode, (void *)insn); + goto unlock;This might happen if a kprobe is installed on the call, no?
Quoted 5 lines
Quoted 3 lines
I don’t know if you want to be more gentle handling of this case (or perhaps modify
can_probe()to prevent such a case).yuck.. yes, that's something that needs consideration.
For jump_label this is avoided by jump_label_text_reserved(), I'm thinking static_call should do the same.
Peter Zijlstra
maintainerPATCH 13/15 x86/static_call: Add inline static call implementation for x86-64
In reply to Peter ZijlstraFrom: Josh Poimboeuf <[email protected]>
Add the inline static call implementation for x86-64. For each key, a temporary trampoline is created, named __static_call_tramp_<key>. The trampoline has an indirect jump to the destination function.
Objtool uses the trampoline naming convention to detect all the call sites. It then annotates those call sites in the .static_call_sites section.
During boot (and module init), the call sites are patched to call directly into the destination function. The temporary trampoline is then no longer used.
--- arch/x86/Kconfig | 3 arch/x86/include/asm/static_call.h | 28 ++++- arch/x86/kernel/asm-offsets.c | 6 + arch/x86/kernel/static_call.c | 12 +- include/linux/static_call.h | 2 tools/objtool/Makefile | 3 tools/objtool/check.c | 125 +++++++++++++++++++++++- tools/objtool/check.h | 2 tools/objtool/elf.h | 1 tools/objtool/include/linux/static_call_types.h | 19 +++ tools/objtool/sync-check.sh | 1 11 files changed, 193 insertions(+), 9 deletions(-) create mode 100644 tools/objtool/include/linux/static_call_types.h
arch/x86/Kconfig +2 -1
@@ -199,6 +199,7 @@ config X86 select HAVE_STACKPROTECTOR if CC_HAS_SANE_STACKPROTECTOR select HAVE_STACK_VALIDATION if X86_64 select HAVE_STATIC_CALL + select HAVE_STATIC_CALL_INLINE if HAVE_STACK_VALIDATION select HAVE_RSEQ select HAVE_SYSCALL_TRACEPOINTS select HAVE_UNSTABLE_SCHED_CLOCK
@@ -213,6 +214,7 @@ config X86 select RTC_MC146818_LIB select SPARSE_IRQ select SRCU + select STACK_VALIDATION if HAVE_STACK_VALIDATION && (HAVE_STATIC_CALL_INLINE || RETPOLINE) select SYSCTL_EXCEPTION_TRACE select THREAD_INFO_IN_TASK select USER_STACKTRACE_SUPPORT
@@ -439,7 +441,6 @@ config GOLDFISH config RETPOLINE bool "Avoid speculative indirect branches in kernel" default y - select STACK_VALIDATION if HAVE_STACK_VALIDATION help Compile kernel with the retpoline compiler options to guard against kernel-to-user data leaks by avoiding speculative indirect
arch/x86/include/asm/static_call.h +26 -2
@@ -2,6 +2,20 @@#ifndef _ASM_STATIC_CALL_H#define _ASM_STATIC_CALL_H +#include <asm/asm-offsets.h> + +#ifdef CONFIG_HAVE_STATIC_CALL_INLINE + +/* +* This trampoline is only used during boot / module init, so it's safe to use +* the indirect branch without a retpoline. +*/ +#define __ARCH_STATIC_CALL_TRAMP_JMP (key, func) \ + ANNOTATE_RETPOLINE_SAFE \ + "jmpq *" __stringify(key) "+" __stringify(SC_KEY_func) "(%rip) \n" + +#else /* !CONFIG_HAVE_STATIC_CALL_INLINE */ +/* * Manually construct a 5-byte direct JMP to prevent the assembler from * optimizing it into a 2-byte JMP.
@@ -12,9 +26,19 @@ ".long " #func " - " __ARCH_STATIC_CALL_JMP_LABEL(key) "\n" \ __ARCH_STATIC_CALL_JMP_LABEL(key) ":" +#endif /* !CONFIG_HAVE_STATIC_CALL_INLINE */ +/* -* This is a permanent trampoline which does a direct jump to the function. -* The direct jump get patched by static_call_update(). +* For CONFIG_HAVE_STATIC_CALL_INLINE, this is a temporary trampoline which +* uses the current value of the key->func pointer to do an indirect jump to +* the function. This trampoline is only used during boot, before the call +* sites get patched by static_call_update(). The name of this trampoline has +* a magical aspect: objtool uses it to find static call sites so it can create +* the .static_call_sites section. +* +* For CONFIG_HAVE_STATIC_CALL, this is a permanent trampoline which +* does a direct jump to the function. The direct jump gets patched by +* static_call_update(). */ #define ARCH_DEFINE_STATIC_CALL_TRAMP (key, func) \ asm(".pushsection .text, \"ax\" \n" \
arch/x86/kernel/asm-offsets.c +6 -0
@@ -12,6 +12,7 @@#include <linux/hardirq.h> #include <linux/suspend.h> #include <linux/kbuild.h> +#include <linux/static_call.h> #include <asm/processor.h> #include <asm/thread_info.h> #include <asm/sigframe.h>
@@ -104,4 +105,9 @@ static void __used common(void)OFFSET (TSS_sp0, tss_struct, x86_tss.sp0);OFFSET (TSS_sp1, tss_struct, x86_tss.sp1);OFFSET (TSS_sp2, tss_struct, x86_tss.sp2); + +#ifdef CONFIG_HAVE_STATIC_CALL_INLINE +BLANK (); +OFFSET (SC_KEY_func, static_call_key, func); +#endif }
arch/x86/kernel/static_call.c +9 -3
@@ -10,16 +10,22 @@void arch_static_call_transform (void *site,void *tramp,void *func) {unsigned char opcodes[CALL_INSN_SIZE ]; -unsigned char insn_opcode; +unsigned char insn_opcode, expected;unsigned long insn;s32 dest_relative;mutex_lock (&text_mutex); - insn = (unsigned long )tramp; +if (IS_ENABLED (CONFIG_HAVE_STATIC_CALL_INLINE )) { + insn = (unsigned long )site; + expected =0xE8 ;/* CALL */ + }else { + insn = (unsigned long )tramp; + expected =0xE9 ;/* JMP */ + } insn_opcode = *(unsigned char *)insn; -if (insn_opcode !=0xE9 ) { +if (insn_opcode != expected) {WARN_ONCE (1 ,"unexpected static call insn opcode 0x%x at %pS" , insn_opcode, (void *)insn); goto unlock;
include/linux/static_call.h +1 -1
@@ -59,7 +59,7 @@#include <linux/cpu.h> #include <linux/static_call_types.h> -#ifdef CONFIG_HAVE_STATIC_CALL +#if defined(CONFIG_HAVE_STATIC_CALL ) && !defined(COMPILE_OFFSETS )#include <asm/static_call.h> extern void arch_static_call_transform (void *site,void *tramp,void *func);#endif
tools/objtool/Makefile +2 -1
@@ -33,7 +33,8 @@ all: $(OBJTOOL) INCLUDES := -I$(srctree)/tools/include \ -I$(srctree)/tools/arch/$(HOSTARCH)/include/uapi \ - -I$(srctree)/tools/objtool/arch/$(ARCH)/include + -I$(srctree)/tools/objtool/arch/$(ARCH)/include \ + -I$(srctree)/tools/objtool/include WARNINGS := $(EXTRA_WARNINGS) -Wno-switch-default -Wno-switch-enum -Wno-packed CFLAGS += -Werror $(WARNINGS) $(KBUILD_HOSTCFLAGS) -g $(INCLUDES) $(LIBELF_FLAGS) LDFLAGS += $(LIBELF_LIBS) $(LIBSUBCMD) $(KBUILD_HOSTLDFLAGS)
tools/objtool/check.c +124 -1
185 lines not on this page · raw message
tools/objtool/check.h +2 -0
@@ -28,6 +28,7 @@ struct insn_state {struct instruction {struct list_head list;struct hlist_node hash; +struct list_head static_call_node;struct section *sec;unsigned long offset;unsigned int len;
@@ -49,6 +50,7 @@ struct objtool_file {struct elf *elf;struct list_head insn_list;DECLARE_HASHTABLE (insn_hash,16 ); +struct list_head static_call_list;bool ignore_unreachables, c_file, hints, rodata; };
tools/objtool/elf.h +1 -0
@@ -51,6 +51,7 @@ struct symbol {unsigned int len;struct symbol *pfunc, *cfunc, *alias;bool uaccess_safe; +bool static_call_tramp; };struct rela {
tools/objtool/include/linux/static_call_types.h new file +19 -0
@@ -0,0 +1,19 @@ +/* SPDX-License-Identifier: GPL-2.0 */ +#ifndef _STATIC_CALL_TYPES_H +#define _STATIC_CALL_TYPES_H + +#include <linux/stringify.h> + +#define STATIC_CALL_TRAMP_PREFIX ____static_call_tramp_ +#define STATIC_CALL_TRAMP_PREFIX_STR __stringify(STATIC_CALL_TRAMP_PREFIX) + +#define STATIC_CALL_TRAMP (key) __PASTE(STATIC_CALL_TRAMP_PREFIX, key) +#define STATIC_CALL_TRAMP_STR (key) __stringify(STATIC_CALL_TRAMP(key)) + +/* The static call site table is created by objtool. */ +struct static_call_site { +s32 addr; +s32 key; +}; + +#endif /* _STATIC_CALL_TYPES_H */
tools/objtool/sync-check.sh +1 -0
@@ -10,6 +10,7 @@ arch/x86/include/asm/insn.h arch/x86/include/asm/inat.h arch/x86/include/asm/inat_types.h arch/x86/include/asm/orc_types.h +include/linux/static_call_types.h ' check()
Re: [PATCH 13/15] x86/static_call: Add inline static call implementation for x86-64
Quoted 147 lines · Peter Zijlstra
From: Josh Poimboeuf <[email protected]>
Add the inline static call implementation for x86-64. For each key, a temporary trampoline is created, named __static_call_tramp_<key>. The trampoline has an indirect jump to the destination function.
Objtool uses the trampoline naming convention to detect all the call sites. It then annotates those call sites in the .static_call_sites section.
During boot (and module init), the call sites are patched to call directly into the destination function. The temporary trampoline is then no longer used.
Cc Steven Rostedt <[email protected]>Cc Julia Cartwright <[email protected]>Cc Ingo Molnar <[email protected]>Cc Ard Biesheuvel <[email protected]>Cc Jason Baron <[email protected]>Cc Linus Torvalds <[email protected]>Cc Jiri Kosina <[email protected]>Cc Thomas Gleixner <[email protected]>Cc Masami Hiramatsu <[email protected]>Cc Borislav Petkov <[email protected]>Cc David Laight <[email protected]>Cc Jessica Yu <[email protected]>Cc Andy Lutomirski <[email protected]>Cc "H. Peter Anvin" <[email protected]>Signed-off-by Josh Poimboeuf <[email protected]>Signed-off-by Peter Zijlstra (Intel) <[email protected]>---
arch/x86/Kconfig | 3 arch/x86/include/asm/static_call.h | 28 ++++- arch/x86/kernel/asm-offsets.c | 6 + arch/x86/kernel/static_call.c | 12 +- include/linux/static_call.h | 2 tools/objtool/Makefile | 3 tools/objtool/check.c | 125 +++++++++++++++++++++++- tools/objtool/check.h | 2 tools/objtool/elf.h | 1 tools/objtool/include/linux/static_call_types.h | 19 +++ tools/objtool/sync-check.sh | 1 11 files changed, 193 insertions(+), 9 deletions(-)create mode 100644
tools/objtool/include/linux/static_call_types.harch/x86/Kconfig +2
-0@@ -199,6 +199,7 @@ config X86 select HAVE_STACKPROTECTOR if CC_HAS_SANE_STACKPROTECTOR select HAVE_STACK_VALIDATION if X86_64 select HAVE_STATIC_CALL + select HAVE_STATIC_CALL_INLINE if HAVE_STACK_VALIDATION select HAVE_RSEQ select HAVE_SYSCALL_TRACEPOINTS select HAVE_UNSTABLE_SCHED_CLOCK@@ -213,6 +214,7 @@ config X86 select RTC_MC146818_LIB select SPARSE_IRQ select SRCU + select STACK_VALIDATION if HAVE_STACK_VALIDATION && (HAVE_STATIC_CALL_INLINE || RETPOLINE) select SYSCTL_EXCEPTION_TRACE select THREAD_INFO_IN_TASK select USER_STACKTRACE_SUPPORT@@ -439,7 +441,6 @@ config GOLDFISHconfig RETPOLINE
bool "Avoid speculative indirect branches in kernel" default yarch/x86/Kconfig +0
-1- select STACK_VALIDATION if HAVE_STACK_VALIDATIONhelp
Compile kernel with the retpoline compiler options to guard against kernel-to-user data leaks by avoiding speculative indirectarch/x86/include/asm/static_call.h +26
-2@@ -2,6 +2,20 @@#ifndef _ASM_STATIC_CALL_H#define _ASM_STATIC_CALL_H +#include <asm/asm-offsets.h> + +#ifdef CONFIG_HAVE_STATIC_CALL_INLINE + +/* +* This trampoline is only used during boot / module init, so it's safe to use +* the indirect branch without a retpoline. +*/ +#define __ARCH_STATIC_CALL_TRAMP_JMP (key, func) \ + ANNOTATE_RETPOLINE_SAFE \ + "jmpq *" __stringify(key) "+" __stringify(SC_KEY_func) "(%rip) \n" + +#else /* !CONFIG_HAVE_STATIC_CALL_INLINE */ +/* * Manually construct a 5-byte direct JMP to prevent the assembler from * optimizing it into a 2-byte JMP. @@ -12,9 +26,19 @@ ".long " #func " - " __ARCH_STATIC_CALL_JMP_LABEL(key) "\n" \ __ARCH_STATIC_CALL_JMP_LABEL(key) ":" +#endif /* !CONFIG_HAVE_STATIC_CALL_INLINE */ +/* -* This is a permanent trampoline which does a direct jump to the function. -* The direct jump get patched by static_call_update(). +* For CONFIG_HAVE_STATIC_CALL_INLINE, this is a temporary trampoline which +* uses the current value of the key->func pointer to do an indirect jump to +* the function. This trampoline is only used during boot, before the call +* sites get patched by static_call_update(). The name of this trampoline has +* a magical aspect: objtool uses it to find static call sites so it can create +* the .static_call_sites section. +* +* For CONFIG_HAVE_STATIC_CALL, this is a permanent trampoline which +* does a direct jump to the function. The direct jump gets patched by +* static_call_update(). */ #define ARCH_DEFINE_STATIC_CALL_TRAMP (key, func) \ asm(".pushsection .text, \"ax\" \n" \arch/x86/kernel/asm-offsets.c +6
-0@@ -12,6 +12,7 @@#include <linux/hardirq.h> #include <linux/suspend.h> #include <linux/kbuild.h> +#include <linux/static_call.h> #include <asm/processor.h> #include <asm/thread_info.h> #include <asm/sigframe.h> @@ -104,4 +105,9 @@ static void __used common(void)OFFSET (TSS_sp0, tss_struct, x86_tss.sp0);OFFSET (TSS_sp1, tss_struct, x86_tss.sp1);OFFSET (TSS_sp2, tss_struct, x86_tss.sp2); + +#ifdef CONFIG_HAVE_STATIC_CALL_INLINE +BLANK (); +OFFSET (SC_KEY_func, static_call_key, func); +#endif }arch/x86/kernel/static_call.c +0
-1@@ -10,16 +10,22 @@void arch_static_call_transform (void *site,void *tramp,void *func) {unsigned char opcodes[CALL_INSN_SIZE ]; -unsigned char insn_opcode;
arch/x86/kernel/static_call.c line 21
Quoted 10 lines
arch/x86/kernel/static_call.c +4
-1+unsigned char insn_opcode, expected;unsigned long insn;s32 dest_relative;mutex_lock (&text_mutex); - insn = (unsigned long )tramp; +if (IS_ENABLED (CONFIG_HAVE_STATIC_CALL_INLINE )) { + insn = (unsigned long )site; + expected =0xE8 ;/* CALL */
RELATIVECALL_OPCODE ?
arch/x86/kernel/static_call.c line 24
Quoted 3 lines
arch/x86/kernel/static_call.c +3
-0+ }else { + insn = (unsigned long )tramp; + expected =0xE9 ;/* JMP */
RELATIVEJUMP_OPCODE ?
( I did not review the objtool parts )
Re: [PATCH 13/15] x86/static_call: Add inline static call implementation for x86-64
In reply to Peter ZijlstraQuoted 9 lines · Peter Zijlstra
arch/x86/include/asm/static_call.h +3
-0@@ -2,6 +2,20 @@#ifndef _ASM_STATIC_CALL_H#define _ASM_STATIC_CALL_H +#include <asm/asm-offsets.h> + +#ifdef CONFIG_HAVE_STATIC_CALL_INLINE
arch/x86/include/asm/static_call.h line 17
Quoted 10 lines
arch/x86/include/asm/static_call.h +10
-0+ +/* +* This trampoline is only used during boot / module init, so it's safe to use +* the indirect branch without a retpoline. +*/ +#define __ARCH_STATIC_CALL_TRAMP_JMP (key, func) \ + ANNOTATE_RETPOLINE_SAFE \ + "jmpq *" __stringify(key) "+" __stringify(SC_KEY_func) "(%rip) \n" + +#else /* !CONFIG_HAVE_STATIC_CALL_INLINE */
I wonder if we can simplify this (and drop the indirect branch) by getting rid of the above cruft, and instead just use the out-of-line trampoline as the default for inline as well.
Then the inline case could fall back to the out-of-line implementation (by patching the trampoline's jmp dest) before static_call_initialized is set.
Signature · 1 line
Josh
Re: [PATCH 13/15] x86/static_call: Add inline static call implementation for x86-64
Quoted 27 lines · Josh Poimboeuf
Quoted 9 lines · Peter Zijlstra
arch/x86/include/asm/static_call.h +3
-0@@ -2,6 +2,20 @@#ifndef _ASM_STATIC_CALL_H#define _ASM_STATIC_CALL_H +#include <asm/asm-offsets.h> + +#ifdef CONFIG_HAVE_STATIC_CALL_INLINE arch/x86/include/asm/static_call.h line 17
Quoted 10 lines
arch/x86/include/asm/static_call.h +10
-0+ +/* +* This trampoline is only used during boot / module init, so it's safe to use +* the indirect branch without a retpoline. +*/ +#define __ARCH_STATIC_CALL_TRAMP_JMP (key, func) \ + ANNOTATE_RETPOLINE_SAFE \ + "jmpq *" __stringify(key) "+" __stringify(SC_KEY_func) "(%rip) \n" + +#else /* !CONFIG_HAVE_STATIC_CALL_INLINE */ I wonder if we can simplify this (and drop the indirect branch) by getting rid of the above cruft, and instead just use the out-of-line trampoline as the default for inline as well.
Quoted 3 lines
Then the inline case could fall back to the out-of-line implementation (by patching the trampoline's jmp dest) before
static_call_initializedis set.
I must be missing some context - but what guarantees that this indirect branch would be exactly 5 bytes long? Isn’t there an assumption that this would be the case? Shouldn’t there be some handling of the padding?
Re: [PATCH 13/15] x86/static_call: Add inline static call implementation for x86-64
Quoted 27 lines · Nadav Amit
Quoted 27 lines · Josh Poimboeuf
Quoted 9 lines · Peter Zijlstra
arch/x86/include/asm/static_call.h +3
-0@@ -2,6 +2,20 @@#ifndef _ASM_STATIC_CALL_H#define _ASM_STATIC_CALL_H +#include <asm/asm-offsets.h> + +#ifdef CONFIG_HAVE_STATIC_CALL_INLINE arch/x86/include/asm/static_call.h line 17
Quoted 10 lines
arch/x86/include/asm/static_call.h +10
-0+ +/* +* This trampoline is only used during boot / module init, so it's safe to use +* the indirect branch without a retpoline. +*/ +#define __ARCH_STATIC_CALL_TRAMP_JMP (key, func) \ + ANNOTATE_RETPOLINE_SAFE \ + "jmpq *" __stringify(key) "+" __stringify(SC_KEY_func) "(%rip) \n" + +#else /* !CONFIG_HAVE_STATIC_CALL_INLINE */ I wonder if we can simplify this (and drop the indirect branch) by getting rid of the above cruft, and instead just use the out-of-line trampoline as the default for inline as well.
Quoted 7 lines
Quoted 3 lines
Then the inline case could fall back to the out-of-line implementation (by patching the trampoline's jmp dest) before
static_call_initializedis set.I must be missing some context - but what guarantees that this indirect branch would be exactly 5 bytes long? Isn’t there an assumption that this would be the case? Shouldn’t there be some handling of the padding?
We don't patch the indirect branch. It's just part of a temporary trampoline which is called by the call site, and which does "jmp key->func" during boot until static call initialization is done.
(Though I'm suggesting removing that.)
Signature · 1 line
Josh
Re: [PATCH 13/15] x86/static_call: Add inline static call implementation for x86-64
Quoted 38 lines · Josh Poimboeuf
Quoted 27 lines · Nadav Amit
Quoted 27 lines · Josh Poimboeuf
Quoted 9 lines · Peter Zijlstra
arch/x86/include/asm/static_call.h +3
-0@@ -2,6 +2,20 @@#ifndef _ASM_STATIC_CALL_H#define _ASM_STATIC_CALL_H +#include <asm/asm-offsets.h> + +#ifdef CONFIG_HAVE_STATIC_CALL_INLINE arch/x86/include/asm/static_call.h line 17
Quoted 10 lines
arch/x86/include/asm/static_call.h +10
-0+ +/* +* This trampoline is only used during boot / module init, so it's safe to use +* the indirect branch without a retpoline. +*/ +#define __ARCH_STATIC_CALL_TRAMP_JMP (key, func) \ + ANNOTATE_RETPOLINE_SAFE \ + "jmpq *" __stringify(key) "+" __stringify(SC_KEY_func) "(%rip) \n" + +#else /* !CONFIG_HAVE_STATIC_CALL_INLINE */ I wonder if we can simplify this (and drop the indirect branch) by getting rid of the above cruft, and instead just use the out-of-line trampoline as the default for inline as well.
Quoted 7 lines
Quoted 3 lines
Then the inline case could fall back to the out-of-line implementation (by patching the trampoline's jmp dest) before
static_call_initializedis set.I must be missing some context - but what guarantees that this indirect branch would be exactly 5 bytes long? Isn’t there an assumption that this would be the case? Shouldn’t there be some handling of the padding?
Quoted 5 lines
We don't patch the indirect branch. It's just part of a temporary trampoline which is called by the call site, and which does "jmp
key->func" during boot until static call initialization is done.(Though I'm suggesting removing that.)
Oh... I see.
On another note - even if this branch is only executed during module initialization, it does seem safer to use a retpoline instead of an indirect branch (consider a branch that is run many times on one hardware thread on SMT, when STIBP is not set, and attacker code is run on the second thread).
I guess you don’t simply call the retpoline code so since you don’t have a clobbered register to hold the target. But it still seems possible to use a retpoline. Anyhow, it might be a moot discussion if this code is removed.
Peter Zijlstra
maintainerRe: [PATCH 13/15] x86/static_call: Add inline static call implementation for x86-64
In reply to Josh PoimboeufQuoted 25 lines · Josh Poimboeuf
Quoted 9 lines · Peter Zijlstra
arch/x86/include/asm/static_call.h +3
-0@@ -2,6 +2,20 @@#ifndef _ASM_STATIC_CALL_H#define _ASM_STATIC_CALL_H +#include <asm/asm-offsets.h> + +#ifdef CONFIG_HAVE_STATIC_CALL_INLINE arch/x86/include/asm/static_call.h line 17
Quoted 10 lines
arch/x86/include/asm/static_call.h +10
-0+ +/* +* This trampoline is only used during boot / module init, so it's safe to use +* the indirect branch without a retpoline. +*/ +#define __ARCH_STATIC_CALL_TRAMP_JMP (key, func) \ + ANNOTATE_RETPOLINE_SAFE \ + "jmpq *" __stringify(key) "+" __stringify(SC_KEY_func) "(%rip) \n" + +#else /* !CONFIG_HAVE_STATIC_CALL_INLINE */ I wonder if we can simplify this (and drop the indirect branch) by getting rid of the above cruft, and instead just use the out-of-line trampoline as the default for inline as well.
Quoted 3 lines
Then the inline case could fall back to the out-of-line implementation (by patching the trampoline's jmp dest) before
static_call_initializedis set.
I think I've got that covered. I changed arch_static_call_transform() to (always) first rewrite the trampoline and then the in-line site.
That way, when early/module crud comes in that still points at the trampoline, it will jump to the right place.
I've not even compiled yet, but it ought to work ;-)
Peter Zijlstra
maintainerPATCH 14/15 static_call: Simple self-test module
In reply to Peter Zijlstra--- lib/Kconfig.debug | 8 ++++++++ lib/Makefile | 1 + lib/test_static_call.c | 41 +++++++++++++++++++++++++++++++++++++++++ 3 files changed, 50 insertions(+)
lib/Kconfig.debug +8 -0
@@ -1955,6 +1955,14 @@ config TEST_STATIC_KEYS If unsure, say N. +config TEST_STATIC_CALL + tristate "Test static call" + depends on m + help + Test the static call interfaces. + + If unsure, say N. + config TEST_KMOD tristate "kmod stress tester" depends on m
lib/Makefile +1 -0
@@ -79,6 +79,7 @@ obj-$(CONFIG_TEST_SORT) += test_sort.o obj-$(CONFIG_TEST_USER_COPY) += test_user_copy.o obj-$(CONFIG_TEST_STATIC_KEYS) += test_static_keys.o obj-$(CONFIG_TEST_STATIC_KEYS) += test_static_key_base.o +obj-$(CONFIG_TEST_STATIC_CALL) += test_static_call.o obj-$(CONFIG_TEST_PRINTF) += test_printf.o obj-$(CONFIG_TEST_BITMAP) += test_bitmap.o obj-$(CONFIG_TEST_STRSCPY) += test_strscpy.o
lib/test_static_call.c new file +41 -0
41 lines not on this page · raw message
Re: [PATCH 14/15] static_call: Simple self-test module
Quoted 12 lines · Peter Zijlstra
Signed-off-by Peter Zijlstra (Intel) <[email protected]>--- lib/Kconfig.debug | 8 ++++++++ lib/Makefile | 1 + lib/test_static_call.c | 41 +++++++++++++++++++++++++++++++++++++++++ 3 files changed, 50 insertions(+)lib/Kconfig.debug +0
-0@@ -1955,6 +1955,14 @@ config TEST_STATIC_KEYS
lib/Kconfig.debug config TEST_STATIC_KEYS · line 1965
Quoted 10 lines
lib/Kconfig.debug +8
-0If unsure, say N. +config TEST_STATIC_CALL + tristate "Test static call" + depends on m + help + Test the static call interfaces. + + If unsure, say N. +
Any reason why we wouldn't just make this a built-in boot time test?
Signature · 1 line
Josh
Peter Zijlstra
maintainerRe: [PATCH 14/15] static_call: Simple self-test module
Quoted 15 lines · Josh Poimboeuf
Quoted 14 lines · Peter Zijlstra
Signed-off-by Peter Zijlstra (Intel) <[email protected]>--- lib/Kconfig.debug | 8 ++++++++ lib/Makefile | 1 + lib/test_static_call.c | 41 +++++++++++++++++++++++++++++++++++++++++ 3 files changed, 50 insertions(+)lib/Kconfig.debug +0
-0@@ -1955,6 +1955,14 @@ config TEST_STATIC_KEYS If unsure, say N.
Quoted 10 lines
lib/Kconfig.debug config TEST_STATIC_KEYS · line 1965
Quoted 8 lines
lib/Kconfig.debug +8
-0+config TEST_STATIC_CALL + tristate "Test static call" + depends on m + help + Test the static call interfaces. + + If unsure, say N. +Any reason why we wouldn't just make this a built-in boot time test?
None what so ever; but I did copy paste from the static_key stuff and that has it for some rasin.
Re: [PATCH 14/15] static_call: Simple self-test module
Quoted 27 lines · Peter Zijlstra
Quoted 15 lines · Josh Poimboeuf
Quoted 14 lines · Peter Zijlstra
Signed-off-by Peter Zijlstra (Intel) <[email protected]>--- lib/Kconfig.debug | 8 ++++++++ lib/Makefile | 1 + lib/test_static_call.c | 41 +++++++++++++++++++++++++++++++++++++++++ 3 files changed, 50 insertions(+)lib/Kconfig.debug +0
-0@@ -1955,6 +1955,14 @@ config TEST_STATIC_KEYS If unsure, say N.Quoted 10 lines
lib/Kconfig.debug config TEST_STATIC_KEYS · line 1965
Quoted 8 lines
lib/Kconfig.debug +8
-0+config TEST_STATIC_CALL + tristate "Test static call" + depends on m + help + Test the static call interfaces. + + If unsure, say N. +Any reason why we wouldn't just make this a built-in boot time test?
Quoted 2 lines
None what so ever; but I did copy paste from the
static_keystuff and that has it for some rasin.
Their functionality is pretty crucial, and I doubt anybody is manually building and loading these tests? Seems like built-in tests would be wiser for both static calls/keys.
Signature · 1 line
Josh
Peter Zijlstra
maintainerPATCH 15/15 tracepoints: Use static_call
In reply to Peter ZijlstraFrom: Steven Rostedt (VMware) <[email protected]>
... Changelog goes here ...
--- include/linux/tracepoint-defs.h | 4 ++ include/linux/tracepoint.h | 73 ++++++++++++++++++++++++++-------------- include/trace/define_trace.h | 14 +++---- kernel/tracepoint.c | 25 +++++++++++-- 4 files changed, 80 insertions(+), 36 deletions(-)
include/linux/tracepoint-defs.h +4 -0
@@ -11,6 +11,8 @@#include <linux/atomic.h> #include <linux/static_key.h> +struct static_call_key ; +struct trace_print_flags {unsigned long mask;const char *name;
@@ -30,6 +32,8 @@ struct tracepoint_func {struct tracepoint {const char *name;/* Tracepoint name */ struct static_key key; +struct static_call_key *static_call_key; +void *iterator;int (*regfunc)(void );void (*unregfunc)(void );struct tracepoint_func __rcu *funcs;
include/linux/tracepoint.h +48 -25
134 lines not on this page · raw message
include/trace/define_trace.h +7 -7
@@ -25,7 +25,7 @@#undef TRACE_EVENT#define TRACE_EVENT (name, proto, args, tstruct, assign, print) \ - DEFINE_TRACE(name) + DEFINE_TRACE(name, PARAMS(proto), PARAMS(args))#undef TRACE_EVENT_CONDITION#define TRACE_EVENT_CONDITION (name, proto, args, cond, tstruct, assign, print) \
@@ -39,12 +39,12 @@#undef TRACE_EVENT_FN#define TRACE_EVENT_FN (name, proto, args, tstruct, \ assign, print, reg, unreg) \ - DEFINE_TRACE_FN(name, reg, unreg) + DEFINE_TRACE_FN(name, reg, unreg, PARAMS(proto), PARAMS(args))#undef TRACE_EVENT_FN_COND#define TRACE_EVENT_FN_COND (name, proto, args, cond, tstruct, \ assign, print, reg, unreg) \ - DEFINE_TRACE_FN(name, reg, unreg) + DEFINE_TRACE_FN(name, reg, unreg, PARAMS(proto), PARAMS(args))#undef TRACE_EVENT_NOP#define TRACE_EVENT_NOP (name, proto, args, struct, assign, print)
@@ -54,15 +54,15 @@#undef DEFINE_EVENT#define DEFINE_EVENT (template, name, proto, args) \ - DEFINE_TRACE(name) + DEFINE_TRACE(name, PARAMS(proto), PARAMS(args))#undef DEFINE_EVENT_FN#define DEFINE_EVENT_FN (template, name, proto, args, reg, unreg) \ - DEFINE_TRACE_FN(name, reg, unreg) + DEFINE_TRACE_FN(name, reg, unreg, PARAMS(proto), PARAMS(args))#undef DEFINE_EVENT_PRINT#define DEFINE_EVENT_PRINT (template, name, proto, args, print) \ - DEFINE_TRACE(name) + DEFINE_TRACE(name, PARAMS(proto), PARAMS(args))#undef DEFINE_EVENT_CONDITION#define DEFINE_EVENT_CONDITION (template, name, proto, args, cond) \
@@ -70,7 +70,7 @@#undef DECLARE_TRACE#define DECLARE_TRACE (name, proto, args) \ - DEFINE_TRACE(name) + DEFINE_TRACE(name, PARAMS(proto), PARAMS(args))#undef TRACE_INCLUDE#undef __TRACE_INCLUDE
kernel/tracepoint.c +21 -4
@@ -127,7 +127,7 @@ static void debug_print_probes(struct trstatic struct tracepoint_func *func_add (struct tracepoint_func **funcs,struct tracepoint_func *tp_func, -int prio) +int prio,int *tot_probes) {struct tracepoint_func *old, *new;int nr_probes =0 ;
@@ -170,11 +170,12 @@ func_add(struct tracepoint_func **funcs, new[nr_probes +1 ].func =NULL ; *funcs = new;debug_print_probes (*funcs); + *tot_probes = nr_probes +1 ;return old; }static void *func_remove (struct tracepoint_func **funcs, -struct tracepoint_func *tp_func) +struct tracepoint_func *tp_func,int *left) {int nr_probes =0 , nr_del =0 , i;struct tracepoint_func *old, *new;
@@ -228,6 +229,7 @@ static int tracepoint_add_func(struct tr struct tracepoint_func *func, int prio) {struct tracepoint_func *old, *tp_funcs; +int probes =0 ;int ret;if (tp->regfunc && !static_key_enabled (&tp->key)) {
@@ -238,7 +240,7 @@ static int tracepoint_add_func(struct tr tp_funcs =rcu_dereference_protected (tp->funcs,lockdep_is_held (&tracepoints_mutex)); - old =func_add (&tp_funcs, func, prio); + old =func_add (&tp_funcs, func, prio, &probes);if (IS_ERR (old)) {WARN_ON_ONCE (PTR_ERR (old) != -ENOMEM );return PTR_ERR (old);
@@ -253,6 +255,13 @@ static int tracepoint_add_func(struct trrcu_assign_pointer (tp->funcs, tp_funcs);if (!static_key_enabled (&tp->key))static_key_slow_inc (&tp->key); + +if (probes ==1 ) { +__static_call_update (tp->static_call_key, tp_funcs->func); + }else { +__static_call_update (tp->static_call_key, tp->iterator); + } +release_probes (old);return 0 ; }
@@ -267,10 +276,11 @@ static int tracepoint_remove_func(struct struct tracepoint_func *func) {struct tracepoint_func *old, *tp_funcs; +int probes_left =0 ; tp_funcs =rcu_dereference_protected (tp->funcs,lockdep_is_held (&tracepoints_mutex)); - old =func_remove (&tp_funcs, func); + old =func_remove (&tp_funcs, func, &probes_left);if (IS_ERR (old)) {WARN_ON_ONCE (PTR_ERR (old) != -ENOMEM );return PTR_ERR (old);
@@ -284,6 +294,13 @@ static int tracepoint_remove_func(structif (static_key_enabled (&tp->key))static_key_slow_dec (&tp->key); } + +if (probes_left ==1 ) { +__static_call_update (tp->static_call_key, tp_funcs->func); + }else { +__static_call_update (tp->static_call_key, tp->iterator); + } +rcu_assign_pointer (tp->funcs, tp_funcs);release_probes (old);return 0 ;