x86/unwind/orc: ORC fixes
Subsystems (5)
PATCH 2/14 objtool: Fix stack offset tracking for indirect CFAs
In reply to Josh PoimboeufWhen the current frame address (CFA) is stored on the stack (i.e., cfa->base == CFI_SP_INDIRECT), objtool neglects to adjust the stack offset when there are subsequent pushes or pops. This results in bad ORC data at the end of the ENTER_IRQ_STACK macro, when it puts the previous stack pointer on the stack and does a subsequent push.
This fixes the following unwinder warning:
WARNING: can't dereference registers at 00000000f0a6bdba for ip interrupt_entry+0x9f/0xa0
--- tools/objtool/check.c | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-)
tools/objtool/check.c +1 -1
@@ -1464,7 +1464,7 @@ static int update_insn_state_regs(struct instruction *insn, struct insn_state *sstruct cfi_reg *cfa = &state->cfa;struct stack_op *op = &insn->stack_op; -if (cfa->base !=CFI_SP ) +if (cfa->base !=CFI_SP && cfa->base !=CFI_SP_INDIRECT )return 0 ;/* push */
Signature · 1 line
2.21.1
PATCH 1/14 x86/dumpstack: Add SHOW_REGS_IRET mode
In reply to Josh PoimboeufNow that __show_regs() has the concept of "modes" to indicate which registers should be displayed, replace show_iret_regs() with a new SHOW_REGS_IRET mode. This is only a cleanup and doesn't change any behavior.
--- arch/x86/include/asm/kdebug.h | 1 + arch/x86/kernel/dumpstack.c | 27 ++++++++++----------------- arch/x86/kernel/process_64.c | 7 ++++++- 3 files changed, 17 insertions(+), 18 deletions(-)
arch/x86/include/asm/kdebug.h +1 -0
@@ -23,6 +23,7 @@ enum die_val { };enum show_regs_mode { +SHOW_REGS_IRET ,SHOW_REGS_SHORT ,/* * For when userspace crashed, but we don't think it's our fault, and
arch/x86/kernel/dumpstack.c +10 -17
@@ -126,15 +126,8 @@ void show_ip(struct pt_regs *regs, const char *loglvl)show_opcodes (regs, loglvl); } -void show_iret_regs (struct pt_regs *regs) -{ -show_ip (regs,KERN_DEFAULT ); -printk (KERN_DEFAULT "RSP: %04x:%016lx EFLAGS: %08lx" , (int )regs->ss, - regs->sp, regs->flags); -} -static void show_regs_if_on_stack (struct stack_info *info,struct pt_regs *regs, -bool partial) +bool iret_only) {/* * These on_stack() checks aren't strictly necessary: the unwind code
@@ -145,17 +138,17 @@ static void show_regs_if_on_stack(struct stack_info *info, struct pt_regs *regs,* next stack, this function will be called again with the same regs so * they can be printed in the right context. */ -if (!partial &&on_stack (info, regs,sizeof (*regs))) { +if (!iret_only &&on_stack (info, regs,sizeof (*regs))) {__show_regs (regs,SHOW_REGS_SHORT ); - }else if (partial &&on_stack (info, (void *)regs +IRET_FRAME_OFFSET , -IRET_FRAME_SIZE )) { + }else if (iret_only &&on_stack (info, (void *)regs +IRET_FRAME_OFFSET , +IRET_FRAME_SIZE )) {/* * When an interrupt or exception occurs in entry code, the * full pt_regs might not have been saved yet. In that case * just print the iret frame. */ -show_iret_regs (regs); +__show_regs (regs,SHOW_REGS_IRET ); } }
@@ -166,13 +159,13 @@ void show_trace_log_lvl(struct task_struct *task, struct pt_regs *regs,struct stack_info stack_info = {0 };unsigned long visit_mask =0 ;int graph_idx =0 ; -bool partial = false; +bool iret_only = false;printk ("%sCall Trace:\n" , log_lvl);unwind_start (&state, task, regs, stack); stack = stack ? :get_stack_pointer (task, regs); - regs =unwind_get_entry_regs (&state, &partial); + regs =unwind_get_entry_regs (&state, &iret_only);/* * Iterate through the stacks, starting with the current stack pointer.
@@ -210,7 +203,7 @@ void show_trace_log_lvl(struct task_struct *task, struct pt_regs *regs,printk ("%s <%s>\n" , log_lvl, stack_name);if (regs) -show_regs_if_on_stack (&stack_info, regs, partial); +show_regs_if_on_stack (&stack_info, regs, iret_only);/* * Scan the stack, printing any text addresses we find. At the
@@ -269,9 +262,9 @@ void show_trace_log_lvl(struct task_struct *task, struct pt_regs *regs,unwind_next_frame (&state);/* if the frame has entry regs, print them */ - regs =unwind_get_entry_regs (&state, &partial); + regs =unwind_get_entry_regs (&state, &iret_only);if (regs) -show_regs_if_on_stack (&stack_info, regs, partial); +show_regs_if_on_stack (&stack_info, regs, iret_only); }if (stack_name)
arch/x86/kernel/process_64.c +6 -1
@@ -71,7 +71,12 @@ void __show_regs(struct pt_regs *regs, enum show_regs_mode mode)unsigned int fsindex, gsindex;unsigned int ds, es; -show_iret_regs (regs); +show_ip (regs,KERN_DEFAULT ); +printk (KERN_DEFAULT "RSP: %04x:%016lx EFLAGS: %08lx" , (int )regs->ss, + regs->sp, regs->flags); + +if (mode ==SHOW_REGS_IRET ) +return ;if (regs->orig_ax !=-1 )pr_cont (" ORIG_RAX: %016lx\n" , regs->orig_ax);
Signature · 1 line
2.21.1
PATCH 3/14 x86/entry/64: Fix unwind hints in register clearing code
In reply to Josh PoimboeufThe PUSH_AND_CLEAR_REGS macro zeroes each register immediately after pushing it. If an NMI or exception hits after a register is cleared, but before the UNWIND_HINT_REGS annotation, the ORC unwinder will wrongly think the previous value of the register was zero. This can confuse the unwinding process and cause it to exit early.
Because ORC is simpler than DWARF, there are a limited number of unwind annotation states, so it's not possible to add an individual unwind hint after each push/clear combination. Instead, the register clearing instructions need to be consolidated and moved to after the UNWIND_HINT_REGS annotation.
--- arch/x86/entry/calling.h | 40 +++++++++++++++++++++------------------- 1 file changed, 21 insertions(+), 19 deletions(-)
arch/x86/entry/calling.h +21 -19
@@ -98,13 +98,6 @@ For 32-bit we have the following conventions - kernel is built with#define SIZEOF_PTREGS 21*8 .macro PUSH_AND_CLEAR_REGS rdx=%rdx rax=%rax save_ret=0 -/* -* Push registers and sanitize registers of values that a -* speculation attack might otherwise want to exploit. The -* lower registers are likely clobbered well before they -* could be put to use in a speculative execution gadget. -* Interleave XOR with PUSH for better uop scheduling: -*/ .if \save_ret pushq %rsi /* pt_regs->si */ movq 8(%rsp), %rsi /* temporarily store the return address in %rsi */
@@ -114,34 +107,43 @@ For 32-bit we have the following conventions - kernel is built with pushq %rsi/* pt_regs->si */ .endif pushq \rdx /* pt_regs->dx */ - xorl %edx, %edx /* nospec dx */ pushq %rcx /* pt_regs->cx */ - xorl %ecx, %ecx /* nospec cx */ pushq \rax /* pt_regs->ax */ pushq %r8 /* pt_regs->r8 */ - xorl %r8d, %r8d /* nospec r8 */ pushq %r9 /* pt_regs->r9 */ - xorl %r9d, %r9d /* nospec r9 */ pushq %r10 /* pt_regs->r10 */ - xorl %r10d, %r10d /* nospec r10 */ pushq %r11 /* pt_regs->r11 */ - xorl %r11d, %r11d /* nospec r11*/ pushq %rbx /* pt_regs->rbx */ - xorl %ebx, %ebx /* nospec rbx*/ pushq %rbp /* pt_regs->rbp */ - xorl %ebp, %ebp /* nospec rbp*/ pushq %r12 /* pt_regs->r12 */ - xorl %r12d, %r12d /* nospec r12*/ pushq %r13 /* pt_regs->r13 */ - xorl %r13d, %r13d /* nospec r13*/ pushq %r14 /* pt_regs->r14 */ - xorl %r14d, %r14d /* nospec r14*/ pushq %r15 /* pt_regs->r15 */ - xorl %r15d, %r15d /* nospec r15*/UNWIND_HINT_REGS + .if \save_ret pushq %rsi /* return address on top of stack */ .endif + +/* +* Sanitize registers of values that a speculation attack might +* otherwise want to exploit. The lower registers are likely clobbered +* well before they could be put to use in a speculative execution +* gadget. +*/ + xorl %edx, %edx /* nospec dx */ + xorl %ecx, %ecx /* nospec cx */ + xorl %r8d, %r8d /* nospec r8 */ + xorl %r9d, %r9d /* nospec r9 */ + xorl %r10d, %r10d /* nospec r10 */ + xorl %r11d, %r11d /* nospec r11 */ + xorl %ebx, %ebx /* nospec rbx */ + xorl %ebp, %ebp /* nospec rbp */ + xorl %r12d, %r12d /* nospec r12 */ + xorl %r13d, %r13d /* nospec r13 */ + xorl %r14d, %r14d /* nospec r14 */ + xorl %r15d, %r15d /* nospec r15 */ + .endm .macro POP_REGS pop_rdi=1 skip_r11rcx=0
Signature · 1 line
2.21.1
PATCH 4/14 x86/entry/64: Fix unwind hints in kernel exit path
In reply to Josh PoimboeufIn swapgs_restore_regs_and_return_to_usermode, after the stack is switched to the trampoline stack, the existing UNWIND_HINT_REGS hint is no longer valid, which can result in the following ORC unwinder warning:
WARNING: can't dereference registers at 000000003aeb0cdd for ip swapgs_restore_regs_and_return_to_usermode+0x93/0xa0
For full correctness, we could try to add complicated unwind hints so the unwinder could continue to find the registers, but when when it's this close to kernel exit, unwind hints aren't really needed anymore and it's fine to just use an empty hint which tells the unwinder to stop.
For consistency, also move the UNWIND_HINT_EMPTY in entry_SYSCALL_64_after_hwframe to a similar location.
--- arch/x86/entry/entry_64.S | 3 ++- 1 file changed, 2 insertions(+), 1 deletion(-)
arch/x86/entry/entry_64.S +2 -1
@@ -249,7 +249,6 @@ SYM_INNER_LABEL(entry_SYSCALL_64_after_hwframe, SYM_L_GLOBAL) */ syscall_return_via_sysret: /* rcx and r11 are already restored (see code above) */ - UNWIND_HINT_EMPTY POP_REGS pop_rdi=0 skip_r11rcx=1 /*
@@ -258,6 +257,7 @@ syscall_return_via_sysret: */ movq %rsp, %rdi movq PER_CPU_VAR(cpu_tss_rw + TSS_sp0), %rsp + UNWIND_HINT_EMPTY pushq RSP-RDI(%rdi) /* RSP */ pushq (%rdi) /* RDI */
@@ -637,6 +637,7 @@ SYM_INNER_LABEL(swapgs_restore_regs_and_return_to_usermode, SYM_L_GLOBAL) */ movq %rsp, %rdi movq PER_CPU_VAR(cpu_tss_rw + TSS_sp0), %rsp + UNWIND_HINT_EMPTY /* Copy the IRET frame to the trampoline stack. */ pushq 6*8(%rdi) /* SS */
Signature · 1 line
2.21.1
PATCH 5/14 x86/entry/64: Fix unwind hints in __switch_to_asm()
In reply to Josh PoimboeufUNWIND_HINT_FUNC has some limitations: specifically, it doesn't reset all the registers to undefined. This causes objtool to get confused about the RBP push in __switch_to_asm(), resulting in bad ORC data.
While __switch_to_asm() does do some stack magic, it's otherwise a normal callable-from-C function, so just annotate it as a function, which makes objtool happy and allows it to produces the correct hints automatically.
--- arch/x86/entry/entry_64.S | 5 ++--- 1 file changed, 2 insertions(+), 3 deletions(-)
arch/x86/entry/entry_64.S +2 -3
@@ -279,8 +279,7 @@ SYM_CODE_END(entry_SYSCALL_64) * %rdi: prev task * %rsi: next task */ -SYM_CODE_START(__switch_to_asm) - UNWIND_HINT_FUNC +SYM_FUNC_START(__switch_to_asm) /* * Save callee-saved registers * This must match the order in inactive_task_frame
@@ -321,7 +320,7 @@ SYM_CODE_START(__switch_to_asm) popq %rbp jmp __switch_to -SYM_CODE_END(__switch_to_asm) +SYM_FUNC_END(__switch_to_asm) /* * A newly forked process directly context switches into this address.
Signature · 1 line
2.21.1
PATCH 10/14 x86/unwind/orc: Prevent unwinding before ORC initialization
In reply to Josh PoimboeufIf the unwinder is called before the ORC data has been initialized, orc_find() returns NULL, and it tries to fall back to using frame pointers. This can cause some unexpected warnings during boot.
Move the 'orc_init' check from orc_find() to __unwind_init(), so that it doesn't even try to unwind from an uninitialized state.
--- arch/x86/kernel/unwind_orc.c | 6 +++--- 1 file changed, 3 insertions(+), 3 deletions(-)
arch/x86/kernel/unwind_orc.c +3 -3
@@ -148,9 +148,6 @@ static struct orc_entry *orc_find(unsigned long ip) {static struct orc_entry *orc; -if (!orc_init) -return NULL ; -if (ip ==0 )return &null_orc_entry;
@@ -591,6 +588,9 @@ EXPORT_SYMBOL_GPL(unwind_next_frame);void __unwind_start (struct unwind_state *state,struct task_struct *task,struct pt_regs *regs,unsigned long *first_frame) { +if (!orc_init) + goto done; +memset (state,0 ,sizeof (*state)); state->task = task;
Signature · 1 line
2.21.1
PATCH 8/14 x86/unwind: Prevent false warnings for non-current tasks
In reply to Josh PoimboeufThere's some daring kernel code out there which dumps the stack of another task without first making sure the task is inactive. If the task happens to be running while the unwinder is reading the stack, unusual unwinder warnings can result.
There's no race-free way for the unwinder to know whether such a warning is legitimate, so just disable unwinder warnings for all non-current tasks.
--- arch/x86/kernel/dumpstack_64.c | 3 ++- arch/x86/kernel/unwind_frame.c | 3 +++ arch/x86/kernel/unwind_orc.c | 40 +++++++++++++++++++--------------- 3 files changed, 28 insertions(+), 18 deletions(-)
arch/x86/kernel/dumpstack_64.c +2 -1
@@ -183,7 +183,8 @@ int get_stack_info(unsigned long *stack, struct task_struct *task,*/ if (visit_mask) {if (*visit_mask & (1UL << info->type)) { -printk_deferred_once (KERN_WARNING "WARNING: stack recursion on stack type %d\n" , info->type); +if (task == current) +printk_deferred_once (KERN_WARNING "WARNING: stack recursion on stack type %d\n" , info->type); goto unknown; } *visit_mask |=1UL << info->type;
arch/x86/kernel/unwind_frame.c +3 -0
@@ -344,6 +344,9 @@ bool unwind_next_frame(struct unwind_state *state)if (IS_ENABLED (CONFIG_X86_32 )) goto the_end; +if (state->task != current) + goto the_end; +if (state->regs) { printk_deferred_once(KERN_WARNING "WARNING: kernel stack regs at %p in %s:%d has bad 'bp' value %p\n",
arch/x86/kernel/unwind_orc.c +23 -17
@@ -8,7 +8,13 @@#include <asm/orc_lookup.h> #define orc_warn (fmt, ...) \ - printk_deferred_once(KERN_WARNING pr_fmt("WARNING: " fmt), ##__VA_ARGS__) + printk_deferred_once(KERN_WARNING "WARNING: " fmt, ##__VA_ARGS__) + +#define orc_warn_current(args...) \ +({ \ + if (state->task == current) \ + orc_warn(args); \ +})extern int __start_orc_unwind_ip[];extern int __stop_orc_unwind_ip[];
@@ -446,8 +452,8 @@ bool unwind_next_frame(struct unwind_state *state)case ORC_REG_R10 :if (!state->regs || !state->full_regs) { -orc_warn ("missing regs for base reg R10 at ip %pB\n" , - (void *)state->ip); +orc_warn_current ("missing R10 value at %pB\n" , + (void *)state->ip); goto err; } sp = state->regs->r10;
@@ -455,8 +461,8 @@ bool unwind_next_frame(struct unwind_state *state)case ORC_REG_R13 :if (!state->regs || !state->full_regs) { -orc_warn ("missing regs for base reg R13 at ip %pB\n" , - (void *)state->ip); +orc_warn_current ("missing R13 value at %pB\n" , + (void *)state->ip); goto err; } sp = state->regs->r13;
@@ -464,8 +470,8 @@ bool unwind_next_frame(struct unwind_state *state)case ORC_REG_DI :if (!state->regs || !state->full_regs) { -orc_warn ("missing regs for base reg DI at ip %pB\n" , - (void *)state->ip); +orc_warn_current ("missing RDI value at %pB\n" , + (void *)state->ip); goto err; } sp = state->regs->di;
@@ -473,15 +479,15 @@ bool unwind_next_frame(struct unwind_state *state)case ORC_REG_DX :if (!state->regs || !state->full_regs) { -orc_warn ("missing regs for base reg DX at ip %pB\n" , - (void *)state->ip); +orc_warn_current ("missing DX value at %pB\n" , + (void *)state->ip); goto err; } sp = state->regs->dx;break ;default : -orc_warn ("unknown SP base reg %d for ip %pB\n" , +orc_warn ("unknown SP base reg %d at %pB\n" , orc->sp_reg, (void *)state->ip); goto err; }
@@ -509,8 +515,8 @@ bool unwind_next_frame(struct unwind_state *state)case ORC_TYPE_REGS :if (!deref_stack_regs (state, sp, &state->ip, &state->sp)) { -orc_warn ("can't dereference registers at %p for ip %pB\n" , - (void *)sp, (void *)orig_ip); +orc_warn_current ("can't access registers at %pB\n" , + (void *)orig_ip); goto err; }
@@ -521,8 +527,8 @@ bool unwind_next_frame(struct unwind_state *state)case ORC_TYPE_REGS_IRET :if (!deref_stack_iret_regs (state, sp, &state->ip, &state->sp)) { -orc_warn ("can't dereference iret registers at %p for ip %pB\n" , - (void *)sp, (void *)orig_ip); +orc_warn_current ("can't access iret registers at %pB\n" , + (void *)orig_ip); goto err; }
@@ -532,7 +538,7 @@ bool unwind_next_frame(struct unwind_state *state)break ;default : -orc_warn ("unknown .orc_unwind entry type %d for ip %pB\n" , +orc_warn ("unknown .orc_unwind entry type %d at %pB\n" , orc->type, (void *)orig_ip);break ; }
@@ -564,8 +570,8 @@ bool unwind_next_frame(struct unwind_state *state)if (state->stack_info.type == prev_type &&on_stack (&state->stack_info, (void *)state->sp,sizeof (long )) && state->sp <= prev_sp) { -orc_warn ("stack going in the wrong direction? ip=%pB\n" , - (void *)orig_ip); +orc_warn_current ("stack going in the wrong direction? at %pB\n" , + (void *)orig_ip); goto err; }
Signature · 1 line
2.21.1
PATCH 7/14 x86/unwind/orc: Convert global variables to static
In reply to Josh PoimboeufThese variables aren't used outside of unwind_orc.c, make them static.
Also annotate some of them with '__ro_after_init', as applicable.
--- arch/x86/kernel/unwind_orc.c | 10 +++++----- 1 file changed, 5 insertions(+), 5 deletions(-)
arch/x86/kernel/unwind_orc.c +5 -5
@@ -15,12 +15,12 @@ extern int __stop_orc_unwind_ip[];extern struct orc_entry __start_orc_unwind[];extern struct orc_entry __stop_orc_unwind[]; -static DEFINE_MUTEX (sort_mutex); -int *cur_orc_ip_table = __start_orc_unwind_ip; -struct orc_entry *cur_orc_table = __start_orc_unwind; +static bool orc_init__ro_after_init ; +static unsigned int lookup_num_blocks__ro_after_init ; -unsigned int lookup_num_blocks; -bool orc_init; +static DEFINE_MUTEX (sort_mutex); +static int *cur_orc_ip_table = __start_orc_unwind_ip; +static struct orc_entry *cur_orc_table = __start_orc_unwind;static inline unsigned long orc_ip (const int *ip) {
Signature · 1 line
2.21.1
PATCH 6/14 x86/entry/64: Fix unwind hints in rewind_stack_do_exit()
In reply to Josh PoimboeufFrom: Jann Horn <[email protected]>
The leaq instruction in rewind_stack_do_exit moves the stack pointer directly below the pt_regs at the top of the task stack before calling do_exit(). Tell the unwinder to expect pt_regs.
--- arch/x86/entry/entry_64.S | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-)
arch/x86/entry/entry_64.S +1 -1
@@ -1739,7 +1739,7 @@ SYM_CODE_START(rewind_stack_do_exit) movq PER_CPU_VAR(cpu_current_top_of_stack), %rax leaq -PTREGS_SIZE(%rax), %rsp - UNWIND_HINT_FUNC sp_offset=PTREGS_SIZE + UNWIND_HINT_REGS call do_exit SYM_CODE_END(rewind_stack_do_exit)
Signature · 1 line
2.21.1
PATCH 11/14 x86/unwind/orc: Fix error path for bad ORC entry type
In reply to Josh PoimboeufIf the ORC entry type is unknown, nothing else can be done other than reporting an error. Exit the function instead of breaking out of the switch statement.
--- arch/x86/kernel/unwind_orc.c | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-)
arch/x86/kernel/unwind_orc.c +1 -1
@@ -537,7 +537,7 @@ bool unwind_next_frame(struct unwind_state *state)default :orc_warn ("unknown .orc_unwind entry type %d at %pB\n" , orc->type, (void *)orig_ip); -break ; + goto err; }/* Find BP: */
Signature · 1 line
2.21.1
PATCH 13/14 x86/unwind/orc: Add more unwinder warnings
In reply to Josh PoimboeufMake sure warnings are displayed for all error scenarios (except when encountering an empty unwind hint).
--- arch/x86/kernel/unwind_orc.c | 29 ++++++++++++++++++++++++----- 1 file changed, 24 insertions(+), 5 deletions(-)
arch/x86/kernel/unwind_orc.c +24 -5
@@ -450,8 +450,15 @@ bool unwind_next_frame(struct unwind_state *state)/* End-of-stack check for kernel threads: */ if (orc->sp_reg ==ORC_REG_UNDEFINED ) { -if (!orc->end) +if (!orc->end) { +/* +* This is reported as an error for the caller, but +* otherwise it isn't worth warning about. In theory +* it can only happen when hitting UNWIND_HINT_EMPTY in +* entry code, close to a kernel exit point. +*/ goto err; + } goto the_end; }
@@ -515,8 +522,11 @@ bool unwind_next_frame(struct unwind_state *state) }if (indirect) { -if (!deref_stack_reg (state, sp, &sp)) +if (!deref_stack_reg (state, sp, &sp)) { +orc_warn_current ("can't access indirect SP at %pB\n" , + (void *)state->ip); goto err; + } }/* Find IP, SP and possibly regs: */
@@ -524,8 +534,11 @@ bool unwind_next_frame(struct unwind_state *state)case ORC_TYPE_CALL : ip_p = sp -sizeof (long ); -if (!deref_stack_reg (state, ip_p, &state->ip)) +if (!deref_stack_reg (state, ip_p, &state->ip)) { +orc_warn_current ("can't access call return IP (0x%lx) at %pB\n" , + ip_p, (void *)orig_ip); goto err; + } state->ip =ftrace_graph_ret_addr (state->task, &state->graph_idx, state->ip, (void *)ip_p);
@@ -577,13 +590,19 @@ bool unwind_next_frame(struct unwind_state *state)break ;case ORC_REG_PREV_SP : -if (!deref_stack_reg (state, sp + orc->bp_offset, &state->bp)) +if (!deref_stack_reg (state, sp + orc->bp_offset, &state->bp)) { +orc_warn_current ("can't access BP (from SP) at %pB\n" , + (void *)orig_ip); goto err; + }break ;case ORC_REG_BP : -if (!deref_stack_reg (state, state->bp + orc->bp_offset, &state->bp)) +if (!deref_stack_reg (state, state->bp + orc->bp_offset, &state->bp)) { +orc_warn_current ("can't access BP at %pB\n" , + (void *)orig_ip); goto err; + }break ;default :
Signature · 1 line
2.21.1
PATCH 14/14 x86/unwind/orc: Add 'unwind_debug' cmdline option
In reply to Josh PoimboeufSometimes the one-line ORC unwinder warnings aren't very helpful. Add a new 'unwind_debug' cmdline option which will dump the full stack contents of the current task when an error condition is encountered.
--- .../admin-guide/kernel-parameters.txt | 6 +++ arch/x86/kernel/unwind_orc.c | 49 ++++++++++++++++++- 2 files changed, 54 insertions(+), 1 deletion(-)
Documentation/admin-guide/kernel-parameters.txt +6 -0
@@ -5098,6 +5098,12 @@ unknown_nmi_panic [X86] Cause panic on unknown NMI. + unwind_debug [X86-64] + Enable unwinder debug output. This can be + useful for debugging certain unwinder error + conditions, including corrupt stacks and + bad/missing unwinder metadata. + usbcore.authorized_default= [USB] Default USB device authorization: (default -1 = authorized except for wireless USB,
arch/x86/kernel/unwind_orc.c +48 -1
67 lines not on this page · raw message
Signature · 1 line
2.21.1
PATCH 9/14 x86/unwind/orc: Don't skip the first frame for inactive tasks
In reply to Josh PoimboeufFrom: Miroslav Benes <[email protected]>
When unwinding an inactive task, the ORC unwinder skips the first frame by default. If both the 'regs' and 'first_frame' parameters of unwind_start() are NULL, 'state->sp' and 'first_frame' are later initialized to the same value for an inactive task. Given there is a "less than or equal to" comparison used at the end of __unwind_start() for skipping stack frames, the first frame is skipped.
Drop the equal part of the comparison and make the behavior equivalent to the frame pointer unwinder.
--- arch/x86/kernel/unwind_orc.c | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-)
arch/x86/kernel/unwind_orc.c +1 -1
@@ -657,7 +657,7 @@ void __unwind_start(struct unwind_state *state, struct task_struct *task,/* Otherwise, skip ahead to the user-specified starting frame: */ while (!unwind_done (state) && (!on_stack (&state->stack_info, first_frame,sizeof (long )) || - state->sp <= (unsigned long )first_frame)) + state->sp < (unsigned long )first_frame))unwind_next_frame (state);return ;
Signature · 1 line
2.21.1
PATCH 12/14 x86/unwind/orc: Fix premature unwind stoppage due to IRET frames
In reply to Josh PoimboeufThe following execution path is possible:
fsnotify()
[ realign the stack and store previous SP in R10 ] <IRQ>
[ only IRET regs saved ] common_interrupt()
interrupt_entry()
<NMI>
[ full pt_regs saved ]
...
[ unwind stack ]
When the unwinder goes through the NMI and the IRQ on the stack, and then sees fsnotify(), it doesn't have access to the value of R10, because it only has the five IRET registers. So the unwind stops prematurely.
However, because the interrupt_entry() code is careful not to clobber R10 before saving the full regs, the unwinder should be able to read R10 from the previously saved full pt_regs associated with the NMI.
Handle this case properly. When encountering an IRET regs frame immediately after a full pt_regs frame, use the pt_regs as a backup which can be used to get the C register values.
Also, note that a call frame resets the 'prev_regs' value, because a function is free to clobber the registers. For this fix to work, the IRET and full regs frames must be adjacent, with no FUNC frames in between. So replace the FUNC hint in interrupt_entry() with an IRET_REGS hint.
--- arch/x86/entry/entry_64.S | 4 +-- arch/x86/include/asm/unwind.h | 2 +- arch/x86/kernel/unwind_orc.c | 51 +++++++++++++++++++++++++++-------- 3 files changed, 43 insertions(+), 14 deletions(-)
arch/x86/entry/entry_64.S +2 -2
@@ -511,7 +511,7 @@ SYM_CODE_END(spurious_entries_start) * +----------------------------------------------------+ */ SYM_CODE_START(interrupt_entry) - UNWIND_HINT_FUNC + UNWIND_HINT_IRET_REGS offset=16 ASM_CLAC cld
@@ -543,9 +543,9 @@ SYM_CODE_START(interrupt_entry) pushq 5*8(%rdi) /* regs->eflags */ pushq 4*8(%rdi) /* regs->cs */ pushq 3*8(%rdi) /* regs->ip */ + UNWIND_HINT_IRET_REGS pushq 2*8(%rdi) /* regs->orig_ax */ pushq 8(%rdi) /* return address */ - UNWIND_HINT_FUNC movq (%rdi), %rdi jmp 2f
arch/x86/include/asm/unwind.h +1 -1
@@ -19,7 +19,7 @@ struct unwind_state {#if defined(CONFIG_UNWINDER_ORC )bool signal, full_regs;unsigned long sp, bp, ip; -struct pt_regs *regs; +struct pt_regs *regs, *prev_regs;#elif defined(CONFIG_UNWINDER_FRAME_POINTER )bool got_irq;unsigned long *bp, *orig_sp, ip;
arch/x86/kernel/unwind_orc.c +40 -11
114 lines not on this page · raw message
Signature · 1 line
2.21.1
cover x86/unwind/orc: ORC fixes
Several ORC unwinder cleanups, fixes, and debug improvements.
Jann Horn (1):
x86/entry/64: Fix unwind hints in rewind_stack_do_exit()
Josh Poimboeuf (12):
x86/dumpstack: Add SHOW_REGS_IRET mode
objtool: Fix stack offset tracking for indirect CFAs x86/entry/64: Fix unwind hints in register clearing code x86/entry/64: Fix unwind hints in kernel exit path x86/entry/64: Fix unwind hints in __switch_to_asm() x86/unwind/orc: Convert global variables to static x86/unwind: Prevent false warnings for non-current tasks x86/unwind/orc: Prevent unwinding before ORC initialization x86/unwind/orc: Fix error path for bad ORC entry type x86/unwind/orc: Fix premature unwind stoppage due to IRET frames x86/unwind/orc: Add more unwinder warnings x86/unwind/orc: Add 'unwind_debug' cmdline option
Miroslav Benes (1):
x86/unwind/orc: Don't skip the first frame for inactive tasks
.../admin-guide/kernel-parameters.txt | 6 + arch/x86/entry/calling.h | 40 ++-- arch/x86/entry/entry_64.S | 14 +- arch/x86/include/asm/kdebug.h | 1 + arch/x86/include/asm/unwind.h | 2 +- arch/x86/kernel/dumpstack.c | 27 +-- arch/x86/kernel/dumpstack_64.c | 3 +- arch/x86/kernel/process_64.c | 7 +- arch/x86/kernel/unwind_frame.c | 3 + arch/x86/kernel/unwind_orc.c | 185 ++++++++++++++---- tools/objtool/check.c | 2 +- 11 files changed, 201 insertions(+), 89 deletions(-)
Signature · 1 line
2.21.1
Re: [PATCH 13/14] x86/unwind/orc: Add more unwinder warnings
In reply to Josh PoimboeufQuoted 2 lines · Josh Poimboeuf
Make sure warnings are displayed for all error scenarios (except when encountering an empty unwind hint).
[...]
arch/x86/kernel/unwind_orc.c bool unwind_next_frame(struct unwind_state *state) · line 460
Quoted 11 lines
/* End-of-stack check for kernel threads: */ if (orc->sp_reg == ORC_REG_UNDEFINED) {arch/x86/kernel/unwind_orc.c +7
-1-if (!orc->end) +if (!orc->end) { +/* +* This is reported as an error for the caller, but +* otherwise it isn't worth warning about. In theory +* it can only happen when hitting UNWIND_HINT_EMPTY in +* entry code, close to a kernel exit point. +*/ goto err;
But UNWIND_HINT_EMPTY sets end=1, right? And this is the branch for end==0. What am I missing?
Quoted 19 lines · Josh Poimboeuf
Several ORC unwinder cleanups, fixes, and debug improvements.
Jann Horn (1):
x86/entry/64: Fix unwind hints inrewind_stack_do_exit()Josh Poimboeuf (12):
x86/dumpstack: AddSHOW_REGS_IRETmode
objtool: Fix stack offset tracking for indirect CFAs x86/entry/64: Fix unwind hints in register clearing code x86/entry/64: Fix unwind hints in kernel exit path x86/entry/64: Fix unwind hints in__switch_to_asm()x86/unwind/orc: Convert global variables to static x86/unwind: Prevent false warnings for non-current tasks x86/unwind/orc: Prevent unwinding before ORC initialization x86/unwind/orc: Fix error path for bad ORC entry type x86/unwind/orc: Fix premature unwind stoppage due to IRET frames x86/unwind/orc: Add more unwinder warnings x86/unwind/orc: Add 'unwind_debug' cmdline option
Quoted 2 lines
Miroslav Benes (1):
x86/unwind/orc: Don't skip the first frame for inactive tasks
Looks like good stuff..
Re: [PATCH 13/14] x86/unwind/orc: Add more unwinder warnings
In reply to Jann HornQuoted 16 lines · Jann Horn
Quoted 2 lines · Josh Poimboeuf
Make sure warnings are displayed for all error scenarios (except when encountering an empty unwind hint).
[...]
arch/x86/kernel/unwind_orc.c bool unwind_next_frame(struct unwind_state *state) · line 460
Quoted 11 lines
/* End-of-stack check for kernel threads: */ if (orc->sp_reg == ORC_REG_UNDEFINED) {arch/x86/kernel/unwind_orc.c +7
-1-if (!orc->end) +if (!orc->end) { +/* +* This is reported as an error for the caller, but +* otherwise it isn't worth warning about. In theory +* it can only happen when hitting UNWIND_HINT_EMPTY in +* entry code, close to a kernel exit point. +*/ goto err;
Quoted 2 lines
But
UNWIND_HINT_EMPTYsets end=1, right? And this is the branch for end==0. What am I missing?
You're right. I need to revisit that comment...
Signature · 1 line
Josh
Re: [PATCH 03/14] x86/entry/64: Fix unwind hints in register clearing code
In reply to Josh PoimboeufQuoted 12 lines · Josh Poimboeuf
The
PUSH_AND_CLEAR_REGSmacro zeroes each register immediately after pushing it. If an NMI or exception hits after a register is cleared, but before theUNWIND_HINT_REGSannotation, the ORC unwinder will wrongly think the previous value of the register was zero. This can confuse the unwinding process and cause it to exit early.
Quoted 5 lines
Because ORC is simpler than DWARF, there are a limited number of unwind annotation states, so it's not possible to add an individual unwind hint after each push/clear combination. Instead, the register clearing instructions need to be consolidated and moved to after the
UNWIND_HINT_REGSannotation.
I don’t suppose you know how bad t he performance hit is on a non-PTI machine?
Quoted 84 lines
Fixes 3f01daecd545 ("x86/entry/64: Introduce the PUSH_AND_CLEAN_REGS macro")Signed-off-by Josh Poimboeuf <[email protected]>---
arch/x86/entry/calling.h| 40 +++++++++++++++++++++-------------------1 file changed, 21 insertions(+), 19 deletions(-)arch/x86/entry/calling.h +0
-0@@ -98,13 +98,6 @@ For 32-bit we have the following conventions - kernel is built with#define SIZEOF_PTREGS 21*8.macro
PUSH_AND_CLEAR_REGSrdx=%rdx rax=%rax save_ret=0arch/x86/entry/calling.h +21
-19-/* -* Push registers and sanitize registers of values that a -* speculation attack might otherwise want to exploit. The -* lower registers are likely clobbered well before they -* could be put to use in a speculative execution gadget. -* Interleave XOR with PUSH for better uop scheduling: -*/ .if \save_ret pushq %rsi /* pt_regs->si */ movq 8(%rsp), %rsi /* temporarily store the return address in %rsi */@@ -114,34 +107,43 @@ For 32-bit we have the following conventions - kernel is built with pushq %rsi/* pt_regs->si */ .endif pushq \rdx /* pt_regs->dx */ - xorl %edx, %edx /* nospec dx */ pushq %rcx /* pt_regs->cx */ - xorl %ecx, %ecx /* nospec cx */ pushq \rax /* pt_regs->ax */ pushq %r8 /* pt_regs->r8 */ - xorl %r8d, %r8d /* nospec r8 */ pushq %r9 /* pt_regs->r9 */ - xorl %r9d, %r9d /* nospec r9 */ pushq %r10 /* pt_regs->r10 */ - xorl %r10d, %r10d /* nospec r10 */ pushq %r11 /* pt_regs->r11 */ - xorl %r11d, %r11d /* nospec r11*/ pushq %rbx /* pt_regs->rbx */ - xorl %ebx, %ebx /* nospec rbx*/ pushq %rbp /* pt_regs->rbp */ - xorl %ebp, %ebp /* nospec rbp*/ pushq %r12 /* pt_regs->r12 */ - xorl %r12d, %r12d /* nospec r12*/ pushq %r13 /* pt_regs->r13 */ - xorl %r13d, %r13d /* nospec r13*/ pushq %r14 /* pt_regs->r14 */ - xorl %r14d, %r14d /* nospec r14*/ pushq %r15 /* pt_regs->r15 */ - xorl %r15d, %r15d /* nospec r15*/UNWIND_HINT_REGS + .if \save_ret pushq %rsi /* return address on top of stack */ .endif + +/* +* Sanitize registers of values that a speculation attack might +* otherwise want to exploit. The lower registers are likely clobbered +* well before they could be put to use in a speculative execution +* gadget. +*/ + xorl %edx, %edx /* nospec dx */ + xorl %ecx, %ecx /* nospec cx */ + xorl %r8d, %r8d /* nospec r8 */ + xorl %r9d, %r9d /* nospec r9 */ + xorl %r10d, %r10d /* nospec r10 */ + xorl %r11d, %r11d /* nospec r11 */ + xorl %ebx, %ebx /* nospec rbx */ + xorl %ebp, %ebp /* nospec rbp */ + xorl %r12d, %r12d /* nospec r12 */ + xorl %r13d, %r13d /* nospec r13 */ + xorl %r14d, %r14d /* nospec r14 */ + xorl %r15d, %r15d /* nospec r15 */ +.endm
.macro
POP_REGSpop_rdi=1 skip_r11rcx=0Signature · 1 line
2.21.1
Re: [PATCH 03/14] x86/entry/64: Fix unwind hints in register clearing code
Quoted 8 lines · Andy Lutomirski
Quoted 8 lines · Josh Poimboeuf
The
PUSH_AND_CLEAR_REGSmacro zeroes each register immediately after pushing it. If an NMI or exception hits after a register is cleared, but before theUNWIND_HINT_REGSannotation, the ORC unwinder will wrongly think the previous value of the register was zero. This can confuse the unwinding process and cause it to exit early.
Quoted 7 lines
Quoted 5 lines
Because ORC is simpler than DWARF, there are a limited number of unwind annotation states, so it's not possible to add an individual unwind hint after each push/clear combination. Instead, the register clearing instructions need to be consolidated and moved to after the
UNWIND_HINT_REGSannotation.I don’t suppose you know how bad t he performance hit is on a non-PTI machine?
Hm, what does it have to do with PTI? Should I run a syscall microbenchmark?
Signature · 1 line
Josh
Peter Zijlstra
maintainerRe: [PATCH 03/14] x86/entry/64: Fix unwind hints in register clearing code
Quoted 15 lines · Josh Poimboeuf
Quoted 14 lines · Andy Lutomirski
Quoted 13 lines · Josh Poimboeuf
The
PUSH_AND_CLEAR_REGSmacro zeroes each register immediately after pushing it. If an NMI or exception hits after a register is cleared, but before theUNWIND_HINT_REGSannotation, the ORC unwinder will wrongly think the previous value of the register was zero. This can confuse the unwinding process and cause it to exit early.Because ORC is simpler than DWARF, there are a limited number of unwind annotation states, so it's not possible to add an individual unwind hint after each push/clear combination. Instead, the register clearing instructions need to be consolidated and moved to after the
UNWIND_HINT_REGSannotation.
Quoted 4 lines
Quoted 1 line
I don’t suppose you know how bad t he performance hit is on a non-PTI machine?
Hm, what does it have to do with PTI? Should I run a syscall microbenchmark?
Mostly that performance with PTI on is abysmal so we don't care about a few cycles.
Re: [PATCH 03/14] x86/entry/64: Fix unwind hints in register clearing code
In reply to Andy LutomirskiQuoted 14 lines · Andy Lutomirski
Quoted 7 lines · Josh Poimboeuf
The
PUSH_AND_CLEAR_REGSmacro zeroes each register immediately after pushing it. If an NMI or exception hits after a register is cleared, but before theUNWIND_HINT_REGSannotation, the ORC unwinder will wrongly think the previous value of the register was zero. This can confuse the unwinding process and cause it to exit early.Quoted 5 lines
Because ORC is simpler than DWARF, there are a limited number of unwind annotation states, so it's not possible to add an individual unwind hint after each push/clear combination. Instead, the register clearing instructions need to be consolidated and moved to after the
UNWIND_HINT_REGSannotation.I don’t suppose you know how bad t he performance hit is on a non-PTI machine?
[...]
Quoted 4 lines
Quoted 4 lines
arch/x86/entry/calling.h +4
-0+ xorl %edx, %edx /* nospec dx */ + xorl %ecx, %ecx /* nospec cx */ + xorl %r8d, %r8d /* nospec r8 */ + xorl %r9d, %r9d /* nospec r9 */
Quoted 8 lines
arch/x86/entry/calling.h For 32-bit we have the following conventions - kernel is built with · line 145
Quoted 8 lines
arch/x86/entry/calling.h +8
-0+ xorl %r10d, %r10d /* nospec r10 */ + xorl %r11d, %r11d /* nospec r11 */ + xorl %ebx, %ebx /* nospec rbx */ + xorl %ebp, %ebp /* nospec rbp */ + xorl %r12d, %r12d /* nospec r12 */ + xorl %r13d, %r13d /* nospec r13 */ + xorl %r14d, %r14d /* nospec r14 */ + xorl %r15d, %r15d /* nospec r15 */
I'm curious what the reason for a performance impact would be. Are you worried about the performance of the instruction decoder or the renamer? As long as those aren't slowed down, this will just end up giving the store uops to the backend earlier, right? (Since the XOR instructions shouldn't reach the backend at all AKAIU.)
Re: [PATCH 01/14] x86/dumpstack: Add SHOW_REGS_IRET mode
In reply to Josh PoimboeufQuoted 29 lines · Josh Poimboeuf
Now that
__show_regs()has the concept of "modes" to indicate which registers should be displayed, replaceshow_iret_regs()with a newSHOW_REGS_IRETmode. This is only a cleanup and doesn't change any behavior.Signed-off-by Josh Poimboeuf <[email protected]>--- arch/x86/include/asm/kdebug.h | 1 + arch/x86/kernel/dumpstack.c | 27 ++++++++++----------------- arch/x86/kernel/process_64.c | 7 ++++++- 3 files changed, 17 insertions(+), 18 deletions(-)arch/x86/include/asm/kdebug.h +1
-0@@ -23,6 +23,7 @@ enum die_val { };enum show_regs_mode { +SHOW_REGS_IRET ,SHOW_REGS_SHORT ,/* * For when userspace crashed, but we don't think it's our fault, and arch/x86/kernel/dumpstack.c +0
-0
arch/x86/kernel/dumpstack.c void show_ip(struct pt_regs *regs, const char *loglvl) · line 135
Quoted 10 lines
arch/x86/kernel/dumpstack.c +0
-7@@ -126,15 +126,8 @@ void show_ip(struct pt_regs *regs, const char *loglvl)show_opcodes (regs, loglvl); } -void show_iret_regs (struct pt_regs *regs) -{ -show_ip (regs,KERN_DEFAULT ); -printk (KERN_DEFAULT "RSP: %04x:%016lx EFLAGS: %08lx" , (int )regs->ss, - regs->sp, regs->flags); -} -
There is also declaration of show_iret_regs() in arch/x86/include/asm/kdebug.h which can be removed now.
Miroslav
Re: [PATCH 10/14] x86/unwind/orc: Prevent unwinding before ORC initialization
In reply to Josh PoimboeufQuoted 5 lines · Josh Poimboeuf
If the unwinder is called before the ORC data has been initialized,
orc_find()returns NULL, and it tries to fall back to using frame pointers. This can cause some unexpected warnings during boot.Move the '
orc_init' check fromorc_find()to__unwind_init(), so that it
s/__unwind_init()/__unwind_start()/
arch/x86/kernel/unwind_orc.c EXPORT_SYMBOL_GPL(unwind_next_frame); · line 595
Quoted 8 lines
void __unwind_start(struct
unwind_state*state, structtask_struct*task,arch/x86/kernel/unwind_orc.c +3
-0struct pt_regs *regs, unsigned long *first_frame) { +if (!orc_init) + goto done; +memset (state,0 ,sizeof (*state)); state->task = task;
Miroslav
Quoted 22 lines · Josh Poimboeuf
Several ORC unwinder cleanups, fixes, and debug improvements.
Jann Horn (1):
x86/entry/64: Fix unwind hints inrewind_stack_do_exit()Josh Poimboeuf (12):
x86/dumpstack: AddSHOW_REGS_IRETmode
objtool: Fix stack offset tracking for indirect CFAs x86/entry/64: Fix unwind hints in register clearing code x86/entry/64: Fix unwind hints in kernel exit path x86/entry/64: Fix unwind hints in__switch_to_asm()x86/unwind/orc: Convert global variables to static x86/unwind: Prevent false warnings for non-current tasks x86/unwind/orc: Prevent unwinding before ORC initialization x86/unwind/orc: Fix error path for bad ORC entry type x86/unwind/orc: Fix premature unwind stoppage due to IRET frames x86/unwind/orc: Add more unwinder warnings x86/unwind/orc: Add 'unwind_debug' cmdline optionMiroslav Benes (1):
x86/unwind/orc: Don't skip the first frame for inactive tasks
Quoted 12 lines
.../admin-guide/kernel-parameters.txt | 6 + arch/x86/entry/calling.h | 40 ++-- arch/x86/entry/entry_64.S | 14 +- arch/x86/include/asm/kdebug.h | 1 + arch/x86/include/asm/unwind.h | 2 +- arch/x86/kernel/dumpstack.c | 27 +-- arch/x86/kernel/dumpstack_64.c | 3 +- arch/x86/kernel/process_64.c | 7 +- arch/x86/kernel/unwind_frame.c | 3 + arch/x86/kernel/unwind_orc.c | 185 ++++++++++++++---- tools/objtool/check.c | 2 +- 11 files changed, 201 insertions(+), 89 deletions(-)
Apart from the two nits I mentioned and Jann's comment on comment, it looks good to me.
M