A patch to arch/i386/kernel/irq.c to help get backtraces of (near) stack overflow situations
I'm working on a system that is using a bit too much stack to work reliably with 4K stacks (OpenSSI with DRBD for the root filesystem replication) and I find that the CONFIG_DEBUG_STACKOVERFLOW stuff works rather poorly - often the stack dump will cause a stack overflow itself.
Here's a little patch to irq.c to make stack overflow dumps use a separate stack.
It's based on 2.6.12 'cos that's how out of date I am. :-)
Maybe it'll be of use to someone.
Please CC [email protected] with any comments, I'm not subscribed to the list.
arch/i386/kernel/irq.c +76 -0
@@ -41,6 +41,23 @@static union irq_ctx *hardirq_ctx[NR_CPUS ];static union irq_ctx *softirq_ctx[NR_CPUS ]; +#ifdef CONFIG_DEBUG_STACKOVERFLOW +static union irq_ctx *overflow_ctx[NR_CPUS ]; +#endif +#ifdef CONFIG_KDB +const char *kdba_irq_ctx_type (int cpu,struct thread_info *tinfo) +{ +if (tinfo == &hardirq_ctx[cpu]->tinfo) +return "hardirq_ctx" ; +if (tinfo == &softirq_ctx[cpu]->tinfo) +return "softirq_ctx" ; +#ifdef CONFIG_DEBUG_STACKOVERFLOW +if (tinfo == &overflow_ctx[cpu]->tinfo) +return "overflow_ctx" ; +#endif +return NULL ; +} +#endif /* CONFIG_KDB */ #endif /*
@@ -53,6 +70,7 @@/* high bits used in ret_from_ code */ int irq = regs->orig_eax &0xff ;#ifdef CONFIG_4KSTACKS +int arg1, arg2, ebx;union irq_ctx *curctx, *irqctx;u32 *isp;#endif
@@ -66,9 +84,38 @@ __asm__ __volatile__("andl %%esp,%0" :"=r" (esp) :"0" (THREAD_SIZE -1 ));if (unlikely (esp < (sizeof (struct thread_info ) +STACK_WARN ))) { +#if CONFIG_4KSTACKS +/* use a special stack for overflow warning, +we want to avoid overflowing while telling +the user about the problem! */ + + curctx = (union irq_ctx *)current_thread_info (); + irqctx = overflow_ctx[smp_processor_id ()]; + +if (curctx == irqctx) { +panic ("double overflow" ); + } + +/* build the stack frame on the oflow stack */ + isp = (u32 *) ((char *)irqctx +sizeof (*irqctx)); + irqctx->tinfo.task = curctx->tinfo.task; + irqctx->tinfo.previous_esp = current_stack_pointer; + + asmvolatile ( +" xchgl %%ebx,%%esp \n" +" call __overflow \n" +" movl %%ebx,%%esp \n" + :"=a" (arg1),"=d" (arg2),"=b" (ebx) + :"0" (irq),"1" (esp),"2" (isp) + :"memory" ,"cc" ,"ecx" + ); + +printk ("Back from overflow\n" ); +#else printk ("do_IRQ: stack overflow: %ld\n" , esp -sizeof (struct thread_info ));dump_stack (); +#endif } }#endif
@@ -109,6 +156,17 @@return 1 ; } +#ifdef CONFIG_DEBUG_STACKOVERFLOW +#ifdef CONFIG_4KSTACKS +static fastcall void __overflow (int irq, long esp) +{ +printk ("do_IRQ: stack overflow: %ld\n" , + esp -sizeof (struct thread_info )); +dump_stack (); +} +#endif +#endif +#ifdef CONFIG_4KSTACKS /*
@@ -121,6 +179,10 @@ static char hardirq_stack[NR_CPUS * THREAD_SIZE] __attribute__((__aligned__(THREAD_SIZE))); +#ifdef CONFIG_DEBUG_STACKOVERFLOW +static char overflow_stack[NR_CPUS * THREAD_SIZE] + __attribute__((__aligned__(THREAD_SIZE))); +#endif /* * allocate per-cpu stacks for hardirq and for softirq processing */
@@ -151,6 +213,20 @@printk ("CPU %u irqstacks, hard=%p soft=%p\n" , cpu,hardirq_ctx[cpu],softirq_ctx[cpu]); +#ifdef CONFIG_DEBUG_STACKOVERFLOW + + irqctx = (union irq_ctx *) &overflow_stack[cpu*THREAD_SIZE ]; + irqctx->tinfo.task =NULL ; + irqctx->tinfo.exec_domain =NULL ; + irqctx->tinfo.cpu = cpu; + irqctx->tinfo.preempt_count =HARDIRQ_OFFSET ;/* ? */ + irqctx->tinfo.addr_limit =MAKE_MM_SEG (0 ); + + overflow_ctx[cpu] = irqctx; + +printk ("CPU %u irqstacks, overflow=%p\n" , + cpu,overflow_ctx[cpu]); +#endif } extern asmlinkage void __do_softirq(void);
Quoted 9 lines · John Hughes
I'm working on a system that is using a bit too much stack to work reliably with 4K stacks (OpenSSI with DRBD for the root filesystem replication) and I find that the
CONFIG_DEBUG_STACKOVERFLOWstuff works rather poorly - often the stack dump will cause a stack overflow itself.Here's a little patch to
irq.cto make stack overflow dumps use a separate stack.It's based on 2.6.12 'cos that's how out of date I am. :-)
in recent kernel's there's an automatic maximum-stack-footprint tracer plugin available. When you suspect stack overflow you can use it by enabling:
CONFIG_STACK_TRACER=y
and boot the kernel with the 'stacktrace' boot option. Then you'll get such "worst-case stack footprint since bootup" entries in /debug/tracing/stack_trace:
# ls -l /debug/tracing/*stack* -rw-r--r-- 1 root root 0 2009-01-03 22:42 /debug/tracing/stack_max_size -r--r--r-- 1 root root 0 2009-01-03 22:42 /debug/tracing/stack_trace
# cat /debug/tracing/*stack*
5624
Depth Size Location (38 entries)
----- ---- --------
0) 5496 232 lookup_address+0x51/0x20e
1) 5264 496 __change_page_attr_set_clr+0xa7/0xa15
2) 4768 128 kernel_map_pages+0x121/0x140
3) 4640 224 get_page_from_freelist+0x4bb/0x6ec
4) 4416 160 __alloc_pages_internal+0x150/0x4af
5) 4256 48 alloc_pages_current+0xbe/0xc7
6) 4208 16 alloc_slab_page+0x1e/0x6e
7) 4192 64 new_slab+0x51/0x1e2
8) 4128 96 __slab_alloc+0x260/0x3fa
9) 4032 80 kmem_cache_alloc+0xa8/0x120
10) 3952 48 radix_tree_preload+0x38/0x86
11) 3904 64 add_to_page_cache_locked+0x51/0xed
12) 3840 32 add_to_page_cache_lru+0x31/0x6b
13) 3808 64 find_or_create_page+0x56/0x7d
14) 3744 80 __getblk+0x136/0x262
15) 3664 32 __bread+0x13/0x82
16) 3632 64 ext3_get_branch+0x7b/0xef
17) 3568 368 ext3_get_blocks_handle+0xa2/0x907
18) 3200 96 ext3_get_block+0xc3/0x101
19) 3104 224 do_mpage_readpage+0x1ad/0x4d0
20) 2880 240 mpage_readpages+0xb6/0xf9
21) 2640 16 ext3_readpages+0x1f/0x21
22) 2624 144 __do_page_cache_readahead+0x147/0x1bd
23) 2480 48 do_page_cache_readahead+0x5b/0x68
24) 2432 112 filemap_fault+0x176/0x34b
25) 2320 160 __do_fault+0x58/0x410
26) 2160 176 handle_mm_fault+0x4b2/0xa3e
27) 1984 800 do_page_fault+0x86d/0xcec
28) 1184 208 page_fault+0x25/0x30
29) 976 16 clear_user+0x30/0x38
30) 960 288 load_elf_binary+0x61c/0x18f0
31) 672 80 search_binary_handler+0xd9/0x279
32) 592 176 load_script+0x1b3/0x1c8
33) 416 80 search_binary_handler+0xd9/0x279
34) 336 96 do_execve+0x1df/0x296
35) 240 64 sys_execve+0x43/0x5e
36) 176 176 stub_execve+0x6a/0xc0It uses precise stack frames and lists their sizes and the total depth of each stack entry.
Hope this helps,
Ingo
Quoted 1 line · Ingo Molnar
in recent kernel's there's an automatic maximum-stack-footprint tracer plugin available.
Lovely. Maybe one day I'll be able to use a recent kernel. :-)
Thanks for the tip.
Quoted 5 lines · John Hughes
Quoted 2 lines · Ingo Molnar
in recent kernel's there's an automatic maximum-stack-footprint tracer plugin available.
Lovely. Maybe one day I'll be able to use a recent kernel. :-)
Curious, is that inability of using recent kernls our fault or your fault? :)
Ingo
Quoted 5 lines · Ingo Molnar
Quoted 2 lines · John Hughes
Ingo Molnar wrote:
Quoted 8 lines
Quoted 4 lines
Quoted 2 lines
in recent kernel's there's an automatic maximum-stack-footprint tracer plugin available.
Lovely. Maybe one day I'll be able to use a recent kernel. :-)
Curious, is that inability of using recent kernls our fault or your fault? :)
Oh, mine, all mine. Just can't find the time. :-)
(Well, I could blame you buggers for moving too fast, but I doubt that would be appreciated :-)).
Quoted 12 lines · John Hughes
Quoted 11 lines · Ingo Molnar
Quoted 7 lines · John Hughes
Quoted 3 lines · Ingo Molnar
in recent kernel's there's an automatic maximum-stack-footprint tracer plugin available.
Lovely. Maybe one day I'll be able to use a recent kernel. :-)
Quoted 7 lines
Quoted 3 lines
Curious, is that inability of using recent kernls our fault or your fault? :)
Oh, mine, all mine. Just can't find the time. :-)
(Well, I could blame you buggers for moving too fast, but I doubt that would be appreciated :-)).
oh, feel free, we'll take it as sincere flattery ;-)
Ingo
Quoted 2 lines · John Hughes
Here's a little patch to
irq.cto make stack overflow dumps use a separate stack.
This has been fixed last year in mainline with04b361abfdc522239e3a071f3afdebf5787d9f03
Morale: If you use ancient kernels you miss a lot of bugfixes.
-Andi
Signature · 1 line
[email protected] -- Speaking for myself only.
Quoted 9 lines · Andi Kleen
Quoted 3 lines · John Hughes
Here's a little patch to
irq.cto make stack overflow dumps use a separate stack.This has been fixed last year in mainline with
04b361abfdc522239e3a071f3afdebf5787d9f03
Ah, thanks, neater than my fix.
Quoted 2 lines
Morale: If you use ancient kernels you miss a lot of bugfixes.
Hey, don't you be ruining my morale, it's low enough already. :-)