debug lockups: Improve lockup detection
Subsystems (3)
Commit-ID: c1dc0b9c0c8979ce4d411caadff5c0d79dee58bc Gitweb: http://git.kernel.org/tip/c1dc0b9c0c8979ce4d411caadff5c0d79dee58bc Author: Ingo Molnar <[email protected]> AuthorDate: Sun, 2 Aug 2009 11:28:21 +0200 Committer: Ingo Molnar <[email protected]> CommitDate: Sun, 2 Aug 2009 13:27:17 +0200
debug lockups: Improve lockup detection
When debugging a recent lockup bug i found various deficiencies in how our current lockup detection helpers work:
- SysRq-L is not very efficient as it uses a workqueue, hence it cannot punch through hard lockups and cannot see through most soft lockups either.
- The SysRq-L code depends on the NMI watchdog - which is off by default.
- We dont print backtraces from the RCU code's built-in 'RCU state machine is stuck' debug code. This debug code tends to be one of the first (and only) mechanisms that show that a lockup has occured.
This patch changes the code so taht we:
- Trigger the NMI backtrace code from SysRq-L instead of using a workqueue (which cannot punch through hard lockups)
- Trigger print-all-CPU-backtraces from the RCU lockup detection code
Also decouple the backtrace printing code from the NMI watchdog:
- Dont use variable size cpumasks (it might not be initialized and they are a bit more fragile anyway)
- Trigger an NMI immediately via an IPI, instead of waiting for the NMI tick to occur. This is a lot faster and can produce more relevant backtraces. It will also work if the NMI watchdog is disabled.
- Dont print the 'dazed and confused' message when we print a backtrace from the NMI
- Do a show_regs() plus a dump_stack() to get maximum info out of the dump. Worst-case we get two stacktraces - which is not a big deal. Sometimes, if register content is corrupted, the precise stack walker in show_regs() wont give us a full backtrace - in this case dump_stack() will do it.
Cc: Paul E. McKenney <[email protected]>
Cc: Peter Zijlstra <[email protected]>
Cc: Andrew Morton <[email protected]>
Cc: Linus Torvalds <[email protected]>
LKML-Reference: <new-submission>
--- arch/x86/kernel/apic/nmi.c | 18 ++++++++++++------ drivers/char/sysrq.c | 8 ++------ kernel/rcutree.c | 7 ++++++- 3 files changed, 20 insertions(+), 13 deletions(-)
arch/x86/kernel/apic/nmi.c +12 -6
@@ -39,7 +39,7 @@int unknown_nmi_panic;int nmi_watchdog_enabled; -static cpumask_var_t backtrace_mask; +static cpumask_t backtrace_mask__read_mostly ;/* nmi_active: * >0: the lapic NMI watchdog is active, but can be disabled
@@ -138,7 +138,6 @@ int __init check_nmi_watchdog(void)if (!prev_nmi_count) goto error; -alloc_cpumask_var (&backtrace_mask,GFP_KERNEL |__GFP_ZERO);printk (KERN_INFO "Testing NMI watchdog ... " );#ifdef CONFIG_SMP
@@ -415,14 +414,17 @@ nmi_watchdog_tick(struct pt_regs *regs, unsigned reason) }/* We can be called before check_nmi_watchdog, hence NULL check. */ -if (backtrace_mask !=NULL &&cpumask_test_cpu (cpu, backtrace_mask)) { +if (cpumask_test_cpu (cpu, &backtrace_mask)) {static DEFINE_SPINLOCK (lock);/* Serialise the printks */ spin_lock (&lock);printk (KERN_WARNING "NMI backtrace for cpu %d\n" , cpu); +show_regs (regs);dump_stack ();spin_unlock (&lock); -cpumask_clear_cpu (cpu, backtrace_mask); +cpumask_clear_cpu (cpu, &backtrace_mask); + + rc =1 ; }/* Could check oops_in_progress here too, but it's safer not to */
@@ -556,10 +558,14 @@ void __trigger_all_cpu_backtrace(void) {int i; -cpumask_copy (backtrace_mask, cpu_online_mask); +cpumask_copy (&backtrace_mask, cpu_online_mask); + +printk (KERN_INFO "sending NMI to all CPUs:\n" ); + apic->send_IPI_all (NMI_VECTOR ); +/* Wait for up to 10 seconds for all CPUs to do the backtrace */ for (i =0 ; i <10 *1000 ; i++) { -if (cpumask_empty (backtrace_mask)) +if (cpumask_empty (&backtrace_mask))break ;mdelay (1 ); }
drivers/char/sysrq.c +2 -6
@@ -24,6 +24,7 @@#include <linux/sysrq.h> #include <linux/kbd_kern.h> #include <linux/proc_fs.h> +#include <linux/nmi.h> #include <linux/quotaops.h> #include <linux/perf_counter.h> #include <linux/kernel.h>
@@ -222,12 +223,7 @@ static DECLARE_WORK(sysrq_showallcpus, sysrq_showregs_othercpus);static void sysrq_handle_showallcpus (int key,struct tty_struct *tty) { -struct pt_regs *regs =get_irq_regs (); -if (regs) { -printk (KERN_INFO "CPU%d:\n" ,smp_processor_id ()); -show_regs (regs); - } -schedule_work (&sysrq_showallcpus); +trigger_all_cpu_backtrace (); }static struct sysrq_key_op sysrq_showallcpus_op = {
kernel/rcutree.c +6 -1
@@ -35,6 +35,7 @@#include <linux/rcupdate.h> #include <linux/interrupt.h> #include <linux/sched.h> +#include <linux/nmi.h> #include <asm/atomic.h> #include <linux/bitops.h> #include <linux/module.h>
@@ -469,6 +470,8 @@ static void print_other_cpu_stall(struct rcu_state *rsp) }printk (" (detected by %d, t=%ld jiffies)\n" ,smp_processor_id (), (long )(jiffies - rsp->gp_start)); +trigger_all_cpu_backtrace (); +force_quiescent_state (rsp,0 );/* Kick them all. */ }
@@ -479,12 +482,14 @@ static void print_cpu_stall(struct rcu_state *rsp)printk (KERN_ERR "INFO: RCU detected CPU %d stall (t=%lu jiffies)\n" ,smp_processor_id (), jiffies - rsp->gp_start); -dump_stack (); +trigger_all_cpu_backtrace (); +spin_lock_irqsave (&rnp->lock, flags);if ((long )(jiffies - rsp->jiffies_stall) >=0 ) rsp->jiffies_stall = jiffies +RCU_SECONDS_TILL_STALL_RECHECK ;spin_unlock_irqrestore (&rnp->lock, flags); +set_need_resched ();/* kick ourselves to get things going. */ }
Paul E. McKenney
Reviewed-byQuoted 45 lines · tip-bot for Ingo Molnar
Commit-ID: c1dc0b9c0c8979ce4d411caadff5c0d79dee58bc Gitweb: http://git.kernel.org/tip/c1dc0b9c0c8979ce4d411caadff5c0d79dee58bc Author: Ingo Molnar <[email protected]> AuthorDate: Sun, 2 Aug 2009 11:28:21 +0200 Committer: Ingo Molnar <[email protected]> CommitDate: Sun, 2 Aug 2009 13:27:17 +0200debug lockups: Improve lockup detection
When debugging a recent lockup bug i found various deficiencies in how our current lockup detection helpers work:
- SysRq-L is not very efficient as it uses a workqueue, hence it cannot punch through hard lockups and cannot see through most soft lockups either.
- The SysRq-L code depends on the NMI watchdog - which is off by default.
- We dont print backtraces from the RCU code's built-in 'RCU state machine is stuck' debug code. This debug code tends to be one of the first (and only) mechanisms that show that a lockup has occured.
This patch changes the code so taht we:
- Trigger the NMI backtrace code from SysRq-L instead of using a workqueue (which cannot punch through hard lockups)
- Trigger print-all-CPU-backtraces from the RCU lockup detection code
Also decouple the backtrace printing code from the NMI watchdog:
- Dont use variable size cpumasks (it might not be initialized and they are a bit more fragile anyway)
- Trigger an NMI immediately via an IPI, instead of waiting for the NMI tick to occur. This is a lot faster and can produce more relevant backtraces. It will also work if the NMI watchdog is disabled.
- Dont print the 'dazed and confused' message when we print a backtrace from the NMI
Quoted 6 lines
- Do a
show_regs()plus adump_stack()to get maximum info out of the dump. Worst-case we get two stacktraces - which is not a big deal. Sometimes, if register content is corrupted, the precise stack walker inshow_regs()wont give us a full backtrace - in this casedump_stack()will do it.
Looks good from an RCU perspective! I will need to dump task stacks in the new version.
Quoted 135 lines
Cc: Paul E. McKenney <[email protected]>
Cc: Peter Zijlstra <[email protected]>
Cc: Andrew Morton <[email protected]>
Cc: Linus Torvalds <[email protected]>
LKML-Reference: <new-submission>Signed-off-by Ingo Molnar <[email protected]>--- arch/x86/kernel/apic/nmi.c | 18 ++++++++++++------ drivers/char/sysrq.c | 8 ++------ kernel/rcutree.c | 7 ++++++- 3 files changed, 20 insertions(+), 13 deletions(-)arch/x86/kernel/apic/nmi.c +12
-6@@ -39,7 +39,7 @@int unknown_nmi_panic;int nmi_watchdog_enabled; -static cpumask_var_t backtrace_mask; +static cpumask_t backtrace_mask__read_mostly ;/* nmi_active: * >0: the lapic NMI watchdog is active, but can be disabled @@ -138,7 +138,6 @@ int __init check_nmi_watchdog(void)if (!prev_nmi_count) goto error; -alloc_cpumask_var (&backtrace_mask,GFP_KERNEL |__GFP_ZERO);printk (KERN_INFO "Testing NMI watchdog ... " );#ifdef CONFIG_SMP @@ -415,14 +414,17 @@ nmi_watchdog_tick(struct pt_regs *regs, unsigned reason) }/* We can be called before check_nmi_watchdog, hence NULL check. */ -if (backtrace_mask !=NULL &&cpumask_test_cpu (cpu, backtrace_mask)) { +if (cpumask_test_cpu (cpu, &backtrace_mask)) {static DEFINE_SPINLOCK (lock);/* Serialise the printks */ spin_lock (&lock);printk (KERN_WARNING "NMI backtrace for cpu %d\n" , cpu); +show_regs (regs);dump_stack ();spin_unlock (&lock); -cpumask_clear_cpu (cpu, backtrace_mask); +cpumask_clear_cpu (cpu, &backtrace_mask); + + rc =1 ; }/* Could check oops_in_progress here too, but it's safer not to */ @@ -556,10 +558,14 @@ void __trigger_all_cpu_backtrace(void) {int i; -cpumask_copy (backtrace_mask, cpu_online_mask); +cpumask_copy (&backtrace_mask, cpu_online_mask); + +printk (KERN_INFO "sending NMI to all CPUs:\n" ); + apic->send_IPI_all (NMI_VECTOR ); +/* Wait for up to 10 seconds for all CPUs to do the backtrace */ for (i =0 ; i <10 *1000 ; i++) { -if (cpumask_empty (backtrace_mask)) +if (cpumask_empty (&backtrace_mask))break ;mdelay (1 ); }drivers/char/sysrq.c +2
-6@@ -24,6 +24,7 @@#include <linux/sysrq.h> #include <linux/kbd_kern.h> #include <linux/proc_fs.h> +#include <linux/nmi.h> #include <linux/quotaops.h> #include <linux/perf_counter.h> #include <linux/kernel.h> @@ -222,12 +223,7 @@ static DECLARE_WORK(sysrq_showallcpus, sysrq_showregs_othercpus);static void sysrq_handle_showallcpus (int key,struct tty_struct *tty) { -struct pt_regs *regs =get_irq_regs (); -if (regs) { -printk (KERN_INFO "CPU%d:\n" ,smp_processor_id ()); -show_regs (regs); - } -schedule_work (&sysrq_showallcpus); +trigger_all_cpu_backtrace (); }static struct sysrq_key_op sysrq_showallcpus_op = {kernel/rcutree.c +6
-1@@ -35,6 +35,7 @@#include <linux/rcupdate.h> #include <linux/interrupt.h> #include <linux/sched.h> +#include <linux/nmi.h> #include <asm/atomic.h> #include <linux/bitops.h> #include <linux/module.h> @@ -469,6 +470,8 @@ static void print_other_cpu_stall(struct rcu_state *rsp) }printk (" (detected by %d, t=%ld jiffies)\n" ,smp_processor_id (), (long )(jiffies - rsp->gp_start)); +trigger_all_cpu_backtrace (); +force_quiescent_state (rsp,0 );/* Kick them all. */ }@@ -479,12 +482,14 @@ static void print_cpu_stall(struct rcu_state *rsp)printk (KERN_ERR "INFO: RCU detected CPU %d stall (t=%lu jiffies)\n" ,smp_processor_id (), jiffies - rsp->gp_start); -dump_stack (); +trigger_all_cpu_backtrace (); +spin_lock_irqsave (&rnp->lock, flags);if ((long )(jiffies - rsp->jiffies_stall) >=0 ) rsp->jiffies_stall = jiffies +RCU_SECONDS_TILL_STALL_RECHECK ;spin_unlock_irqrestore (&rnp->lock, flags); +set_need_resched ();/* kick ourselves to get things going. */ }
Quoted 19 lines · On Sun, 2 Aug 2009 13:09:34 GMT tip-bot for Ingo Molnar <[email protected]> wrote:
Commit-ID: c1dc0b9c0c8979ce4d411caadff5c0d79dee58bc Gitweb: http://git.kernel.org/tip/c1dc0b9c0c8979ce4d411caadff5c0d79dee58bc Author: Ingo Molnar <[email protected]> AuthorDate: Sun, 2 Aug 2009 11:28:21 +0200 Committer: Ingo Molnar <[email protected]> CommitDate: Sun, 2 Aug 2009 13:27:17 +0200drivers/char/sysrq.c +1
-0@@ -24,6 +24,7 @@#include <linux/sysrq.h> #include <linux/kbd_kern.h> #include <linux/proc_fs.h> +#include <linux/nmi.h> #include <linux/quotaops.h> #include <linux/perf_counter.h> #include <linux/kernel.h> @@ -222,12 +223,7 @@ static DECLARE_WORK(sysrq_showallcpus, sysrq_showregs_othercpus);
drivers/char/sysrq.c static DECLARE_WORK(sysrq_showallcpus, sysrq_showregs_othercpus); · line 227
Quoted 10 lines
drivers/char/sysrq.c +1
-6static void sysrq_handle_showallcpus (int key,struct tty_struct *tty) { -struct pt_regs *regs =get_irq_regs (); -if (regs) { -printk (KERN_INFO "CPU%d:\n" ,smp_processor_id ()); -show_regs (regs); - } -schedule_work (&sysrq_showallcpus); +trigger_all_cpu_backtrace (); }
I think this just broke all non-x86 non-sparc SMP architectures.
Quoted 14 lines
drivers/char/sysrq.c +0
-0static struct sysrq_key_op sysrq_showallcpus_op = {kernel/rcutree.c +1
-0@@ -35,6 +35,7 @@#include <linux/rcupdate.h> #include <linux/interrupt.h> #include <linux/sched.h> +#include <linux/nmi.h> #include <asm/atomic.h> #include <linux/bitops.h> #include <linux/module.h>
kernel/rcutree.c static void print_other_cpu_stall(struct rcu_state *rsp) · line 473
Quoted 4 lines
kernel/rcutree.c +1
-0@@ -469,6 +470,8 @@ static void print_other_cpu_stall(struct rcu_state *rsp) }printk (" (detected by %d, t=%ld jiffies)\n" ,smp_processor_id (), (long )(jiffies - rsp->gp_start)); +trigger_all_cpu_backtrace ();
Be aware that trigger_all_cpu_backtrace() is a PITA when you have a lot of CPUs.
If a callsite is careful to ensure that the most important information is emitted last then that might improve things.
otoh, log buffer overflow will truncate, I think. So that info needs to be emitted first too ;)
It's a PITA.
Quoted 21 lines · Andrew Morton
Quoted 19 lines · On Sun, 2 Aug 2009 13:09:34 GMT tip-bot for Ingo Molnar <[email protected]> wrote:
Commit-ID: c1dc0b9c0c8979ce4d411caadff5c0d79dee58bc Gitweb: http://git.kernel.org/tip/c1dc0b9c0c8979ce4d411caadff5c0d79dee58bc Author: Ingo Molnar <[email protected]> AuthorDate: Sun, 2 Aug 2009 11:28:21 +0200 Committer: Ingo Molnar <[email protected]> CommitDate: Sun, 2 Aug 2009 13:27:17 +0200drivers/char/sysrq.c +1
-0@@ -24,6 +24,7 @@#include <linux/sysrq.h> #include <linux/kbd_kern.h> #include <linux/proc_fs.h> +#include <linux/nmi.h> #include <linux/quotaops.h> #include <linux/perf_counter.h> #include <linux/kernel.h> @@ -222,12 +223,7 @@ static DECLARE_WORK(sysrq_showallcpus, sysrq_showregs_othercpus);
Quoted 12 lines
drivers/char/sysrq.c static DECLARE_WORK(sysrq_showallcpus, sysrq_showregs_othercpus); · line 227
Quoted 10 lines
drivers/char/sysrq.c +1
-6static void sysrq_handle_showallcpus (int key,struct tty_struct *tty) { -struct pt_regs *regs =get_irq_regs (); -if (regs) { -printk (KERN_INFO "CPU%d:\n" ,smp_processor_id ()); -show_regs (regs); - } -schedule_work (&sysrq_showallcpus); +trigger_all_cpu_backtrace (); }I think this just broke all non-x86 non-sparc SMP architectures.
Yeah - it 'broke' them in the sense of them not having a working trigger_all_cpu_backtrace() implementation to begin with. (which breaks/degrades spinlock-debug to begin with so it's an existing problem)
One solution would be to do a generic trigger_all_cpu_backtrace() implementation that does the above schedule_work() approach.
I never understood why we proliferated all these different backtrace-triggering mechanisms instead of doing one good approach that everything uses.
Quoted 22 lines
Quoted 14 lines
drivers/char/sysrq.c +0
-0static struct sysrq_key_op sysrq_showallcpus_op = {kernel/rcutree.c +1
-0@@ -35,6 +35,7 @@#include <linux/rcupdate.h> #include <linux/interrupt.h> #include <linux/sched.h> +#include <linux/nmi.h> #include <asm/atomic.h> #include <linux/bitops.h> #include <linux/module.h> kernel/rcutree.c static void print_other_cpu_stall(struct rcu_state *rsp) · line 473
Quoted 4 lines
kernel/rcutree.c +1
-0@@ -469,6 +470,8 @@ static void print_other_cpu_stall(struct rcu_state *rsp) }printk (" (detected by %d, t=%ld jiffies)\n" ,smp_processor_id (), (long )(jiffies - rsp->gp_start)); +trigger_all_cpu_backtrace ();Be aware that
trigger_all_cpu_backtrace()is a PITA when you have a lot of CPUs.
Quoted 7 lines
If a callsite is careful to ensure that the most important information is emitted last then that might improve things.
otoh, log buffer overflow will truncate, I think. So that info needs to be emitted first too ;)
It's a PITA.
Yeah, it is - i'd expect larger systems to have larger log buffers. Lack of info was obviously a showstopper with the highest priority.
Ingo
Quoted 4 lines · On Sun, 2 Aug 2009 21:26:57 +0200 Ingo Molnar <[email protected]> wrote:
Quoted 1 line
I think this just broke all non-x86 non-sparc SMP architectures.
Yeah - it 'broke' them in the sense of them not having a working
trigger_all_cpu_backtrace()implementation to begin with.
c'mon. It broke them in the sense that sysrq-l went from "works" to "doesn't work".
It would take months for the relevant arch maintainers to even find out about this, after which they're left with dud kernels out in the field.
It's better to break the build or to emit warnings than to silently and secretly break their stuff.
include/linux/nmi.h renamed from include/linux/nmi.h~a +3 -0
@@ -29,6 +29,9 @@ static inline void acpi_nmi_enable(void)#endif #ifndef trigger_all_cpu_backtrace +#ifdef CONFIG_SMP +#warning This architecture is missing a trigger_all_cpu_backtrace() implementation +#endif #define trigger_all_cpu_backtrace () do { } while (0)#endif
_
Quoted 9 lines · Andrew Morton
Quoted 4 lines · On Sun, 2 Aug 2009 21:26:57 +0200 Ingo Molnar <[email protected]> wrote:
Quoted 1 line
I think this just broke all non-x86 non-sparc SMP architectures.
Yeah - it 'broke' them in the sense of them not having a working
trigger_all_cpu_backtrace()implementation to begin with.c'mon. It broke them in the sense that sysrq-l went from "works" to "doesn't work".
You are right (i broke it with my patch) but the thing is, sysrq-l almost useless currently: it uses schedule_work() which assumes a mostly working system with full irqs and scheduling working fine. Now, i dont need sysrq-l on mostly working systems.
So the 'breakage' is of something that was largely useless: and now you put the onus of implementing it for _all_ architectures (which i dont use) on me?
If that's the requirement then i'll have to keep this as a local debug hack and not do an upstream solution - i dont have the resources to do it for all ~10 SMP architectures.
sysrq-l has been messed up really and now that messup limits the adoption of the much more useful solution? I didnt make this thing up, i tried to use it on a locked up system and wondered why it emits nothing and why it uses a separate facility instead of an existing trigger-backtraces facility (which the spinlock-debug code uses).
Quoted 6 lines
It would take months for the relevant arch maintainers to even find out about this, after which they're left with dud kernels out in the field.
It's better to break the build or to emit warnings than to silently and secretly break their stuff.
But that warning will bounce the ball back to me, wont it? My patch will be blamed for 'breaking' those architectures, right?
Ingo
Quoted 13 lines · On Sun, 2 Aug 2009 22:41:50 +0200 Ingo Molnar <[email protected]> wrote:
Quoted 9 lines · Andrew Morton
Quoted 4 lines · On Sun, 2 Aug 2009 21:26:57 +0200 Ingo Molnar <[email protected]> wrote:
Quoted 1 line
I think this just broke all non-x86 non-sparc SMP architectures.
Yeah - it 'broke' them in the sense of them not having a working
trigger_all_cpu_backtrace()implementation to begin with.c'mon. It broke them in the sense that sysrq-l went from "works" to "doesn't work".
Quoted 8 lines
You are right (i broke it with my patch) but the thing is, sysrq-l almost useless currently: it uses
schedule_work()which assumes a mostly working system with full irqs and scheduling working fine. Now, i dont need sysrq-l on mostly working systems.So the 'breakage' is of something that was largely useless: and now you put the onus of implementing it for _all_ architectures (which i dont use) on me?
I never said that.
It's appropriate that those architectures be left with their existing level of functionality/usefulness, as you're already discussing.
Quoted 5 lines
Quoted 2 lines
It's better to break the build or to emit warnings than to silently and secretly break their stuff.
But that warning will bounce the ball back to me, wont it? My patch will be blamed for 'breaking' those architectures, right?
It's a very crude and somewhat rude way of communicating information to other architecture maintainers.
A better way would be to send them an email explaining the problem and outlining some solutions, no?
Quoted 25 lines · Andrew Morton
Quoted 19 lines · On Sun, 2 Aug 2009 22:41:50 +0200 Ingo Molnar <[email protected]> wrote:
Quoted 9 lines · Andrew Morton
Quoted 4 lines · On Sun, 2 Aug 2009 21:26:57 +0200 Ingo Molnar <[email protected]> wrote:
Quoted 1 line
I think this just broke all non-x86 non-sparc SMP architectures.
Yeah - it 'broke' them in the sense of them not having a working
trigger_all_cpu_backtrace()implementation to begin with.c'mon. It broke them in the sense that sysrq-l went from "works" to "doesn't work".
You are right (i broke it with my patch) but the thing is, sysrq-l almost useless currently: it uses
schedule_work()which assumes a mostly working system with full irqs and scheduling working fine. Now, i dont need sysrq-l on mostly working systems.Quoted 3 lines
So the 'breakage' is of something that was largely useless: and now you put the onus of implementing it for _all_ architectures (which i dont use) on me?
Quoted 5 lines
I never said that.
It's appropriate that those architectures be left with their existing level of functionality/usefulness, as you're already discussing.
Ok, agreed.
Quoted 11 lines
Quoted 5 lines
Quoted 2 lines
It's better to break the build or to emit warnings than to silently and secretly break their stuff.
But that warning will bounce the ball back to me, wont it? My patch will be blamed for 'breaking' those architectures, right?
It's a very crude and somewhat rude way of communicating information to other architecture maintainers.
A better way would be to send them an email explaining the problem and outlining some solutions, no?
I've restored the generic fallback code so there should be no change in functionality. I'll test it and push it out - you can check that patch via the commit notification email.
Thanks,
Ingo
[tip:core/debug] debug lockups: Improve lockup detection, fix generic arch fallback
In reply to Andrew MortonCommit-ID: 47cab6a722d44c71c4f8224017ef548522243cf4 Gitweb: http://git.kernel.org/tip/47cab6a722d44c71c4f8224017ef548522243cf4 Author: Ingo Molnar <[email protected]> AuthorDate: Mon, 3 Aug 2009 09:31:54 +0200 Committer: Ingo Molnar <[email protected]> CommitDate: Mon, 3 Aug 2009 09:56:52 +0200
debug lockups: Improve lockup detection, fix generic arch fallback
As Andrew noted, my previous patch ("debug lockups: Improve lockup detection") broke/removed SysRq-L support from architecture that do not provide a __trigger_all_cpu_backtrace implementation.
Restore a fallback path and clean up the SysRq-L machinery a bit:
- Rename the arch method to arch_trigger_all_cpu_backtrace()
- Simplify the define
- Document the method a bit - in the hope of more architectures adding support for it.
[ The patch touches Sparc code for the rename. ]
Cc: Paul E. McKenney <[email protected]>
Cc: Peter Zijlstra <[email protected]>
Cc: Andrew Morton <[email protected]>
Cc: Linus Torvalds <[email protected]>
Cc: "David S. Miller" <[email protected]>
LKML-Reference: <[email protected]>
--- arch/sparc/include/asm/irq_64.h | 4 ++-- arch/sparc/kernel/process_64.c | 4 ++-- arch/x86/include/asm/nmi.h | 4 ++-- arch/x86/kernel/apic/nmi.c | 2 +- drivers/char/sysrq.c | 15 ++++++++++++++- include/linux/nmi.h | 19 +++++++++++++++++-- 6 files changed, 38 insertions(+), 10 deletions(-)
arch/sparc/include/asm/irq_64.h +2 -2
@@ -89,8 +89,8 @@ static inline unsigned long get_softint(void)return retval; } -void __trigger_all_cpu_backtrace (void ); -#define trigger_all_cpu_backtrace () __trigger_all_cpu_backtrace() +void arch_trigger_all_cpu_backtrace (void ); +#define arch_trigger_all_cpu_backtrace arch_trigger_all_cpu_backtraceextern void *hardirq_stack[NR_CPUS ];extern void *softirq_stack[NR_CPUS ];
arch/sparc/kernel/process_64.c +2 -2
@@ -251,7 +251,7 @@ static void __global_reg_poll(struct global_reg_snapshot *gp) } } -void __trigger_all_cpu_backtrace (void ) +void arch_trigger_all_cpu_backtrace (void ) {struct thread_info *tp =current_thread_info ();struct pt_regs *regs =get_irq_regs ();
@@ -304,7 +304,7 @@ void __trigger_all_cpu_backtrace(void)static void sysrq_handle_globreg (int key,struct tty_struct *tty) { -__trigger_all_cpu_backtrace (); +arch_trigger_all_cpu_backtrace (); }static struct sysrq_key_op sparc_globalreg_op = {
arch/x86/include/asm/nmi.h +2 -2
@@ -45,8 +45,8 @@ extern int proc_nmi_enabled(struct ctl_table *, int , struct file *, void __user *, size_t *, loff_t *);extern int unknown_nmi_panic; -void __trigger_all_cpu_backtrace (void ); -#define trigger_all_cpu_backtrace () __trigger_all_cpu_backtrace() +void arch_trigger_all_cpu_backtrace (void ); +#define arch_trigger_all_cpu_backtrace arch_trigger_all_cpu_backtracestatic inline void localise_nmi_watchdog (void ) {
arch/x86/kernel/apic/nmi.c +1 -1
@@ -554,7 +554,7 @@ int do_nmi_callback(struct pt_regs *regs, int cpu)return 0 ; } -void __trigger_all_cpu_backtrace (void ) +void arch_trigger_all_cpu_backtrace (void ) {int i;
drivers/char/sysrq.c +14 -1
@@ -223,7 +223,20 @@ static DECLARE_WORK(sysrq_showallcpus, sysrq_showregs_othercpus);static void sysrq_handle_showallcpus (int key,struct tty_struct *tty) { -trigger_all_cpu_backtrace (); +/* +* Fall back to the workqueue based printing if the +* backtrace printing did not succeed or the +* architecture has no support for it: +*/ +if (!trigger_all_cpu_backtrace ()) { +struct pt_regs *regs =get_irq_regs (); + +if (regs) { +printk (KERN_INFO "CPU%d:\n" ,smp_processor_id ()); +show_regs (regs); + } +schedule_work (&sysrq_showallcpus); + } }static struct sysrq_key_op sysrq_showallcpus_op = {
include/linux/nmi.h +17 -2
@@ -28,8 +28,23 @@ static inline void acpi_nmi_disable(void) { }static inline void acpi_nmi_enable (void ) { }#endif -#ifndef trigger_all_cpu_backtrace -#define trigger_all_cpu_backtrace () do { } while (0) +/* +* Create trigger_all_cpu_backtrace() out of the arch-provided +* base function. Return whether such support was available, +* to allow calling code to fall back to some other mechanism: +*/ +#ifdef arch_trigger_all_cpu_backtrace +static inline bool trigger_all_cpu_backtrace (void ) +{ +arch_trigger_all_cpu_backtrace (); + +return true; +} +#else +static inline bool trigger_all_cpu_backtrace (void ) +{ +return false; +}#endif #endif
include/linux/nmi.h static inline void acpi_nmi_enable(void) · line 36
Quoted 6 lines · Andrew Morton
#ifndef
trigger_all_cpu_backtraceinclude/linux/nmi.h renamed from include/linux/nmi.h~a +3
-0+#ifdef CONFIG_SMP +#warning This architecture is missing a trigger_all_cpu_backtrace() implementation +#endif #define trigger_all_cpu_backtrace () do { } while (0)#endif
I think a better solution will be to make this function return 1 if it generated a backtrace.
That way we can fall back to the generic schedule_work based codepath.
Albeit the best and cleanest solution would be to implement a workqueue based trigger_all_cpu_backtrace() __weak variant and throw away the workqueue bits from sysrq.c.
Ingo