tracing: fix circular dead lock
PATCH 1/1 tracing: fix circular dead lock
When we cat <debugfs>/tracing/stack_trace, we may cause circular lock:sys_read()
t_start()
arch_spin_lock(&max_stack_lock);
t_show()
seq_printf(), vsnprintf() .... /* they are all trace-able,
when they are traced, max_stack_lock may be required again. */
The following script can trigger this circular dead lock very easy:
#!/bin/bash
echo 1 > /proc/sys/kernel/stack_tracer_enabled
mount -t debugfs xxx /mnt > /dev/null 2>&1
(
# make check_stack() zealous to require max_stack_lock
for ((; ;))
{
echo 1 > /mnt/tracing/stack_max_size
}
) &for ((; ;))
{
cat /mnt/tracing/stack_trace > /dev/null
}
To fix this bug, we increase the percpu trace_active before require the lock.
---
kernel/trace/trace_stack.c +24 -0
@@ -157,6 +157,7 @@ stack_max_size_write(struct file *filp, const char __user *ubuf,unsigned long val, flags;char buf[64 ];int ret; +int cpu;if (count >=sizeof (buf))return -EINVAL ;
@@ -171,9 +172,20 @@ stack_max_size_write(struct file *filp, const char __user *ubuf,return ret;local_irq_save (flags); + +/* +* In case we trace inside arch_spin_lock() or after (NMI), +* we will cause circular lock, so we also need to increase +* the percpu trace_active here. +*/ + cpu =smp_processor_id (); +per_cpu (trace_active, cpu)++; +arch_spin_lock (&max_stack_lock); *ptr = val;arch_spin_unlock (&max_stack_lock); + +per_cpu (trace_active, cpu)--;local_irq_restore (flags);return count;
@@ -206,7 +218,13 @@ t_next(struct seq_file *m, void *v, loff_t *pos)static void *t_start (struct seq_file *m,loff_t *pos) { +int cpu; +local_irq_disable (); + + cpu =smp_processor_id (); +per_cpu (trace_active, cpu)++; +arch_spin_lock (&max_stack_lock);if (*pos ==0 )
@@ -217,7 +235,13 @@ static void *t_start(struct seq_file *m, loff_t *pos)static void t_stop (struct seq_file *m,void *p) { +int cpu; +arch_spin_unlock (&max_stack_lock); + + cpu =smp_processor_id (); +per_cpu (trace_active, cpu)--; +local_irq_enable (); }
Steven Rostedt
maintainerQuoted 31 lines · Lai Jiangshan
When we cat <debugfs>/tracing/stack_trace, we may cause circular lock:
sys_read()
t_start()arch_spin_lock(&max_stack_lock);
t_show()
seq_printf(),vsnprintf().... /* they are all trace-able,
when they are traced,max_stack_lockmay be required again. */The following script can trigger this circular dead lock very easy:
#!/bin/bashecho 1 >
/proc/sys/kernel/stack_tracer_enabledmount -t debugfs xxx /mnt >
/dev/null2>&1(
# make check_stack() zealous to require max_stack_lock for ((; ;)) { echo 1 > /mnt/tracing/stack_max_size } ) &for ((; ;))
{
cat/mnt/tracing/stack_trace>/dev/null
}
Quoted 5 lines
To fix this bug, we increase the percpu
trace_activebefore require the lock.Reported-by Li Zefan <[email protected]>Signed-off-by Lai Jiangshan <[email protected]>
Thanks Lai! I'll try to get this into 33.
-- Steve
[tip:tracing/urgent] tracing: Fix circular dead lock in stack trace
In reply to Lai JiangshanCommit-ID: 4f48f8b7fd18c44f8478174f9925cc3c059c6ce4 Gitweb: http://git.kernel.org/tip/4f48f8b7fd18c44f8478174f9925cc3c059c6ce4 Author: Lai Jiangshan <[email protected]> AuthorDate: Tue, 2 Feb 2010 15:32:09 +0800 Committer: Steven Rostedt <[email protected]> CommitDate: Tue, 2 Feb 2010 10:20:18 -0500
tracing: Fix circular dead lock in stack trace
When we cat <debugfs>/tracing/stack_trace, we may cause circular lock:sys_read()
t_start()
arch_spin_lock(&max_stack_lock);
t_show()
seq_printf(), vsnprintf() .... /* they are all trace-able,
when they are traced, max_stack_lock may be required again. */
The following script can trigger this circular dead lock very easy:
#!/bin/bash
echo 1 > /proc/sys/kernel/stack_tracer_enabled
mount -t debugfs xxx /mnt > /dev/null 2>&1
(
# make check_stack() zealous to require max_stack_lock
for ((; ;))
{
echo 1 > /mnt/tracing/stack_max_size
}
) &for ((; ;))
{
cat /mnt/tracing/stack_trace > /dev/null
}
To fix this bug, we increase the percpu trace_active before require the lock.
LKML-Reference: <[email protected]>
--- kernel/trace/trace_stack.c | 24 ++++++++++++++++++++++++ 1 files changed, 24 insertions(+), 0 deletions(-)
kernel/trace/trace_stack.c +24 -0
@@ -157,6 +157,7 @@ stack_max_size_write(struct file *filp, const char __user *ubuf,unsigned long val, flags;char buf[64 ];int ret; +int cpu;if (count >=sizeof (buf))return -EINVAL ;
@@ -171,9 +172,20 @@ stack_max_size_write(struct file *filp, const char __user *ubuf,return ret;local_irq_save (flags); + +/* +* In case we trace inside arch_spin_lock() or after (NMI), +* we will cause circular lock, so we also need to increase +* the percpu trace_active here. +*/ + cpu =smp_processor_id (); +per_cpu (trace_active, cpu)++; +arch_spin_lock (&max_stack_lock); *ptr = val;arch_spin_unlock (&max_stack_lock); + +per_cpu (trace_active, cpu)--;local_irq_restore (flags);return count;
@@ -206,7 +218,13 @@ t_next(struct seq_file *m, void *v, loff_t *pos)static void *t_start (struct seq_file *m,loff_t *pos) { +int cpu; +local_irq_disable (); + + cpu =smp_processor_id (); +per_cpu (trace_active, cpu)++; +arch_spin_lock (&max_stack_lock);if (*pos ==0 )
@@ -217,7 +235,13 @@ static void *t_start(struct seq_file *m, loff_t *pos)static void t_stop (struct seq_file *m,void *p) { +int cpu; +arch_spin_unlock (&max_stack_lock); + + cpu =smp_processor_id (); +per_cpu (trace_active, cpu)--; +local_irq_enable (); }