Skip to contentSkip to conversation

tracing fixes

GIT PULL Ingo Molnar 2 participants 4 messages Feb 14, 2010 – Feb 15, 2010 Tracing

lore
Share & tools
lore
#0 lore raw

Linus,

Please pull the latest tracing-fixes-for-linus git tree from:

git://git.kernel.org/pub/scm/linux/kernel/git/tip/linux-2.6-tip.git tracing-fixes-for-linus

Thanks,

Ingo

------------------>
Heiko Carstens (1):
tracing/kprobes: Fix probe parsing

Lai Jiangshan (1):
tracing: Fix circular dead lock in stack trace

 kernel/trace/trace_kprobe.c |    2 +-
 kernel/trace/trace_stack.c  |   24 ++++++++++++++++++++++++
 2 files changed, 25 insertions(+), 1 deletions(-)
kernel/trace/trace_kprobe.c +1 -1
index 6ea90c0..50b1b82 100644
@@ -689,7 +689,7 @@ static int create_trace_probe(int argc, char **argv)
 			return -EINVAL;
 		}
 		/* an address specified */
-		ret = strict_strtoul(&argv[0][2], 0, (unsigned long *)&addr);
+		ret = strict_strtoul(&argv[1][0], 0, (unsigned long *)&addr);
 		if (ret) {
 			pr_info("Failed to parse address.\n");
 			return ret;
kernel/trace/trace_stack.c +24 -0
index 678a512..f4bc9b2 100644
@@ -157,6 +157,7 @@ stack_max_size_write(struct file *filp, const char __user *ubuf,
 	unsigned long val, flags;
 	char buf[64];
 	int ret;
+	int cpu;
 
 	if (count >= sizeof(buf))
 		return -EINVAL;
@@ -171,9 +172,20 @@ stack_max_size_write(struct file *filp, const char __user *ubuf,
 		return ret;
 
 	local_irq_save(flags);
+
+	/*
+	 * In case we trace inside arch_spin_lock() or after (NMI),
+	 * we will cause circular lock, so we also need to increase
+	 * the percpu trace_active here.
+	 */
+	cpu = smp_processor_id();
+	per_cpu(trace_active, cpu)++;
+
 	arch_spin_lock(&max_stack_lock);
 	*ptr = val;
 	arch_spin_unlock(&max_stack_lock);
+
+	per_cpu(trace_active, cpu)--;
 	local_irq_restore(flags);
 
 	return count;
@@ -206,7 +218,13 @@ t_next(struct seq_file *m, void *v, loff_t *pos)
 
 static void *t_start(struct seq_file *m, loff_t *pos)
 {
+	int cpu;
+
 	local_irq_disable();
+
+	cpu = smp_processor_id();
+	per_cpu(trace_active, cpu)++;
+
 	arch_spin_lock(&max_stack_lock);
 
 	if (*pos == 0)
@@ -217,7 +235,13 @@ static void *t_start(struct seq_file *m, loff_t *pos)
 
 static void t_stop(struct seq_file *m, void *p)
 {
+	int cpu;
+
 	arch_spin_unlock(&max_stack_lock);
+
+	cpu = smp_processor_id();
+	per_cpu(trace_active, cpu)--;
+
 	local_irq_enable();
 }
 
#1 lore raw

Hi Ingo,

Could you push this commit too?

5ecaafdbf44b1ba400b746c60c401d54c7ee0863
kprobes: Add mcount to the kprobes blacklist

Since this bug can be easily reproduced with kprobe-tracer.

Thank you,

Quoted 17 lines · Ingo Molnar

Linus,

Please pull the latest tracing-fixes-for-linus git tree from:

git://git.kernel.org/pub/scm/linux/kernel/git/tip/linux-2.6-tip.git tracing-fixes-for-linus

Thanks,

Ingo

------------------>
Heiko Carstens (1):
tracing/kprobes: Fix probe parsing

Lai Jiangshan (1):
tracing: Fix circular dead lock in stack trace

Signature · 7 lines
Masami Hiramatsu

Software Engineer
Hitachi Computer Products (America), Inc.
Software Solutions Division

e-mail: [email protected]
#2 lore raw
Quoted 8 lines · Masami Hiramatsu

Hi Ingo,

Could you push this commit too?

5ecaafdbf44b1ba400b746c60c401d54c7ee0863
kprobes: Add mcount to the kprobes blacklist

Since this bug can be easily reproduced with kprobe-tracer.

Ok, i've cherry-picked it into urgent.

I'm wondering, how complete is the kprobes blacklist? You can test it by putting a probe on every single function in the system via something like:

  for N in $(cut -d' ' -f3 /proc/kallsyms  | cut -f1); do
    perf probe $N
    perf probe -d $N
  done

(note: I havent tested it myself yet.)

Thanks,

Ingo

#3 lore raw
Quoted 14 lines · Ingo Molnar
Quoted 8 lines · Masami Hiramatsu

Hi Ingo,

Could you push this commit too?

5ecaafdbf44b1ba400b746c60c401d54c7ee0863
kprobes: Add mcount to the kprobes blacklist

Since this bug can be easily reproduced with kprobe-tracer.

Ok, i've cherry-picked it into urgent.

Quoted 7 lines

I'm wondering, how complete is the kprobes blacklist? You can test it by putting a probe on every single function in the system via something like:

  for N in $(cut -d' ' -f3 /proc/kallsyms  | cut -f1); do
    perf probe $N
    perf probe -d $N
  done

Yeah, something like that, and I used kprobe stress test tool which I attached for that :)

From http://lkml.org/lkml/2009/8/13/497
---
Usage
-----
kprobestest [-s SYMLIST] [-b BLACKLIST] [-w WHITELIST]
Run stress test. If SYMLIST file is specified, use it as an initial symbol list (This is useful for verifying white list after diagnosing all symbols).
kprobestest cleanup
Cleanup all lists

How to Work
-----------
This tool list up all symbols in the kernel via /proc/kallsyms, and sorts it into groups (each of them including 64 symbols in default). And then, it tests each group by using kprobe-tracer. If a kernel crash occurred, that group is moved into 'failed' dir. If the group passed the test, this script moves it into 'passed' dir and saves kprobe_profile into 'passed/profiles/'.
After testing all groups, all 'failed' groups are merged and sorted into smaller groups (divided by 4, in default). And those are tested again. This loop will be repeated until all group has just 1 symbol. Finally, the script sorts all 'passed' symbols into 'tested', 'untested', and 'missed' based on profiles.

Note
----
- This script just gives us some clues to the blacklisted functions. In some cases, a combination of probe points will cause a problem, but each of them doesn't cause the problem alone.
---

Thank you,

Signature · 7 lines
Masami Hiramatsu

Software Engineer
Hitachi Computer Products (America), Inc.
Software Solutions Division

e-mail: [email protected]

#!/bin/bash
#
# kprobestest: Kprobes stress test tool
# Written by Masami Hiramatsu <[email protected]> #
# Usage:
# $ kprobestest [-s SYMLIST] [-b BLACKLIST] [-w WHITELIST]

#        Run stress test. If SYMLIST file is specified, use it as 
#        an initial symbol list (This is useful for verifying white list
#        after diagnosing all symbols).

#
# $ kprobestest cleanup
# Cleanup all lists

DEBUGFS=/debug
INITNR=64
DIV=4
SYMFILE=syms.list
FAILFILE=black.list
FAILBKUP=1

function do_test () {
# Do some benchmark
for i in {1..4} ; do
sleep 0.5
echo -n "."
done
}

function usage () {
echo "Usage: kprobestest [cleanup] [-s SYMLIST] [-b BLACKLIST] [-w WHITELIST]" exit 0
}

function cleanup_test () {
rm -rf $SYMFILE failed passed testing unset backup exit 0
}

# Parse arguments
WHITELIST=
BLACKLIST=
SYMLIST=

while [ "$1" ]; do
case $1 in
cleanup)

    cleanup_test
    ;;
  run)
    # ignore
    ;;
  -s)
    SYMLIST=$2
    shift 1
    ;;
  -b)
    BLACKLIST=$2
    shift 1
    ;;
  -w)
    WHITELIST=$2
    shift 1
    ;;
  *)
    usage
    ;;
esac
shift 1
done

# Show configurations
echo "Kprobe stress test starting."

[ -f "$BLACKLIST" ] && echo "Blacklist: $BLACKLIST" || BLACKLIST=""
[ -f "$WHITELIST" ] && echo "Whitelist: $WHITELIST" || WHITELIST=""

[ -f "$SYMLIST" ] && echo "Symlist: $SYMLIST" || SYMLIST=""

function make_filter () {
local EXP=""
if [ -z "$WHITELIST" -a -z "$BLACKLIST" ]; then
echo "s/^$//g"
else
for i in cat $WHITELIST $BLACKLIST ;do

    [ -z "$EXP" ] && EXP="^$i\$" || EXP="$EXP\\|^$i\$"

done ; EXP="s/$EXP//g"
echo $EXP
fi
}

function list_allsyms () {
local sym
local out=0
for sym in sort /proc/kallsyms | egrep '[0-9a-f]+ [Tt] [^[]*$' | cut -d\ -f 3;do

  [ $sym  = "_stext" ] && out=1 && continue
  [ $sym  = "__kprobes_text_start" ] && out=0 && continue
  [ $sym  = "__kprobes_text_end" ] && out=1 && continue
  [ $sym  = "_etext" ] && break

[ $out -eq 1 ] && echo $sym
done
}

function prep_testing () {
local i=0
local n=0
local NR=$1
local fname=

echo "Grouping symbols: $NR"

fname=printf "list-%03d.%d" $i $NR
cat $SYMFILE | while read ln; do
[ -z "$ln" ] && continue
echo "$ln" >> testing/$fname
n=$((n+1))
if [ $n -eq $NR ]; then

    n=0
    i=$((i+1))
    fname=`printf "list-%03d.%d" $i $NR`

fi
done
sync
}

function init_first () {
local EXP
EXP=make_filter
if [ -f "$SYMLIST" ]; then
cat $SYMLIST | sed $EXP > $SYMFILE
else
echo -n "Generating symbol list from /proc/kallsyms..." list_allsyms | sed $EXP > $SYMFILE
echo "done. " wc -l $SYMFILE | cut -f1 -d\ "symbols listed."
fi
mkdir testing
mkdir failed
mkdir backup
mkdir unset
mkdir passed
mkdir passed/profiles
prep_testing $INITNR
}

function get_max_nr () {
wc -l failed/list-* unset/list-* 2>/dev/null |\
awk '/^ *[0-9]+ .*list.*$/{ if (nr < $1) nr=$1 } BEGIN { nr=0 } END { print nr}' }

function init_next () {
local NR
NR=get_max_nr
[ $NR -eq 0 ] && return 1
[ $NR -eq 1 ] && return 2
[ $NR -le $DIV ] && NR=1 || NR=expr $NR / $DIV

cat failed/* unset/* > $SYMFILE
if [ $FAILBKUP ]; then
cp failed/* unset/* backup/
fi
rm failed/* unset/*

prep_testing $NR
return 0
}

# Initialize symbols
if [ ! -d testing ]; then
init_first
elif [ -z "ls testing/" ]; then
init_next
fi

function set_probes () {
local s
for s in cat $1; do
echo "p:$s" $s >> $DEBUGFS/tracing/kprobe_events [ $? -ne 0 ] && return 1
echo 1 > $DEBUGFS/tracing/events/kprobes/$s/enable
done
return 0
}

function clear_probes () {
echo > $DEBUGFS/tracing/kprobe_events
}

function save_profile () {
cat $DEBUGFS/tracing/kprobe_profile > passed/profiles/$1.prof
}

clear_probes

echo "Starting tests.."

RET=0

while [ $RET -eq 0 ]; do

for list in cd testing/; ls; do
echo -n $list
mv testing/$list failed/
sync;sync
echo -n "sync.."
set_probes failed/$list
if [ $? -ne 0 ]; then
clear_probes
sync;sync
echo "can not set"
mv failed/$list unset/
sync;sync
else
do_test
save_profile $list
clear_probes
sync;sync
echo "done"
mv failed/$list passed/
sync;sync
fi
done

init_next
RET=$?
done

if [ $RET -eq 1 ];then
# No failed symbols
echo "no failed symbols found."
else
echo "found failed symbols:"
cat failed/* | tee $FAILFILE
rm failed/*
fi

cat unset/* > "unset.list"
rm unset/*

function profile_symbols () {
local s h m
rm -f tested.list missed.list untested.list cat passed/profiles/*.prof | while read s h m ;do
if [ $h -ne 0 ]; then
echo $s >> tested.list
elif [ $m -ne 0 ]; then
echo $s >> missed.list
else
echo $s >> untested.list
fi
done
}

echo -n "Profiling symbols..."
profile_symbols
echo done
echo tested: wc -l tested.list | cut -d\ -f1 symbols echo missed: wc -l missed.list | cut -d\ -f1 symbols echo untested: wc -l untested.list | cut -d\ -f1 symbols echo unset: wc -l unset.list | cut -d\ -f1 symbols