tracing: make a snapshot feature available from userspace
Subsystems (3)
v4 cover tracing: make a snapshot feature available from userspace
Hi, Steven,
I updated the patch which replaces static variable 'old_tracer'.
v3->v4:
[1/3] tracing: replace static old_tracer with trace iterator's pointer to the
original tracer's name
- changed to comparison between pointers instead of strcmp
(v3: https://lkml.org/lkml/2012/12/19/35)
---
Hiraku Toyooka (3):
tracing: replace static old_tracer with trace iterator's pointer to the original tracer's name
tracing: make a snapshot feature available from userspace
tracing: add description of snapshot to Documentation/trace/ftrace.txt
Documentation/trace/ftrace.txt | 83 +++++++++++++++++++++ include/linux/ftrace_event.h | 3 + kernel/trace/Kconfig | 10 +++ kernel/trace/trace.c | 156 ++++++++++++++++++++++++++++++++++------ kernel/trace/trace.h | 1 5 files changed, 228 insertions(+), 25 deletions(-)
Signature · 4 lines
Hiraku TOYOOKA Linux Technology Center Yokohama Research Laboratory Hitachi Ltd.
PATCH v4 1/3 tracing: replace static old_tracer with trace iterator's pointer to the original tracer's name
Currently the trace buffer read functions use a static variable "old_tracer" for detecting if the current tracer changes. This was suitable for a single trace file ("trace"), but to add a snapshot feature that will use the same function for its file, a check against a static variable is not sufficient.
To use the output functions for two different files, instead of storing the current tracer in a static variable, as the trace iterator descriptor contains a pointer to the original current tracer's name, that pointer can now be used to check if the current tracer has changed between different reads of the trace file.
--- kernel/trace/trace.c | 22 +++++++++------------- 1 file changed, 9 insertions(+), 13 deletions(-)
kernel/trace/trace.c +9 -13
@@ -1948,18 +1948,20 @@ void tracing_iter_reset(struct trace_iterator *iter, int cpu)static void *s_start (struct seq_file *m,loff_t *pos) {struct trace_iterator *iter = m->private; -static struct tracer *old_tracer;int cpu_file = iter->cpu_file;void *p =NULL ;loff_t l =0 ;int cpu; -/* copy the tracer to avoid using a global lock all around */ +/* +* copy the tracer to avoid using a global lock all around. +* iter->trace is a copy of current_trace, the pointer to the +* name may be used instead of a strcmp(), as iter->trace->name +* will point to the same string as current_trace->name. +*/ mutex_lock (&trace_types_lock); -if (unlikely (old_tracer != current_trace && current_trace)) { - old_tracer = current_trace; +if (unlikely (current_trace && iter->trace->name != current_trace->name)) *iter->trace = *current_trace; - }mutex_unlock (&trace_types_lock);atomic_inc (&trace_record_cmdline_disabled);
@@ -3481,7 +3483,6 @@ tracing_read_pipe(struct file *filp, char __user *ubuf, size_t cnt, loff_t *ppos) { struct trace_iterator *iter = filp->private_data; -static struct tracer *old_tracer;ssize_t sret;/* return any leftover data */
@@ -3493,10 +3494,8 @@ tracing_read_pipe(struct file *filp, char __user *ubuf,/* copy the tracer to avoid using a global lock all around */ mutex_lock (&trace_types_lock); -if (unlikely (old_tracer != current_trace && current_trace)) { - old_tracer = current_trace; +if (unlikely (current_trace && iter->trace->name != current_trace->name)) *iter->trace = *current_trace; - }mutex_unlock (&trace_types_lock);/*
@@ -3652,7 +3651,6 @@ static ssize_t tracing_splice_read_pipe(struct file *filp, .ops = &tracing_pipe_buf_ops, .spd_release = tracing_spd_release_pipe, }; -static struct tracer *old_tracer;ssize_t ret;size_t rem;unsigned int i;
@@ -3662,10 +3660,8 @@ static ssize_t tracing_splice_read_pipe(struct file *filp,/* copy the tracer to avoid using a global lock all around */ mutex_lock (&trace_types_lock); -if (unlikely (old_tracer != current_trace && current_trace)) { - old_tracer = current_trace; +if (unlikely (current_trace && iter->trace->name != current_trace->name)) *iter->trace = *current_trace; - }mutex_unlock (&trace_types_lock);mutex_lock (&iter->mutex);
PATCH v4 2/3 tracing: make a snapshot feature available from userspace
In reply to Hiraku ToyookaFtrace has a snapshot feature available from kernel space and latency tracers (e.g. irqsoff) are using it. This patch enables user applictions to take a snapshot via debugfs.
Add "snapshot" debugfs file in "tracing" directory.
snapshot:
This is used to take a snapshot and to read the output of the snapshot.
# echo 1 > snapshot
This will allocate the spare buffer for snapshot (if it is not allocated), and take a snapshot.
# cat snapshot
This will show contents of the snapshot.
# echo 0 > snapshot
This will free the snapshot if it is allocated.
Any other positive values will clear the snapshot contents if the snapshot is allocated, or return EINVAL if it is not allocated.
--- include/linux/ftrace_event.h | 3 + kernel/trace/Kconfig | 10 +++ kernel/trace/trace.c | 134 ++++++++++++++++++++++++++++++++++++++---- kernel/trace/trace.h | 1 4 files changed, 136 insertions(+), 12 deletions(-)
include/linux/ftrace_event.h +3 -0
@@ -84,6 +84,9 @@ struct trace_iterator {long idx;cpumask_var_t started; + +/* it's true when current open file is snapshot */ +bool snapshot; };enum trace_iter_flags {
kernel/trace/Kconfig +10 -0
@@ -250,6 +250,16 @@ config FTRACE_SYSCALLS help Basic tracer to catch the syscall entry and exit events. +config TRACER_SNAPSHOT + bool "Create a snapshot trace buffer" + select TRACER_MAX_TRACE + help + Allow tracing users to take snapshot of the current buffer using the + ftrace interface, e.g.: + + echo 1 > /sys/kernel/debug/tracing/snapshot + cat snapshot + config TRACE_BRANCH_PROFILING bool select GENERIC_TRACER
kernel/trace/trace.c +122 -12
227 lines not on this page · raw message
kernel/trace/trace.h +1 -0
@@ -287,6 +287,7 @@ struct tracer {struct tracer_flags *flags;bool print_max;bool use_max_tr; +bool allocated_snapshot; };
PATCH v4 3/3 tracing: add description of snapshot to Documentation/trace/ftrace.txt
In reply to Hiraku ToyookaThis patch adds snapshot description in ftrace documentation. This description includes what the snapshot is and how to use it.
--- Documentation/trace/ftrace.txt | 83 ++++++++++++++++++++++++++++++++++++++++ 1 file changed, 83 insertions(+)
Documentation/trace/ftrace.txt +83 -0
@@ -1842,6 +1842,89 @@ an error. # cat buffer_size_kb 85 +Snapshot +-------- +CONFIG_TRACER_SNAPSHOT makes a generic snapshot feature +available to all non latency tracers. (Latency tracers which +record max latency, such as "irqsoff" or "wakeup", can't use +this feature, since those are already using the snapshot +mechanism internally.) + +Snapshot preserves a current trace buffer at a particular point +in time without stopping tracing. Ftrace swaps the current +buffer with a spare buffer, and tracing continues in the new +current (=previous spare) buffer. + +The following debugfs files in "tracing" are related to this +feature: + + snapshot: + + This is used to take a snapshot and to read the output + of the snapshot. Echo 1 into this file to allocate a + spare buffer and to take a snapshot (swap), then read + the snapshot from this file in the same format as + "trace" (described above in the section "The File + System"). Both reads snapshot and tracing are executable + in parallel. When the spare buffer is allocated, echoing + 0 frees it, and echoing else (positive) values clear the + snapshot contents. + More details are shown in the table below. + + status\input | 0 | 1 | else | + --------------+------------+------------+------------+ + not allocated |(do nothing)| alloc+swap | EINVAL | + --------------+------------+------------+------------+ + allocated | free | swap | clear | + --------------+------------+------------+------------+ + +Here is an example of using the snapshot feature. + + # echo 1 > events/sched/enable + # echo 1 > snapshot + # cat snapshot +# tracer: nop +# +# entries-in-buffer/entries-written: 71/71 #P:8 +# +# _-----=> irqs-off +# / _----=> need-resched +# | / _---=> hardirq/softirq +# || / _--=> preempt-depth +# ||| / delay +# TASK-PID CPU# |||| TIMESTAMP FUNCTION +# | | | |||| | | + <idle>-0 [005] d... 2440.603828: sched_switch: prev_comm=swapper/5 prev_pid=0 prev_prio=120 prev_state=R ==> next_comm=snapshot-test-2 next_pid=2242 next_prio=120 + sleep-2242 [005] d... 2440.603846: sched_switch: prev_comm=snapshot-test-2 prev_pid=2242 prev_prio=120 prev_state=R ==> next_comm=kworker/5:1 next_pid=60 next_prio=120 +[...] + <idle>-0 [002] d... 2440.707230: sched_switch: prev_comm=swapper/2 prev_pid=0 prev_prio=120 prev_state=R ==> next_comm=snapshot-test-2 next_pid=2229 next_prio=120 + + # cat trace +# tracer: nop +# +# entries-in-buffer/entries-written: 77/77 #P:8 +# +# _-----=> irqs-off +# / _----=> need-resched +# | / _---=> hardirq/softirq +# || / _--=> preempt-depth +# ||| / delay +# TASK-PID CPU# |||| TIMESTAMP FUNCTION +# | | | |||| | | + <idle>-0 [007] d... 2440.707395: sched_switch: prev_comm=swapper/7 prev_pid=0 prev_prio=120 prev_state=R ==> next_comm=snapshot-test-2 next_pid=2243 next_prio=120 + snapshot-test-2-2229 [002] d... 2440.707438: sched_switch: prev_comm=snapshot-test-2 prev_pid=2229 prev_prio=120 prev_state=S ==> next_comm=swapper/2 next_pid=0 next_prio=120 +[...] + + +If you try to use this snapshot feature when current tracer is +one of the latency tracers, you will get the following results. + + # echo wakeup > current_tracer + # echo 1 > snapshot +bash: echo: write error: Device or resource busy + # cat snapshot +cat: snapshot: Device or resource busy + ----------- More details can be found in the source code, in the
[tip:perf/core] tracing: Replace static old_tracer check of tracer name
In reply to Hiraku ToyookaCommit-ID: 2fd196ec1eab2623096e7fc7e6f3976160392bce Gitweb: http://git.kernel.org/tip/2fd196ec1eab2623096e7fc7e6f3976160392bce Author: Hiraku Toyooka <[email protected]> AuthorDate: Wed, 26 Dec 2012 11:52:52 +0900 Committer: Steven Rostedt <[email protected]> CommitDate: Wed, 30 Jan 2013 11:02:05 -0500
tracing: Replace static old_tracer check of tracer name
Currently the trace buffer read functions use a static variable "old_tracer" for detecting if the current tracer changes. This was suitable for a single trace file ("trace"), but to add a snapshot feature that will use the same function for its file, a check against a static variable is not sufficient.
To use the output functions for two different files, instead of storing the current tracer in a static variable, as the trace iterator descriptor contains a pointer to the original current tracer's name, that pointer can now be used to check if the current tracer has changed between different reads of the trace file.
--- kernel/trace/trace.c | 22 +++++++++------------- 1 file changed, 9 insertions(+), 13 deletions(-)
kernel/trace/trace.c +9 -13
@@ -1948,18 +1948,20 @@ void tracing_iter_reset(struct trace_iterator *iter, int cpu)static void *s_start (struct seq_file *m,loff_t *pos) {struct trace_iterator *iter = m->private; -static struct tracer *old_tracer;int cpu_file = iter->cpu_file;void *p =NULL ;loff_t l =0 ;int cpu; -/* copy the tracer to avoid using a global lock all around */ +/* +* copy the tracer to avoid using a global lock all around. +* iter->trace is a copy of current_trace, the pointer to the +* name may be used instead of a strcmp(), as iter->trace->name +* will point to the same string as current_trace->name. +*/ mutex_lock (&trace_types_lock); -if (unlikely (old_tracer != current_trace && current_trace)) { - old_tracer = current_trace; +if (unlikely (current_trace && iter->trace->name != current_trace->name)) *iter->trace = *current_trace; - }mutex_unlock (&trace_types_lock);atomic_inc (&trace_record_cmdline_disabled);
@@ -3494,7 +3496,6 @@ tracing_read_pipe(struct file *filp, char __user *ubuf, size_t cnt, loff_t *ppos) { struct trace_iterator *iter = filp->private_data; -static struct tracer *old_tracer;ssize_t sret;/* return any leftover data */
@@ -3506,10 +3507,8 @@ tracing_read_pipe(struct file *filp, char __user *ubuf,/* copy the tracer to avoid using a global lock all around */ mutex_lock (&trace_types_lock); -if (unlikely (old_tracer != current_trace && current_trace)) { - old_tracer = current_trace; +if (unlikely (current_trace && iter->trace->name != current_trace->name)) *iter->trace = *current_trace; - }mutex_unlock (&trace_types_lock);/*
@@ -3665,7 +3664,6 @@ static ssize_t tracing_splice_read_pipe(struct file *filp, .ops = &tracing_pipe_buf_ops, .spd_release = tracing_spd_release_pipe, }; -static struct tracer *old_tracer;ssize_t ret;size_t rem;unsigned int i;
@@ -3675,10 +3673,8 @@ static ssize_t tracing_splice_read_pipe(struct file *filp,/* copy the tracer to avoid using a global lock all around */ mutex_lock (&trace_types_lock); -if (unlikely (old_tracer != current_trace && current_trace)) { - old_tracer = current_trace; +if (unlikely (current_trace && iter->trace->name != current_trace->name)) *iter->trace = *current_trace; - }mutex_unlock (&trace_types_lock);mutex_lock (&iter->mutex);
[tip:perf/core] tracing: Make a snapshot feature available from userspace
In reply to Hiraku ToyookaCommit-ID: debdd57f5145f3c6a4b3f8d0126abd1a2def7fc6 Gitweb: http://git.kernel.org/tip/debdd57f5145f3c6a4b3f8d0126abd1a2def7fc6 Author: Hiraku Toyooka <[email protected]> AuthorDate: Wed, 26 Dec 2012 11:53:00 +0900 Committer: Steven Rostedt <[email protected]> CommitDate: Wed, 30 Jan 2013 11:02:06 -0500
tracing: Make a snapshot feature available from userspace
Ftrace has a snapshot feature available from kernel space and latency tracers (e.g. irqsoff) are using it. This patch enables user applictions to take a snapshot via debugfs.
Add "snapshot" debugfs file in "tracing" directory.
snapshot:
This is used to take a snapshot and to read the output of the snapshot.
# echo 1 > snapshot
This will allocate the spare buffer for snapshot (if it is not allocated), and take a snapshot.
# cat snapshot
This will show contents of the snapshot.
# echo 0 > snapshot
This will free the snapshot if it is allocated.
Any other positive values will clear the snapshot contents if the snapshot is allocated, or return EINVAL if it is not allocated.
[
Fixed irqsoff selftest and also a conflict with a change that fixes the update_max_tr.
]
--- include/linux/ftrace_event.h | 3 + kernel/trace/Kconfig | 10 +++ kernel/trace/trace.c | 166 ++++++++++++++++++++++++++++++++++++------- kernel/trace/trace.h | 1 + 4 files changed, 154 insertions(+), 26 deletions(-)
include/linux/ftrace_event.h +3 -0
@@ -83,6 +83,9 @@ struct trace_iterator {long idx;cpumask_var_t started; + +/* it's true when current open file is snapshot */ +bool snapshot; };enum trace_iter_flags {
kernel/trace/Kconfig +10 -0
@@ -253,6 +253,16 @@ config FTRACE_SYSCALLS help Basic tracer to catch the syscall entry and exit events. +config TRACER_SNAPSHOT + bool "Create a snapshot trace buffer" + select TRACER_MAX_TRACE + help + Allow tracing users to take snapshot of the current buffer using the + ftrace interface, e.g.: + + echo 1 > /sys/kernel/debug/tracing/snapshot + cat snapshot + config TRACE_BRANCH_PROFILING bool select GENERIC_TRACER
kernel/trace/trace.c +140 -26
273 lines not on this page · raw message
kernel/trace/trace.h +1 -0
@@ -287,6 +287,7 @@ struct tracer {struct tracer_flags *flags;bool print_max;bool use_max_tr; +bool allocated_snapshot; };
[tip:perf/core] tracing: Add documentation of snapshot utility
In reply to Hiraku ToyookaCommit-ID: c1043fcda1b9e8e5144cfdaee7be262c50dbdead Gitweb: http://git.kernel.org/tip/c1043fcda1b9e8e5144cfdaee7be262c50dbdead Author: Hiraku Toyooka <[email protected]> AuthorDate: Wed, 26 Dec 2012 11:53:09 +0900 Committer: Steven Rostedt <[email protected]> CommitDate: Wed, 30 Jan 2013 11:02:07 -0500
tracing: Add documentation of snapshot utility
This patch adds snapshot description in ftrace documentation. This description includes what the snapshot is and how to use it.
--- Documentation/trace/ftrace.txt | 83 ++++++++++++++++++++++++++++++++++++++++++ 1 file changed, 83 insertions(+)
Documentation/trace/ftrace.txt +83 -0
@@ -1842,6 +1842,89 @@ an error. # cat buffer_size_kb 85 +Snapshot +-------- +CONFIG_TRACER_SNAPSHOT makes a generic snapshot feature +available to all non latency tracers. (Latency tracers which +record max latency, such as "irqsoff" or "wakeup", can't use +this feature, since those are already using the snapshot +mechanism internally.) + +Snapshot preserves a current trace buffer at a particular point +in time without stopping tracing. Ftrace swaps the current +buffer with a spare buffer, and tracing continues in the new +current (=previous spare) buffer. + +The following debugfs files in "tracing" are related to this +feature: + + snapshot: + + This is used to take a snapshot and to read the output + of the snapshot. Echo 1 into this file to allocate a + spare buffer and to take a snapshot (swap), then read + the snapshot from this file in the same format as + "trace" (described above in the section "The File + System"). Both reads snapshot and tracing are executable + in parallel. When the spare buffer is allocated, echoing + 0 frees it, and echoing else (positive) values clear the + snapshot contents. + More details are shown in the table below. + + status\input | 0 | 1 | else | + --------------+------------+------------+------------+ + not allocated |(do nothing)| alloc+swap | EINVAL | + --------------+------------+------------+------------+ + allocated | free | swap | clear | + --------------+------------+------------+------------+ + +Here is an example of using the snapshot feature. + + # echo 1 > events/sched/enable + # echo 1 > snapshot + # cat snapshot +# tracer: nop +# +# entries-in-buffer/entries-written: 71/71 #P:8 +# +# _-----=> irqs-off +# / _----=> need-resched +# | / _---=> hardirq/softirq +# || / _--=> preempt-depth +# ||| / delay +# TASK-PID CPU# |||| TIMESTAMP FUNCTION +# | | | |||| | | + <idle>-0 [005] d... 2440.603828: sched_switch: prev_comm=swapper/5 prev_pid=0 prev_prio=120 prev_state=R ==> next_comm=snapshot-test-2 next_pid=2242 next_prio=120 + sleep-2242 [005] d... 2440.603846: sched_switch: prev_comm=snapshot-test-2 prev_pid=2242 prev_prio=120 prev_state=R ==> next_comm=kworker/5:1 next_pid=60 next_prio=120 +[...] + <idle>-0 [002] d... 2440.707230: sched_switch: prev_comm=swapper/2 prev_pid=0 prev_prio=120 prev_state=R ==> next_comm=snapshot-test-2 next_pid=2229 next_prio=120 + + # cat trace +# tracer: nop +# +# entries-in-buffer/entries-written: 77/77 #P:8 +# +# _-----=> irqs-off +# / _----=> need-resched +# | / _---=> hardirq/softirq +# || / _--=> preempt-depth +# ||| / delay +# TASK-PID CPU# |||| TIMESTAMP FUNCTION +# | | | |||| | | + <idle>-0 [007] d... 2440.707395: sched_switch: prev_comm=swapper/7 prev_pid=0 prev_prio=120 prev_state=R ==> next_comm=snapshot-test-2 next_pid=2243 next_prio=120 + snapshot-test-2-2229 [002] d... 2440.707438: sched_switch: prev_comm=snapshot-test-2 prev_pid=2229 prev_prio=120 prev_state=S ==> next_comm=swapper/2 next_pid=0 next_prio=120 +[...] + + +If you try to use this snapshot feature when current tracer is +one of the latency tracers, you will get the following results. + + # echo wakeup > current_tracer + # echo 1 > snapshot +bash: echo: write error: Device or resource busy + # cat snapshot +cat: snapshot: Device or resource busy + ----------- More details can be found in the source code, in the