tracing: updates for 3.5
Subsystems (4)
Steven Rostedt
maintainerIngo,
Please pull the latest tip/perf/core tree, which can be found at:
git://git.kernel.org/pub/scm/linux/kernel/git/rostedt/linux-trace.git
tip/perf/core
Head SHA1: 61463fad01fe416ab55d56a970a707b33d081436
Dan Carpenter (1):
tracing: Remove an unneeded check in trace_seq_buffer()
Steven Rostedt (3):
tracing: Add percpu buffers for trace_printk() ftrace/x86: Have arch x86_64 use breakpoints instead of stop machine ftrace/x86: Remove the complex ftrace NMI handling code
Vaibhav Nagarnaik (1):
ring-buffer: Add per_cpu ring buffer control files
----
arch/x86/Kconfig | 1 - arch/x86/include/asm/ftrace.h | 4 + arch/x86/kernel/ftrace.c | 512 +++++++++++++++++++++++++++-------------- arch/x86/kernel/nmi.c | 10 +- arch/x86/kernel/traps.c | 9 +- include/linux/ftrace.h | 6 + include/linux/kernel.h | 13 +- include/linux/ring_buffer.h | 6 +- kernel/trace/ring_buffer.c | 248 +++++++++++--------- kernel/trace/trace.c | 379 ++++++++++++++++++++++-------- kernel/trace/trace.h | 4 +- kernel/trace/trace_printk.c | 4 + 12 files changed, 813 insertions(+), 383 deletions(-)
Steven Rostedt
maintainerPATCH 1/5 tracing: Add percpu buffers for trace_printk()
From: Steven Rostedt <[email protected]>
Currently, trace_printk() uses a single buffer to write into to calculate the size and format needed to save the trace. To do this safely in an SMP environment, a spin_lock() is taken to only allow one writer at a time to the buffer. But this could also affect what is being traced, and add synchronization that would not be there otherwise.
Ideally, using percpu buffers would be useful, but since trace_printk() is only used in development, having per cpu buffers for something never used is a waste of space. Thus, the use of the trace_bprintk() format section is changed to be used for static fmts as well as dynamic ones. Then at boot up, we can check if the section that holds the trace_printk formats is non-empty, and if it does contain something, then we know a trace_printk() has been added to the kernel. At this time the trace_printk per cpu buffers are allocated. A check is also done at module load time in case a module is added that contains a trace_printk().
Once the buffers are allocated, they are never freed. If you use a trace_printk() then you should know what you are doing.
A buffer is made for each type of context:
normal
softirq
irq
nmi
The context is checked and the appropriate buffer is used. This allows for totally lockless usage of trace_printk(), and they no longer even disable interrupts.
--- include/linux/kernel.h | 13 +-- kernel/trace/trace.c | 184 +++++++++++++++++++++++++++++++------------ kernel/trace/trace.h | 2 + kernel/trace/trace_printk.c | 4 + 4 files changed, 148 insertions(+), 55 deletions(-)
include/linux/kernel.h +7 -6
@@ -480,15 +480,16 @@ do { \ #define trace_printk(fmt, args...) \ do { \ + static const char *trace_printk_fmt \ + __attribute__((section("__trace_printk_fmt"))) = \ + __builtin_constant_p(fmt) ? fmt : NULL; \ + \ __trace_printk_check_format(fmt, ##args); \ - if (__builtin_constant_p(fmt)) { \ - static const char *trace_printk_fmt \ - __attribute__((section("__trace_printk_fmt"))) = \ - __builtin_constant_p(fmt) ? fmt : NULL; \ \ + if (__builtin_constant_p(fmt)) \ __trace_bprintk(_THIS_IP_, trace_printk_fmt, ##args); \ - } else \ - __trace_printk(_THIS_IP_, fmt, ##args); \ + else \ + __trace_printk(_THIS_IP_, fmt, ##args); \ } while (0) extern __printf(2, 3)
kernel/trace/trace.c +135 -49
267 lines not on this page · raw message
kernel/trace/trace.h +2 -0
@@ -826,6 +826,8 @@ extern struct list_head ftrace_events;extern const char *__start___trace_bprintk_fmt[];extern const char *__stop___trace_bprintk_fmt[]; +void trace_printk_init_buffers (void ); +#undef FTRACE_ENTRY#define FTRACE_ENTRY (call, struct_name, id, tstruct, print, filter) \ extern struct ftrace_event_call \
kernel/trace/trace_printk.c +4 -0
@@ -51,6 +51,10 @@ void hold_module_trace_bprintk_format(const char **start, const char **end)const char **iter;char *fmt; +/* allocate the trace_printk per cpu buffers */ +if (start != end) +trace_printk_init_buffers (); +mutex_lock (&btrace_mutex);for (iter = start; iter < end; iter++) {struct trace_bprintk_fmt *tb_fmt =lookup_format (*iter);
Signature · 1 line
1.7.9.5
Steven Rostedt
maintainerPATCH 2/5 tracing: Remove an unneeded check in trace_seq_buffer()
In reply to Steven RostedtFrom: Dan Carpenter <[email protected]>
memcpy() returns a pointer to "bug". Hopefully, it's not NULL here or we would already have Oopsed.
--- kernel/trace/trace.c | 5 +---- 1 file changed, 1 insertion(+), 4 deletions(-)
kernel/trace/trace.c +1 -4
@@ -629,7 +629,6 @@ ssize_t trace_seq_to_user(struct trace_seq *s, char __user *ubuf, size_t cnt)static ssize_t trace_seq_to_buffer (struct trace_seq *s,void *buf,size_t cnt) {int len; -void *ret;if (s->len <= s->readpos)return -EBUSY ;
@@ -637,9 +636,7 @@ static ssize_t trace_seq_to_buffer(struct trace_seq *s, void *buf, size_t cnt) len = s->len - s->readpos;if (cnt > len) cnt = len; - ret =memcpy (buf, s->buffer + s->readpos, cnt); -if (!ret) -return -EFAULT ; +memcpy (buf, s->buffer + s->readpos, cnt); s->readpos += cnt;return cnt;
Signature · 1 line
1.7.9.5
Steven Rostedt
maintainerPATCH 3/5 ring-buffer: Add per_cpu ring buffer control files
In reply to Steven RostedtFrom: Vaibhav Nagarnaik <[email protected]>
Add a debugfs entry under per_cpu/ folder for each cpu called buffer_size_kb to control the ring buffer size for each CPU independently.
If the global file buffer_size_kb is used to set size, the individual ring buffers will be adjusted to the given size. The buffer_size_kb will report the common size to maintain backward compatibility.
If the buffer_size_kb file under the per_cpu/ directory is used to change buffer size for a specific CPU, only the size of the respective ring buffer is updated. When tracing/buffer_size_kb is read, it reports 'X' to indicate that sizes of per_cpu ring buffers are not equivalent.
--- include/linux/ring_buffer.h | 6 +- kernel/trace/ring_buffer.c | 248 ++++++++++++++++++++++++------------------- kernel/trace/trace.c | 190 ++++++++++++++++++++++++++------- kernel/trace/trace.h | 2 +- 4 files changed, 297 insertions(+), 149 deletions(-)
include/linux/ring_buffer.h +4 -2
@@ -96,9 +96,11 @@ __ring_buffer_alloc(unsigned long size, unsigned flags, struct lock_class_key *k__ring_buffer_alloc ((size), (flags), &__key); \ }) +#define RING_BUFFER_ALL_CPUS -1 +void ring_buffer_free (struct ring_buffer *buffer); -int ring_buffer_resize (struct ring_buffer *buffer,unsigned long size); +int ring_buffer_resize (struct ring_buffer *buffer,unsigned long size,int cpu);void ring_buffer_change_overwrite (struct ring_buffer *buffer,int val);
@@ -129,7 +131,7 @@ ring_buffer_read(struct ring_buffer_iter *iter, u64 *ts);void ring_buffer_iter_reset (struct ring_buffer_iter *iter);int ring_buffer_iter_empty (struct ring_buffer_iter *iter); -unsigned long ring_buffer_size (struct ring_buffer *buffer); +unsigned long ring_buffer_size (struct ring_buffer *buffer,int cpu);void ring_buffer_reset_cpu (struct ring_buffer *buffer,int cpu);void ring_buffer_reset (struct ring_buffer *buffer);
kernel/trace/ring_buffer.c +140 -108
421 lines not on this page · raw message
kernel/trace/trace.c +152 -38
349 lines not on this page · raw message
kernel/trace/trace.h +1 -1
@@ -131,6 +131,7 @@ struct trace_array_cpu {atomic_t disabled;void *buffer_page;/* ring buffer spare */ +unsigned long entries;unsigned long saved_latency;unsigned long critical_start;unsigned long critical_end;
@@ -152,7 +153,6 @@ struct trace_array_cpu {*/ struct trace_array {struct ring_buffer *buffer; -unsigned long entries;int cpu;int buffer_disabled;cycle_t time_start;
Signature · 1 line
1.7.9.5
Steven Rostedt
maintainerPATCH 4/5 ftrace/x86: Have arch x86_64 use breakpoints instead of stop machine
In reply to Steven RostedtFrom: Steven Rostedt <[email protected]>
This method changes x86 to add a breakpoint to the mcount locations instead of calling stop machine.
Now that iret can be handled by NMIs, we perform the following to update code:
1) Add a breakpoint to all locations that will be modified
2) Sync all cores
3) Update all locations to be either a nop or call (except breakpoint op)
4) Sync all cores
5) Remove the breakpoint with the new code.
6) Sync all cores
[
Added updates that Masami suggested:
Use unlikely(modifying_ftrace_code) in int3 trap to keep kprobes efficient. Don't use NOTIFY_* in ftrace handler in int3 as it is not a notifier.
]
--- arch/x86/include/asm/ftrace.h | 4 + arch/x86/kernel/ftrace.c | 343 +++++++++++++++++++++++++++++++++++++++++ arch/x86/kernel/traps.c | 9 +- include/linux/ftrace.h | 6 + 4 files changed, 361 insertions(+), 1 deletion(-)
arch/x86/include/asm/ftrace.h +4 -0
@@ -34,6 +34,7 @@#ifndef __ASSEMBLY__extern void mcount (void ); +extern int modifying_ftrace_code;static inline unsigned long ftrace_call_adjust (unsigned long addr) {
@@ -50,6 +51,9 @@ struct dyn_arch_ftrace {/* No extra data needed for x86 */ }; +int ftrace_int3_handler (int cmd,const char *str, +struct pt_regs *regs,long err,int trap,int sig); +#endif /* CONFIG_DYNAMIC_FTRACE */ #endif /* __ASSEMBLY__ */ #endif /* CONFIG_FUNCTION_TRACER */
arch/x86/kernel/ftrace.c +343 -0
355 lines not on this page · raw message
arch/x86/kernel/traps.c +8 -1
@@ -50,6 +50,7 @@#include <asm/processor.h> #include <asm/debugreg.h> #include <linux/atomic.h> +#include <asm/ftrace.h> #include <asm/traps.h> #include <asm/desc.h> #include <asm/i387.h>
@@ -303,8 +304,14 @@ gp_in_kernel: }/* May run on IST stack. */ -dotraplinkage void __kprobes do_int3(struct pt_regs *regs, long error_code) +dotraplinkage void __kprobes notrace do_int3(struct pt_regs *regs, long error_code) { +#ifdef CONFIG_FUNCTION_TRACER +/* ftrace must be first, everything else may cause a recursive crash */ +if (unlikely (modifying_ftrace_code) && +ftrace_int3_handler (DIE_INT3 ,"int3" , regs, error_code,3 ,SIGTRAP )) +return ; +#endif #ifdef CONFIG_KGDB_LOW_LEVEL_TRAP if (kgdb_ll_trap (DIE_INT3 ,"int3" , regs, error_code,X86_TRAP_BP ,SIGTRAP ) ==NOTIFY_STOP )
include/linux/ftrace.h +6 -0
@@ -286,6 +286,12 @@ struct ftrace_rec_iter *ftrace_rec_iter_start(void);struct ftrace_rec_iter *ftrace_rec_iter_next (struct ftrace_rec_iter *iter);struct dyn_ftrace *ftrace_rec_iter_record (struct ftrace_rec_iter *iter); +#define for_ftrace_rec_iter (iter) \ + for (iter = ftrace_rec_iter_start(); \ + iter; \ + iter = ftrace_rec_iter_next(iter)) + +int ftrace_update_record (struct dyn_ftrace *rec,int enable);int ftrace_test_record (struct dyn_ftrace *rec,int enable);void ftrace_run_stop_machine (int command);
Signature · 1 line
1.7.9.5
Re: [PATCH 4/5] ftrace/x86: Have arch x86_64 use breakpoints instead of stop machine
Quoted 19 lines · Steven Rostedt
From: Steven Rostedt <[email protected]>
This method changes x86 to add a breakpoint to the mcount locations instead of calling stop machine.
Now that iret can be handled by NMIs, we perform the following to update code:
1) Add a breakpoint to all locations that will be modified
2) Sync all cores
3) Update all locations to be either a nop or call (except breakpoint op)
4) Sync all cores
5) Remove the breakpoint with the new code.
Quoted 7 lines
6) Sync all cores
[
Added updates that Masami suggested:
Useunlikely(modifying_ftrace_code)in int3 trap to keep kprobes efficient. Don't use NOTIFY_* in ftrace handler in int3 as it is not a notifier.
]
I just have a few comments :)
[...]
Quoted 5 lines
arch/x86/kernel/ftrace.c +5
-0+int modifying_ftrace_code__read_mostly ; + +/* +* A breakpoint was added to the code address we are about to +* modify, and this is the handle that will just skip over it.
arch/x86/kernel/ftrace.c int ftrace_update_ftrace_func(ftrace_func_t func) · line 352
Quoted 10 lines
arch/x86/kernel/ftrace.c +10
-0+* We are either changing a nop into a trace call, or a trace +* call to a nop. While the change is taking place, we treat +* it just like it was a nop. +*/ +int ftrace_int3_handler (int cmd,const char *str, +struct pt_regs *regs,long err,int trap,int sig) +{ +if (!modifying_ftrace_code || cmd !=DIE_INT3 || !regs) +return 0 ; +
At this moment, all of these checks are meaningless, because this is not a handler but just a fixup code in int3 isr. :) So, I think you can just define "int ftrace_int3_fixup(regs)" for this purpose.
Quoted 7 lines
arch/x86/kernel/ftrace.c +7
-0+if (!ftrace_location (regs->ip -1 )) +return 0 ; + + regs->ip +=MCOUNT_INSN_SIZE -1 ; + +return 1 ; +}
[...]
Quoted 13 lines
arch/x86/kernel/traps.c +1
-0@@ -50,6 +50,7 @@#include <asm/processor.h> #include <asm/debugreg.h> #include <linux/atomic.h> +#include <asm/ftrace.h> #include <asm/traps.h> #include <asm/desc.h> #include <asm/i387.h>
arch/x86/kernel/traps.c gp_in_kernel: · line 309
Quoted 7 lines
arch/x86/kernel/traps.c +2
-1@@ -303,8 +304,14 @@ gp_in_kernel: }/* May run on IST stack. */ -dotraplinkage void __kprobes do_int3(struct pt_regs *regs, long error_code) +dotraplinkage void __kprobes notrace do_int3(struct pt_regs *regs, long error_code) { +#ifdef CONFIG_FUNCTION_TRACER
I guess CONFIG_DYNAMIC_FTRACE ?
arch/x86/kernel/traps.c gp_in_kernel: · line 314
Quoted 5 lines
arch/x86/kernel/traps.c +5
-0+/* ftrace must be first, everything else may cause a recursive crash */ +if (unlikely (modifying_ftrace_code) && +ftrace_int3_handler (DIE_INT3 ,"int3" , regs, error_code,3 ,SIGTRAP )) +return ; +#endif
Thank you!
Signature · 4 lines
Masami HIRAMATSU Software Platform Research Dept. Linux Technology Center Hitachi, Ltd., Yokohama Research Laboratory E-mail: [email protected]
Steven Rostedt
maintainerRe: [PATCH 4/5] ftrace/x86: Have arch x86_64 use breakpoints instead of stop machine
Quoted 22 lines · Masami Hiramatsu
Quoted 21 lines · Steven Rostedt
From: Steven Rostedt <[email protected]>
This method changes x86 to add a breakpoint to the mcount locations instead of calling stop machine.
Now that iret can be handled by NMIs, we perform the following to update code:
1) Add a breakpoint to all locations that will be modified
2) Sync all cores
3) Update all locations to be either a nop or call (except breakpoint op)
4) Sync all cores
5) Remove the breakpoint with the new code.
6) Sync all cores
Quoted 8 lines
Quoted 5 lines
[
Added updates that Masami suggested:
Useunlikely(modifying_ftrace_code)in int3 trap to keep kprobes efficient. Don't use NOTIFY_* in ftrace handler in int3 as it is not a notifier.
]I just have a few comments :)
Thanks Masami,
below is the changes I made. I'll post a v2.
-- Steve
arch/x86/include/asm/ftrace.h +1 -2
@@ -51,8 +51,7 @@ struct dyn_arch_ftrace {/* No extra data needed for x86 */ }; -int ftrace_int3_handler (int cmd,const char *str, -struct pt_regs *regs,long err,int trap,int sig); +int ftrace_int3_handler (struct pt_regs *regs);#endif /* CONFIG_DYNAMIC_FTRACE */ #endif /* __ASSEMBLY__ */
arch/x86/kernel/ftrace.c +2 -3
@@ -344,10 +344,9 @@ int modifying_ftrace_code __read_mostly;* call to a nop. While the change is taking place, we treat * it just like it was a nop. */ -int ftrace_int3_handler (int cmd,const char *str, -struct pt_regs *regs,long err,int trap,int sig) +int ftrace_int3_handler (struct pt_regs *regs) { -if (!modifying_ftrace_code || cmd !=DIE_INT3 || !regs) +if (WARN_ON_ONCE (!regs))return 0 ;if (!ftrace_location (regs->ip -1 ))
arch/x86/kernel/traps.c +2 -3
@@ -306,10 +306,9 @@ gp_in_kernel:/* May run on IST stack. */ dotraplinkage void __kprobes notrace do_int3(struct pt_regs *regs, long error_code) { -#ifdef CONFIG_FUNCTION_TRACER +#ifdef CONFIG_DYNAMIC_FTRACE /* ftrace must be first, everything else may cause a recursive crash */ -if (unlikely (modifying_ftrace_code) && -ftrace_int3_handler (DIE_INT3 ,"int3" , regs, error_code,3 ,SIGTRAP )) +if (unlikely (modifying_ftrace_code) &&ftrace_int3_handler (regs))return ;#endif #ifdef CONFIG_KGDB_LOW_LEVEL_TRAP
Steven Rostedt
maintainerPATCH 5/5 ftrace/x86: Remove the complex ftrace NMI handling code
In reply to Steven RostedtFrom: Steven Rostedt <[email protected]>
As ftrace function tracing would require modifying code that could be executed in NMI context, which is not stopped with stop_machine(), ftrace had to do a complex algorithm with various stages of setup and memory barriers to make it work.
With the new breakpoint method, this is no longer required. The changes to the code can be done without any problem in NMI context, as well as without stop machine altogether. Remove the complex code as it is no longer needed.
Also, a lot of the notrace annotations could be removed from the NMI code as it is now safe to trace them. With the exception of do_nmi itself, which does some special work to handle running in the debug stack. The breakpoint method can cause NMIs to double nest the debug stack if it's not setup properly, and that is done in do_nmi(), thus that function must not be traced.
(Note the arch sh may want to do the same)
--- arch/x86/Kconfig | 1 - arch/x86/kernel/ftrace.c | 169 +--------------------------------------------- arch/x86/kernel/nmi.c | 10 +-- 3 files changed, 6 insertions(+), 174 deletions(-)
arch/x86/Kconfig +0 -1
@@ -40,7 +40,6 @@ config X86 select HAVE_FUNCTION_GRAPH_TRACER select HAVE_FUNCTION_GRAPH_FP_TEST select HAVE_FUNCTION_TRACE_MCOUNT_TEST - select HAVE_FTRACE_NMI_ENTER if DYNAMIC_FTRACE select HAVE_SYSCALL_TRACEPOINTS select HAVE_KVM select HAVE_ARCH_KGDB
arch/x86/kernel/ftrace.c +1 -168
199 lines not on this page · raw message
arch/x86/kernel/nmi.c +5 -5
@@ -84,7 +84,7 @@ __setup("unknown_nmi_panic", setup_unknown_nmi_panic);#define nmi_to_desc (type) (&nmi_desc[type]) -static int notrace __kprobes nmi_handle(unsigned int type, struct pt_regs *regs, bool b2b) +static int __kprobes nmi_handle (unsigned int type,struct pt_regs *regs,bool b2b) {struct nmi_desc *desc =nmi_to_desc (type);struct nmiaction *a;
@@ -209,7 +209,7 @@ void unregister_nmi_handler(unsigned int type, const char *name)EXPORT_SYMBOL_GPL (unregister_nmi_handler); -static notrace __kprobes void +static __kprobes void pci_serr_error (unsigned char reason,struct pt_regs *regs) { pr_emerg("NMI: PCI system error (SERR) for reason %02x on CPU %d.\n",
@@ -236,7 +236,7 @@ pci_serr_error(unsigned char reason, struct pt_regs *regs)outb (reason,NMI_REASON_PORT ); } -static notrace __kprobes void +static __kprobes void io_check_error (unsigned char reason,struct pt_regs *regs) {unsigned long i;
@@ -263,7 +263,7 @@ io_check_error(unsigned char reason, struct pt_regs *regs)outb (reason,NMI_REASON_PORT ); } -static notrace __kprobes void +static __kprobes void unknown_nmi_error (unsigned char reason,struct pt_regs *regs) {int handled;
@@ -305,7 +305,7 @@ unknown_nmi_error(unsigned char reason, struct pt_regs *regs) static DEFINE_PER_CPU(bool, swallow_nmi); static DEFINE_PER_CPU(unsigned long, last_nmi_rip); -static notrace __kprobes void default_do_nmi(struct pt_regs *regs) +static __kprobes void default_do_nmi (struct pt_regs *regs) {unsigned char reason =0 ;int handled;
Signature · 1 line
1.7.9.5