ftrace: trace irq disabled critical timings
PATCH 15/19 ftrace: trace irq disabled critical timings
From: Steven Rostedt <[email protected]>
This patch adds latency tracing for critical timings (how long interrupts are disabled for).
"irqsoff" is added to /debugfs/tracing/available_tracers
Note:
tracing_max_latency
also holds the max latency for irqsoff (in usecs).
(default to large number so one must start latency tracing)
tracing_thresh
threshold (in usecs) to always print out if irqs off is detected to be longer than stated here. If irq_thresh is non-zero, then max_irq_latency is ignored.
Here's an example of a trace with ftrace_enabled = 0
=======
preemption latency trace v1.1.5 on 2.6.24-rc7
--------------------------------------------------------------------
latency: 100 us, #3/3, CPU#1 | (M:rt VP:0, KP:0, SP:0 HP:0 #P:2)
-----------------
| task: swapper-0 (uid:0 nice:0 policy:0 rt_prio:0)
----------------- => started at: _spin_lock_irqsave+0x2a/0xb7
=> ended at: _spin_unlock_irqrestore+0x32/0x5f
_------=> CPU#
/ _-----=> irqs-off
| / _----=> need-resched
|| / _---=> hardirq/softirq
||| / _--=> preempt-depth
|||| /
||||| delay
cmd pid ||||| time | caller
\ / ||||| \ | /
swapper-0 1d.s3 0us+: _spin_lock_irqsave+0x2a/0xb7 (e1000_update_stats+0x47/0x64c [e1000])
swapper-0 1d.s3 100us : _spin_unlock_irqrestore+0x32/0x5f (e1000_update_stats+0x641/0x64c [e1000])
swapper-0 1d.s3 100us : trace_hardirqs_on_caller+0x75/0x89 (_spin_unlock_irqrestore+0x32/0x5f)vim:ft=help
=======
And this is a trace with ftrace_enabled == 1
=======
preemption latency trace v1.1.5 on 2.6.24-rc7
--------------------------------------------------------------------
latency: 102 us, #12/12, CPU#1 | (M:rt VP:0, KP:0, SP:0 HP:0 #P:2)
-----------------
| task: swapper-0 (uid:0 nice:0 policy:0 rt_prio:0)
----------------- => started at: _spin_lock_irqsave+0x2a/0xb7
=> ended at: _spin_unlock_irqrestore+0x32/0x5f
_------=> CPU#
/ _-----=> irqs-off
| / _----=> need-resched
|| / _---=> hardirq/softirq
||| / _--=> preempt-depth
|||| /
||||| delay
cmd pid ||||| time | caller
\ / ||||| \ | /
swapper-0 1dNs3 0us+: _spin_lock_irqsave+0x2a/0xb7 (e1000_update_stats+0x47/0x64c [e1000])
swapper-0 1dNs3 46us : e1000_read_phy_reg+0x16/0x225 [e1000] (e1000_update_stats+0x5e2/0x64c [e1000])
swapper-0 1dNs3 46us : e1000_swfw_sync_acquire+0x10/0x99 [e1000] (e1000_read_phy_reg+0x49/0x225 [e1000])
swapper-0 1dNs3 46us : e1000_get_hw_eeprom_semaphore+0x12/0xa6 [e1000] (e1000_swfw_sync_acquire+0x36/0x99 [e1000])
swapper-0 1dNs3 47us : __const_udelay+0x9/0x47 (e1000_read_phy_reg+0x116/0x225 [e1000])
swapper-0 1dNs3 47us+: __delay+0x9/0x50 (__const_udelay+0x45/0x47)
swapper-0 1dNs3 97us : preempt_schedule+0xc/0x84 (__delay+0x4e/0x50)
swapper-0 1dNs3 98us : e1000_swfw_sync_release+0xc/0x55 [e1000] (e1000_read_phy_reg+0x211/0x225 [e1000])
swapper-0 1dNs3 99us+: e1000_put_hw_eeprom_semaphore+0x9/0x35 [e1000] (e1000_swfw_sync_release+0x50/0x55 [e1000])
swapper-0 1dNs3 101us : _spin_unlock_irqrestore+0xe/0x5f (e1000_update_stats+0x641/0x64c [e1000])
swapper-0 1dNs3 102us : _spin_unlock_irqrestore+0x32/0x5f (e1000_update_stats+0x641/0x64c [e1000])
swapper-0 1dNs3 102us : trace_hardirqs_on_caller+0x75/0x89 (_spin_unlock_irqrestore+0x32/0x5f)vim:ft=help
=======
--- arch/x86/kernel/process_64.c | 3 arch/x86/lib/Makefile | 1 arch/x86/lib/thunk_32.S | 47 +++++ arch/x86/lib/thunk_64.S | 19 +- include/asm-x86/irqflags.h | 24 -- include/linux/ftrace.h | 8 include/linux/irqflags.h | 12 + kernel/fork.c | 2 kernel/lockdep.c | 23 ++ kernel/printk.c | 2 kernel/trace/Kconfig | 18 + kernel/trace/Makefile | 1 kernel/trace/trace_irqsoff.c | 402 +++++++++++++++++++++++++++++++++++++++++++ 13 files changed, 531 insertions(+), 31 deletions(-)
arch/x86/kernel/process_64.c +3 -0
@@ -189,7 +189,10 @@ void cpu_idle(void)*/ local_irq_disable ();enter_idle (); +/* Don't trace irqs off for idle */ +stop_critical_timings ();idle (); +start_critical_timings ();/* In many cases the interrupt that ended idle has already called exit_idle. But some idle loops can be woken up without interrupt. */
arch/x86/lib/Makefile +1 -0
@@ -12,6 +12,7 @@ ifeq ($(CONFIG_X86_32),y) lib-y += checksum_32.o lib-y += strstr_32.o lib-y += bitops_32.o semaphore_32.o string_32.o + lib-y += thunk_32.o lib-$(CONFIG_X86_USE_3DNOW) += mmx_32.o else
arch/x86/lib/thunk_32.S new file +47 -0
47 lines not on this page · raw message
arch/x86/lib/thunk_64.S +17 -2
@@ -2,6 +2,7 @@ * Save registers before calling assembly functions. This avoids * disturbance of register allocation in some inline assembly constructs. * Copyright 2001,2002 by Andi Kleen, SuSE Labs. + * Added trace_hardirqs callers - Copyright 2007 Steven Rostedt, Red Hat, Inc. * Subject to the GNU public license, v.2. No warranty of any kind. */
@@ -47,8 +48,22 @@ thunk __up_wakeup,__up #ifdef CONFIG_TRACE_IRQFLAGS - thunk trace_hardirqs_on_thunk,trace_hardirqs_on - thunk trace_hardirqs_off_thunk,trace_hardirqs_off + /* put return address in rdi (arg1) */ + .macro thunk_ra name,func + .globl \name +\name: + CFI_STARTPROC + SAVE_ARGS + /* SAVE_ARGS pushs 9 elements */ + /* the next element would be the rip */ + movq 9*8(%rsp), %rdi + call \func + jmp restore + CFI_ENDPROC + .endm + + thunk_ra trace_hardirqs_on_thunk,trace_hardirqs_on_caller + thunk_ra trace_hardirqs_off_thunk,trace_hardirqs_off_caller #endif #ifdef CONFIG_DEBUG_LOCK_ALLOC
include/asm-x86/irqflags.h +2 -22
@@ -183,8 +183,6 @@ static inline void trace_hardirqs_fixup(* have a reliable stack. x86_64 only. */ #define SWAPGS_UNSAFE_STACK swapgs -#define ARCH_TRACE_IRQS_ON call trace_hardirqs_on_thunk -#define ARCH_TRACE_IRQS_OFF call trace_hardirqs_off_thunk#define ARCH_LOCKDEP_SYS_EXIT call lockdep_sys_exit_thunk#define ARCH_LOCKDEP_SYS_EXIT_IRQ \ TRACE_IRQS_ON; \
@@ -196,24 +194,6 @@ static inline void trace_hardirqs_fixup(TRACE_IRQS_OFF ;#else -#define ARCH_TRACE_IRQS_ON \ - pushl %eax; \ - pushl %ecx; \ - pushl %edx; \ - call trace_hardirqs_on; \ - popl %edx; \ - popl %ecx; \ - popl %eax; - -#define ARCH_TRACE_IRQS_OFF \ - pushl %eax; \ - pushl %ecx; \ - pushl %edx; \ - call trace_hardirqs_off; \ - popl %edx; \ - popl %ecx; \ - popl %eax; -#define ARCH_LOCKDEP_SYS_EXIT \ pushl %eax; \ pushl %ecx; \
@@ -227,8 +207,8 @@ static inline void trace_hardirqs_fixup(#endif #ifdef CONFIG_TRACE_IRQFLAGS -# define TRACE_IRQS_ON ARCH_TRACE_IRQS_ON -# define TRACE_IRQS_OFF ARCH_TRACE_IRQS_OFF +# define TRACE_IRQS_ON call trace_hardirqs_on_thunk; +# define TRACE_IRQS_OFF call trace_hardirqs_off_thunk;#else # define TRACE_IRQS_ON # define TRACE_IRQS_OFF
include/linux/ftrace.h +8 -0
@@ -50,4 +50,12 @@ extern void mcount(void);# define CALLER_ADDR5 0UL#endif +#ifdef CONFIG_IRQSOFF_TRACER + extern void notrace time_hardirqs_on(unsigned long a0, unsigned long a1); + extern void notrace time_hardirqs_off(unsigned long a0, unsigned long a1); +#else +# define time_hardirqs_on (a0, a1) do { } while (0) +# define time_hardirqs_off (a0, a1) do { } while (0) +#endif +#endif /* _LINUX_FTRACE_H */
include/linux/irqflags.h +10 -2
@@ -12,10 +12,10 @@#define _LINUX_TRACE_IRQFLAGS_H#ifdef CONFIG_TRACE_IRQFLAGS -extern void trace_hardirqs_on (void ); -extern void trace_hardirqs_off (void );extern void trace_softirqs_on (unsigned long ip);extern void trace_softirqs_off (unsigned long ip); +extern void trace_hardirqs_on (void ); +extern void trace_hardirqs_off (void );# define trace_hardirq_context (p) ((p)->hardirq_context)# define trace_softirq_context (p) ((p)->softirq_context)# define trace_hardirqs_enabled (p) ((p)->hardirqs_enabled)
@@ -41,6 +41,14 @@# define INIT_TRACE_IRQFLAGS #endif +#ifdef CONFIG_IRQSOFF_TRACER +extern void stop_critical_timings (void ); +extern void start_critical_timings (void ); +#else +# define stop_critical_timings () do { } while (0) +# define start_critical_timings () do { } while (0) +#endif +#ifdef CONFIG_TRACE_IRQFLAGS_SUPPORT #include <asm/irqflags.h>
kernel/fork.c +1 -1
@@ -1041,7 +1041,7 @@ static struct task_struct *copy_process(rt_mutex_init_task (p); -#ifdef CONFIG_TRACE_IRQFLAGS +#if defined(CONFIG_TRACE_IRQFLAGS ) && defined(CONFIG_LOCKDEP )DEBUG_LOCKS_WARN_ON (!p->hardirqs_enabled);DEBUG_LOCKS_WARN_ON (!p->softirqs_enabled);#endif
kernel/lockdep.c +19 -4
@@ -39,6 +39,7 @@#include <linux/irqflags.h> #include <linux/utsname.h> #include <linux/hash.h> +#include <linux/ftrace.h> #include <asm/sections.h>
@@ -978,7 +979,7 @@ check_noncircular(struct lock_class *soureturn 1 ; } -#ifdef CONFIG_TRACE_IRQFLAGS +#if defined(CONFIG_TRACE_IRQFLAGS ) && defined(CONFIG_PROVE_LOCKING )/* * Forwards and backwards subgraph searching, for the purposes of * proving that two subgraphs can be connected by a new dependency
@@ -1676,7 +1677,7 @@ valid_state(struct task_struct *curr, ststatic int mark_lock (struct task_struct *curr,struct held_lock *this,enum lock_usage_bit new_bit); -#ifdef CONFIG_TRACE_IRQFLAGS +#if defined(CONFIG_TRACE_IRQFLAGS ) && defined(CONFIG_PROVE_LOCKING )/* * print irq inversion bug:
@@ -2009,11 +2010,13 @@ void early_boot_irqs_on(void)/* * Hardirqs will be enabled: */ -void trace_hardirqs_on (void ) +void notrace trace_hardirqs_on_caller(unsigned long a0) {struct task_struct *curr = current;unsigned long ip; +time_hardirqs_on (CALLER_ADDR0 , a0); +if (unlikely (!debug_locks || current->lockdep_recursion))return ;
@@ -2051,16 +2054,23 @@ void trace_hardirqs_on(void) curr->hardirq_enable_event = ++curr->irq_events;debug_atomic_inc (&hardirqs_on_events); } +EXPORT_SYMBOL (trace_hardirqs_on_caller); +void notrace trace_hardirqs_on(void) +{ +trace_hardirqs_on_caller (CALLER_ADDR0 ); +}EXPORT_SYMBOL (trace_hardirqs_on);/* * Hardirqs were disabled: */ -void trace_hardirqs_off (void ) +void notrace trace_hardirqs_off_caller(unsigned long a0) {struct task_struct *curr = current; +time_hardirqs_off (CALLER_ADDR0 , a0); +if (unlikely (!debug_locks || current->lockdep_recursion))return ;
@@ -2078,7 +2088,12 @@ void trace_hardirqs_off(void) }else debug_atomic_inc (&redundant_hardirqs_off); } +EXPORT_SYMBOL (trace_hardirqs_off_caller); +void notrace trace_hardirqs_off(void) +{ +trace_hardirqs_off_caller (CALLER_ADDR0 ); +}EXPORT_SYMBOL (trace_hardirqs_off);/*
kernel/printk.c +2 -0
@@ -990,7 +990,9 @@ void release_console_sem(void) _log_end = log_end; con_start = log_end;/* Flush */ spin_unlock (&logbuf_lock); +stop_critical_timings ();/* don't trace print latency */ call_console_drivers (_con_start, _log_end); +start_critical_timings ();local_irq_restore (flags); } console_locked =0 ;
kernel/trace/Kconfig +18 -0
@@ -26,6 +26,24 @@ config FTRACE (the bootup default), then the overhead of the instructions is very small and not measurable even in micro-benchmarks. +config IRQSOFF_TRACER + bool "Interrupts-off Latency Tracer" + default n + depends on TRACE_IRQFLAGS_SUPPORT + depends on GENERIC_TIME + select TRACE_IRQFLAGS + select TRACING + select TRACER_MAX_TRACE + help + This option measures the time spent in irqs-off critical + sections, with microsecond accuracy. + + The default measurement method is a maximum search, which is + disabled by default and can be runtime (re-)started + via: + + echo 0 > /debugfs/tracing/tracing_max_latency + config SCHED_TRACER bool "Scheduling Latency Tracer" depends on DEBUG_KERNEL
kernel/trace/Makefile +1 -0
@@ -3,6 +3,7 @@ obj-$(CONFIG_FTRACE) += libftrace.o obj-$(CONFIG_TRACING) += trace.o obj-$(CONFIG_CONTEXT_SWITCH_TRACER) += trace_sched_switch.o obj-$(CONFIG_FTRACE) += trace_functions.o +obj-$(CONFIG_IRQSOFF_TRACER) += trace_irqsoff.o obj-$(CONFIG_SCHED_TRACER) += trace_sched_wakeup.o libftrace-y := ftrace.o
kernel/trace/trace_irqsoff.c new file +402 -0
402 lines not on this page · raw message