tracer: use per-cpu trace buffers
PATCH 1/1 tracer: use per-cpu trace buffers
This patch addresses two issues:
- fixes boot hang with CONFIG_IPIPE_TRACE, !CONFIG_IPIPE_TRACE_VMALLOC, and large CONFIG_NR_CPUS (due to the huge trace buffers this implies, even if the system has only a few CPUs in fact)
- improves NUMA performance by allocating the buffers node-local (hackbench indicates a speed-up of 2 on a 8-way box when the tracer is armed)
Jan
Signature · 2 lines
Siemens AG, Corporate Technology, CT SE 2 Corporate Competence Center Embedded Linux
--- kernel/ipipe/tracer.c | 150 +++++++++++++++++++++++--------------------------- 1 file changed, 71 insertions(+), 79 deletions(-)
kernel/ipipe/tracer.c +71 -79
@@ -2,7 +2,7 @@* kernel/ipipe/tracer.c * * Copyright (C) 2005 Luotao Fu. -* 2005-2007 Jan Kiszka. +* 2005-2008 Jan Kiszka. * * This program is free software; you can redistribute it and/or modify * it under the terms of the GNU General Public License as published by
@@ -64,8 +64,7 @@#define IPIPE_TFLG_CURRENT_DOMAIN (point) \ ((point->flags & IPIPE_TFLG_CURRDOM_MASK) >> IPIPE_TFLG_CURRDOM_SHIFT) - -struct ipipe_trace_point { +struct ipipe_trace_point {short type;short flags;unsigned long eip;
@@ -74,7 +73,7 @@ struct ipipe_trace_point{unsigned long long timestamp; }; -struct ipipe_trace_path { +struct ipipe_trace_path {volatile int flags;int dump_lock;/* separated from flags due to cross-cpu access */ int trace_pos;/* next point to fill */
@@ -100,29 +99,18 @@ enum ipipe_trace_type#define IPIPE_TYPE_MASK 0x0007#define IPIPE_TYPE_BITS 3 -#ifdef CONFIG_IPIPE_TRACE_VMALLOC - -static struct ipipe_trace_path *trace_paths[NR_CPUS ]; - +static DEFINE_PER_CPU(struct ipipe_trace_path *, trace_path);#else /* !CONFIG_IPIPE_TRACE_VMALLOC */ - -static struct ipipe_trace_path trace_paths[NR_CPUS ][IPIPE_TRACE_PATHS ] = - { [0 ...NR_CPUS -1 ] = - { [0 ...IPIPE_TRACE_PATHS -1 ] = - { .begin =-1 , .end =-1 } - } - }; +static DEFINE_PER_CPU(struct ipipe_trace_path, trace_path[IPIPE_TRACE_PATHS]) = + { [0 ...IPIPE_TRACE_PATHS -1 ] = { .begin =-1 , .end =-1 } };#endif /* CONFIG_IPIPE_TRACE_VMALLOC */ int ipipe_trace_enable =0 ; -static int active_path[NR_CPUS ] = - { [0 ...NR_CPUS -1 ] =IPIPE_DEFAULT_ACTIVE }; -static int max_path[NR_CPUS ] = - { [0 ...NR_CPUS -1 ] =IPIPE_DEFAULT_MAX }; -static int frozen_path[NR_CPUS ] = - { [0 ...NR_CPUS -1 ] =IPIPE_DEFAULT_FROZEN }; +static DEFINE_PER_CPU(int, active_path) = { IPIPE_DEFAULT_ACTIVE }; +static DEFINE_PER_CPU(int, max_path) = { IPIPE_DEFAULT_MAX }; +static DEFINE_PER_CPU(int, frozen_path) = { IPIPE_DEFAULT_FROZEN };static IPIPE_DEFINE_SPINLOCK (global_path_lock);static int pre_trace =IPIPE_DEFAULT_PRE_TRACE ;static int post_trace =IPIPE_DEFAULT_POST_TRACE ;
@@ -169,7 +157,7 @@ __ipipe_store_domain_states(struct ipipe } } -static notrace int __ipipe_get_free_trace_path(int old, int cpu_id) +static notrace int __ipipe_get_free_trace_path(int old, int cpu) {int new_active = old;struct ipipe_trace_path *tp;
@@ -177,9 +165,9 @@ static notrace int __ipipe_get_free_tracdo {if (++new_active ==IPIPE_TRACE_PATHS ) new_active =0 ; - tp = &trace_paths[cpu_id][new_active]; - }while ((new_active == max_path[cpu_id]) || - (new_active == frozen_path[cpu_id]) || + tp = &per_cpu (trace_path, cpu)[new_active]; + }while (new_active ==per_cpu (max_path, cpu) || + new_active ==per_cpu (frozen_path, cpu) || tp->dump_lock);return new_active;
@@ -203,30 +191,30 @@ __ipipe_migrate_pre_trace(struct ipipe_t } static notrace struct ipipe_trace_path * -__ipipe_trace_end (int cpu_id,struct ipipe_trace_path *tp,int pos) +__ipipe_trace_end (int cpu,struct ipipe_trace_path *tp,int pos) {struct ipipe_trace_path *old_tp = tp; -long active = active_path[cpu_id]; +long active =per_cpu (active_path, cpu);unsigned long long length;/* do we have a new worst case? */ length = tp->point[tp->end].timestamp - tp->point[tp->begin].timestamp; -if (length > (trace_paths[cpu_id][max_path[cpu_id]]).length) { +if (length >per_cpu (trace_path, cpu)[per_cpu (max_path, cpu)].length) {/* we need protection here against other cpus trying to start a proc dump */ spin_lock (&global_path_lock);/* active path holds new worst case */ tp->length = length; - max_path[cpu_id] = active; +per_cpu (max_path, cpu) = active;/* find next unused trace path */ - active =__ipipe_get_free_trace_path (active, cpu_id); + active =__ipipe_get_free_trace_path (active, cpu);spin_unlock (&global_path_lock); - tp = &trace_paths[cpu_id][active]; + tp = &per_cpu (trace_path, cpu)[active];/* migrate last entries for pre-tracing */ __ipipe_migrate_pre_trace (tp, old_tp, pos);
@@ -236,11 +224,11 @@ __ipipe_trace_end(int cpu_id, struct ipi } static notrace struct ipipe_trace_path * -__ipipe_trace_freeze (int cpu_id,struct ipipe_trace_path *tp,int pos) +__ipipe_trace_freeze (int cpu,struct ipipe_trace_path *tp,int pos) {struct ipipe_trace_path *old_tp = tp; -long active = active_path[cpu_id]; -int i; +long active =per_cpu (active_path, cpu); +int n;/* frozen paths have no core (begin=end) */ tp->begin = tp->end;
@@ -249,21 +237,21 @@ __ipipe_trace_freeze(int cpu_id, struct* to set their frozen path or to start a proc dump */ spin_lock (&global_path_lock); - frozen_path[cpu_id] = active; +per_cpu (frozen_path, cpu) = active;/* find next unused trace path */ - active =__ipipe_get_free_trace_path (active, cpu_id); + active =__ipipe_get_free_trace_path (active, cpu);/* check if this is the first frozen path */ -for (i =0 ; i <NR_CPUS && trace_paths[i] !=NULL ; i++) { -if ((i != cpu_id) && - (trace_paths[i][frozen_path[i]].end >=0 )) +for_each_possible_cpu (n) { +if (n != cpu && +per_cpu (trace_path, n)[per_cpu (frozen_path, n)].end >=0 ) tp->end =-1 ; }spin_unlock (&global_path_lock); - tp = &trace_paths[cpu_id][active]; + tp = &per_cpu (trace_path, cpu)[active];/* migrate last entries for pre-tracing */ __ipipe_migrate_pre_trace (tp, old_tp, pos);
@@ -279,13 +267,13 @@ __ipipe_trace(enum ipipe_trace_type typeint pos, next_pos, begin;struct ipipe_trace_point *point;unsigned long flags; -int cpu_id; +int cpu;local_irq_save_hw_notrace (flags); - cpu_id =ipipe_processor_id (); + cpu =ipipe_processor_id (); restart: - tp = old_tp = &trace_paths[cpu_id][active_path[cpu_id]]; + tp = old_tp = &per_cpu (trace_path, cpu)[per_cpu (active_path, cpu)];/* here starts a race window with NMIs - catched below */
@@ -312,7 +300,8 @@ __ipipe_trace(enum ipipe_trace_type type/* check active_path again - some nasty NMI may have switched * it meanwhile */ -if (unlikely (tp != &trace_paths[cpu_id][active_path[cpu_id]])) { +if (unlikely (tp != + &per_cpu (trace_path, cpu)[per_cpu (active_path, cpu)])) {/* release lock on wrong path and restart */ tp->flags &= ~IPIPE_TFLG_NMI_LOCK ;
@@ -353,7 +342,7 @@ __ipipe_trace(enum ipipe_trace_type typeif ((unlikely (type ==IPIPE_TRACE_FREEZE ) || (unlikely (eip >= trigger_begin && eip <= trigger_end) && type ==IPIPE_TRACE_FUNC )) && - (trace_paths[cpu_id][frozen_path[cpu_id]].begin <0 ) && +per_cpu (trace_path, cpu)[per_cpu (frozen_path, cpu)].begin <0 && !(tp->flags &IPIPE_TFLG_FREEZING )) { tp->post_trace = post_trace +1 ; tp->flags |=IPIPE_TFLG_FREEZING ;
@@ -376,9 +365,9 @@ __ipipe_trace(enum ipipe_trace_type type enforce_end:if (tp->flags &IPIPE_TFLG_FREEZING ) - tp =__ipipe_trace_freeze (cpu_id, tp, pos); + tp =__ipipe_trace_freeze (cpu, tp, pos);else - tp =__ipipe_trace_end (cpu_id, tp, pos); + tp =__ipipe_trace_end (cpu, tp, pos);/* reset the active path, maybe already start a new one */ tp->begin = (type == IPIPE_TRACE_BEGIN) ?
@@ -388,7 +377,7 @@ __ipipe_trace(enum ipipe_trace_type type tp->flags =0 ;/* update active_path not earlier to avoid races with NMIs */ - active_path[cpu_id] = tp - trace_paths[cpu_id]; +per_cpu (active_path, cpu) = tp -per_cpu (trace_path, cpu); }/* we still have old_tp and point,
@@ -412,14 +401,14 @@ __ipipe_trace(enum ipipe_trace_type typestatic unsigned long __ipipe_global_path_lock (void ) {unsigned long flags; -int cpu_id; +int cpu;struct ipipe_trace_path *tp;spin_lock_irqsave (&global_path_lock, flags); - cpu_id =ipipe_processor_id (); + cpu =ipipe_processor_id (); restart: - tp = &trace_paths[cpu_id][active_path[cpu_id]]; + tp = &per_cpu (trace_path, cpu)[per_cpu (active_path, cpu)];/* here is small race window with NMIs - catched below */
@@ -430,7 +419,7 @@ static unsigned long __ipipe_global_path/* check active_path again - some nasty NMI may have switched * it meanwhile */ -if (tp != &trace_paths[cpu_id][active_path[cpu_id]]) { +if (tp != &per_cpu (trace_path, cpu)[per_cpu (active_path, cpu)]) {/* release lock on wrong path and restart */ tp->flags &= ~IPIPE_TFLG_NMI_LOCK ;
@@ -444,14 +433,14 @@ static unsigned long __ipipe_global_pathstatic void __ipipe_global_path_unlock (unsigned long flags) { -int cpu_id; +int cpu;struct ipipe_trace_path *tp;/* release spinlock first - it's not involved in the NMI issue */ __ipipe_spin_unlock_irqbegin (&global_path_lock); - cpu_id =ipipe_processor_id (); - tp = &trace_paths[cpu_id][active_path[cpu_id]]; + cpu =ipipe_processor_id (); + tp = &per_cpu (trace_path, cpu)[per_cpu (active_path, cpu)]; tp->flags &= ~IPIPE_TFLG_NMI_LOCK ;
@@ -513,15 +502,15 @@ EXPORT_SYMBOL(ipipe_trace_pid);int ipipe_trace_max_reset (void ) { -int cpu_id; +int cpu;unsigned long flags;struct ipipe_trace_path *path;int ret =0 ; flags =__ipipe_global_path_lock (); -for (cpu_id =0 ; cpu_id <NR_CPUS && trace_paths[cpu_id] !=NULL ; cpu_id++) { - path = &trace_paths[cpu_id][max_path[cpu_id]]; +for_each_possible_cpu (cpu) { + path = &per_cpu (trace_path, cpu)[per_cpu (max_path, cpu)];if (path->dump_lock) { ret = -EBUSY ;
@@ -542,15 +531,15 @@ EXPORT_SYMBOL(ipipe_trace_max_reset);int ipipe_trace_frozen_reset (void ) { -int cpu_id; +int cpu;unsigned long flags;struct ipipe_trace_path *path;int ret =0 ; flags =__ipipe_global_path_lock (); -for_each_online_cpu (cpu_id) { - path = &trace_paths[cpu_id][frozen_path[cpu_id]]; +for_each_online_cpu (cpu) { + path = &per_cpu (trace_path, cpu)[per_cpu (frozen_path, cpu)];if (path->dump_lock) { ret = -EBUSY ;
@@ -606,7 +595,7 @@ __ipipe_get_task_info(char *task_info, svoid ipipe_trace_panic_freeze (void ) {unsigned long flags; -int cpu_id; +int cpu;if (!ipipe_trace_enable)return ;
@@ -614,9 +603,9 @@ void ipipe_trace_panic_freeze(void) ipipe_trace_enable =0 ;local_irq_save_hw_notrace (flags); - cpu_id =ipipe_processor_id (); + cpu =ipipe_processor_id (); - panic_path = &trace_paths[cpu_id][active_path[cpu_id]]; + panic_path = &per_cpu (trace_path, cpu)[per_cpu (active_path, cpu)];local_irq_restore_hw (flags); }
@@ -889,7 +878,7 @@ static void *__ipipe_max_prtrace_start(smutex_lock (&out_mutex);if (!n) { -struct ipipe_trace_path *path; +struct ipipe_trace_path *tp;unsigned long length_usecs;int points, cpu;unsigned long flags;
@@ -902,10 +891,10 @@ static void *__ipipe_max_prtrace_start(s/* find the longest of all per-cpu paths */ print_path =NULL ;for_each_online_cpu (cpu) { - path = &trace_paths[cpu][max_path[cpu]]; + tp = &per_cpu (trace_path, cpu)[per_cpu (max_path, cpu)];if ((print_path ==NULL ) || - (path->length > print_path->length)) { - print_path = path; + (tp->length > print_path->length)) { + print_path = tp;break ; } }
@@ -1052,7 +1041,7 @@ static void *__ipipe_frozen_prtrace_starmutex_lock (&out_mutex);if (!n) { -struct ipipe_trace_path *path; +struct ipipe_trace_path *tp;int cpu;unsigned long flags;
@@ -1064,9 +1053,9 @@ static void *__ipipe_frozen_prtrace_star/* find the first of all per-cpu frozen paths */ print_path =NULL ;for_each_online_cpu (cpu) { - path = &trace_paths[cpu][frozen_path[cpu]]; -if (path->end >=0 ) { - print_path = path; + tp = &per_cpu (trace_path, cpu)[per_cpu (frozen_path, cpu)]; +if (tp->end >=0 ) { + print_path = tp;break ; } }
@@ -1282,19 +1271,22 @@ void __init __ipipe_init_tracer(void)int cpu, path;for_each_possible_cpu (cpu) { - trace_paths[cpu] =vmalloc ( -sizeof (struct ipipe_trace_path ) *IPIPE_TRACE_PATHS ); -if (trace_paths[cpu] ==NULL ) { +struct ipipe_trace_path *tp_buf; + + tp_buf =vmalloc_node (sizeof (struct ipipe_trace_path ) * +IPIPE_TRACE_PATHS ,cpu_to_node (cpu)); +if (!tp_buf) {printk (KERN_ERR "I-pipe: " "insufficient memory for trace buffer.\n" );return ; } -memset (trace_paths[cpu],0 , -sizeof (struct ipipe_trace_path ) *IPIPE_TRACE_PATHS ); +memset (tp_buf,0 , +sizeof (struct ipipe_trace_path ) *IPIPE_TRACE_PATHS );for (path =0 ; path <IPIPE_TRACE_PATHS ; path++) { - trace_paths[cpu][path].begin =-1 ; - trace_paths[cpu][path].end =-1 ; + tp_buf[path].begin =-1 ; + tp_buf[path].end =-1 ; } +per_cpu (trace_path, cpu) = tp_buf; }#endif /* CONFIG_IPIPE_TRACE_VMALLOC */ ipipe_trace_enable =CONFIG_IPIPE_TRACE_ENABLE_VALUE ;
Quoted 5 lines · Jan Kiszka
This patch addresses two issues:
- fixes boot hang with
CONFIG_IPIPE_TRACE, !CONFIG_IPIPE_TRACE_VMALLOC, and largeCONFIG_NR_CPUS(due to the huge trace buffers this implies, even if the system has only a few CPUs in fact)
This is something I observed even on UP systems: the kernel does not boot on ARM if CONFIG_IPIPE_TRACE_VMALLOC is not selected.
Signature · 1 line
Gilles Chanteperdrix.
Quoted 10 lines · Gilles Chanteperdrix
Quoted 5 lines · Jan Kiszka
This patch addresses two issues:
- fixes boot hang with
CONFIG_IPIPE_TRACE, !CONFIG_IPIPE_TRACE_VMALLOC, and largeCONFIG_NR_CPUS(due to the huge trace buffers this implies, even if the system has only a few CPUs in fact)This is something I observed even on UP systems: the kernel does not boot on ARM if
CONFIG_IPIPE_TRACE_VMALLOCis not selected.
Yeah, well known, that's why we have vmalloc'ed trace buffers.
I just wondered again if this is an arch-specific issue, or if it depends on the RAM size. If it is arch-specific, you are welcome to change the default of IPIPE_TRACE_VMALLOC ("default y if ARM || ..."). Or reduce IPIPE_TRACE_SHIFT for ARM, given there is a reasonable value (/wrt to the tracer) your boards still boot with.
Jan
Quoted 12 lines · Jan Kiszka
Quoted 10 lines · Gilles Chanteperdrix
Quoted 5 lines · Jan Kiszka
This patch addresses two issues:
- fixes boot hang with
CONFIG_IPIPE_TRACE, !CONFIG_IPIPE_TRACE_VMALLOC, and largeCONFIG_NR_CPUS(due to the huge trace buffers this implies, even if the system has only a few CPUs in fact)This is something I observed even on UP systems: the kernel does not boot on ARM if
CONFIG_IPIPE_TRACE_VMALLOCis not selected.
Quoted 7 lines
Yeah, well known, that's why we have vmalloc'ed trace buffers.
I just wondered again if this is an arch-specific issue, or if it depends on the RAM size. If it is arch-specific, you are welcome to change the default of
IPIPE_TRACE_VMALLOC("default y if ARM || ..."). Or reduceIPIPE_TRACE_SHIFTfor ARM, given there is a reasonable value (/wrt to the tracer) your boards still boot with.
I will do some tests and propose something like that. I had this problem with linux 2.6.15 where the initial image (I guess text + data + bss) must fit in 3 MB. I have not tested since then.
Signature · 1 line
Gilles Chanteperdrix.