pid and current-domain tracing
PATCH 1/1 pid and current-domain tracing
Hi,
here are three patches, two enhancing the ipipe tracer, the third propagating the new features to Xenomai.
The tracer gains support for recording a Linux pid + a priority value. The priority can be set to an arbitrary value (12 bit, signed), but will typically be related to the pid. When the tracer's verbose mode is active, it tries to resolve the pid to a task_struct and prints the first chars of the command line followed by the priority in the related lines.
This feature is applied to Xenomai for tracing RT-task switches and wakeups. Kind of poor man's LTT, but with the advantage of per-function tracing. I already used this while trying to understand my prio-inversion issue.
Finally, the tracer is enhanced by a facility to track the currently active ipipe domain. The field for displaying domain stall flags is reused for this.
Some example from my Pentium-133 box:
Quoted 36 lines
I-pipe frozen back-tracing service on 2.6.16.16/ipipe-1.3-05
------------------------------------------------------------
Freeze: 6070092534285 cycles, Trace Points: 1000 (+10)+----- Hard IRQs ('|': locked) |+---- <unused> ||+--- <unused> |||+-- Xenomai ||||+- Linux ('*': domain stalled, '+': current, '#': current+stalled) ||||| +---------- Delay flag ('+': > 1 us, '!': > 10 us) ||||| | +- NMI noise ('N') ||||| | | Type User Val. Time Delay Function (Parent) : +func -3221+ 2.105 n_tty_set_room+0x8 (n_tty_receive_buf+0xe4b) : +func -3219+ 2.060 kill_fasync+0x9 (n_tty_receive_buf+0xe78) : +func -3217+ 1.458 __wake_up+0xe (n_tty_receive_buf+0xe97) : +func -3216+ 2.421 __ipipe_test_and_stall_root+0x8 (__wake_up+0x1a) : #func -3213+ 2.135 __wake_up_common+0xe (__wake_up+0x50) : #func -3211+ 6.090 default_wake_function+0x8 (__wake_up_common+0x33) :| #func -3205+ 2.676 __ipipe_handle_irq+0xe (common_interrupt+0x18) :| #func -3203+ 2.360 __ipipe_ack_common_irq+0xa (__ipipe_handle_irq+0x80) :| #func -3200+ 3.428 ipipe_test_and_stall_pipeline_from+0x8 (__ipipe_ack_common_irq+0x16) :| #func -3197+ 4.511 mask_and_ack_8259A+0xb (__ipipe_ack_common_irq+0x3f) :| #func -3192+ 2.571 __ipipe_dispatch_wired+0xe (__ipipe_handle_irq+0x8a) :| #*func -3190+ 2.766 xnintr_irq_handler+0xb (__ipipe_dispatch_wired+0x7d) :| #*func -3187+ 4.902 rt_irqbench_task_irq+0xa [xeno_irqbench] (xnintr_irq_handler+0x24) :| #*func -3182+ 3.451 rtdm_event_signal+0x9 (rt_irqbench_task_irq+0x46 [xeno_irqbench]) :| #*func -3179+ 3.796 xnsynch_flush+0xe (rtdm_event_signal+0x2d) :| #*func -3175+ 3.263 xnpod_resume_thread+0xe (xnsynch_flush+0x76) :| #*[ 922] irqloop 99 -3171+ 4.375 xnpod_resume_thread+0x48 (xnsynch_flush+0x76) :| #*func -3167+ 2.992 xnpod_schedule+0xe (rtdm_event_signal+0x36) :| #*func -3164+ 2.511 rthal_irq_end+0x8 (xnintr_irq_handler+0x41) :| #*func -3162+ 3.969 enable_8259A_irq+0x9 (rthal_irq_end+0x2e) :| #*func -3158+ 2.511 xnpod_schedule+0xe (xnintr_irq_handler+0x5f) :| #*[ 0] swapper 0 -3155+ 8.669 xnpod_schedule+0x82 (xnintr_irq_handler+0x5f) :| #*func -3146+ 7.766 __switch_to+0xe (xnpod_schedule+0x5e8)
Quoted 8 lines
:| #*[ 922] irqloop 99 -3139+ 6.992 xnpod_schedule+0x6bc (xnpod_suspend_thread+0xed) :| #*func -3132! 18.015 __ipipe_restore_pipeline_head+0x8 (rtdm_event_timedwait+0xea) : +*func -3114+ 2.774 __ipipe_syscall_root+0x9 (system_call+0x20) : +*func -3111+ 2.323 __ipipe_dispatch_event+0xe (__ipipe_syscall_root+0x55) : +*func -3109+ 4.676 hisyscall_event+0xe (__ipipe_dispatch_event+0x5e) : +*func -3104+ 2.135 sys_rtdm_ioctl+0x8 (hisyscall_event+0x140) : +*func -3102+ 2.157 _rtdm_ioctl+0xe (sys_rtdm_ioctl+0x16) : +*func -3100+ 2.052 rtdm_context_get+0x9 (_rtdm_ioctl+0x1f)
Jan
--- include/linux/ipipe_trace.h | 3 - kernel/ipipe/tracer.c | 129 ++++++++++++++++++++++++++++++++++---------- 2 files changed, 102 insertions(+), 30 deletions(-)
include/linux/ipipe_trace.h +2 -1
@@ -2,7 +2,7 @@* include/linux/ipipe_trace.h * * Copyright (C) 2005 Luotao Fu. -* 2005 Jan Kiszka. +* 2005, 2006 Jan Kiszka. * * This program is free software; you can redistribute it and/or modify * it under the terms of the GNU General Public License as published by
@@ -27,6 +27,7 @@ void ipipe_trace_begin(unsigned long v);void ipipe_trace_end (unsigned long v);void ipipe_trace_freeze (unsigned long v);void ipipe_trace_special (unsigned char special_id,unsigned long v); +void ipipe_trace_pid (pid_t pid,short prio);int ipipe_trace_max_reset (void );int ipipe_trace_frozen_reset (void );
kernel/ipipe/tracer.c +100 -29
205 lines not on this page · raw message
--- include/asm-generic/hal.h | 7 +++++++ include/asm-generic/system.h | 1 + include/asm-sim/system.h | 1 + include/asm-uvm/system.h | 1 + include/nucleus/trace.h | 1 + ksrc/nucleus/pod.c | 24 ++++++++++++++++++++++++ ksrc/nucleus/shadow.c | 9 +++++++-- 7 files changed, 42 insertions(+), 2 deletions(-)
include/asm-generic/hal.h +7 -0
@@ -645,6 +645,12 @@ static inline int rthal_trace_special_u6return 0 ; } +static inline int rthal_trace_pid (pid_t pid,short prio) +{ +ipipe_trace_pid (pid, prio); +return 0 ; +} +static inline int rthal_trace_panic_freeze (void ) {ipipe_trace_panic_freeze ();
@@ -667,6 +673,7 @@ static inline int rthal_trace_panic_dump#define rthal_trace_user_freeze (v, once) ({-ENOSYS;})#define rthal_trace_special (id, v) ({-ENOSYS;})#define rthal_trace_special_u64 (id, v) ({-ENOSYS;}) +#define rthal_trace_pid (pid, prio) ({-ENOSYS;})#define rthal_trace_panic_freeze () ({-ENOSYS;})#define rthal_trace_panic_dump () ({-ENOSYS;})
include/asm-generic/system.h +1 -0
@@ -50,6 +50,7 @@#define xnarch_trace_user_freeze (v, once) rthal_trace_user_freeze(v, once)#define xnarch_trace_special (id, v) rthal_trace_special(id, v)#define xnarch_trace_special_u64 (id, v) rthal_trace_special_u64(id, v) +#define xnarch_trace_pid (pid, prio) rthal_trace_pid(pid, prio)#define xnarch_trace_panic_freeze () rthal_trace_panic_freeze()#define xnarch_trace_panic_dump () rthal_trace_panic_dump()
include/asm-sim/system.h +1 -0
@@ -772,6 +772,7 @@ while(0)#define xnarch_trace_user_freeze (v, once) ({-ENOSYS;})#define xnarch_trace_special (id, v) ({-ENOSYS;})#define xnarch_trace_special_u64 (id, v) ({-ENOSYS;}) +#define xnarch_trace_pid (pid, prio) ({-ENOSYS;})#define xnarch_trace_panic_freeze () ({-ENOSYS;})#define xnarch_trace_panic_dump () ({-ENOSYS;})
include/asm-uvm/system.h +1 -0
@@ -794,6 +794,7 @@ static inline void xnarch_sysfree (void#define xnarch_trace_user_freeze (v, once) ({-ENOSYS;})#define xnarch_trace_special (id, v) ({-ENOSYS;})#define xnarch_trace_special_u64 (id, v) ({-ENOSYS;}) +#define xnarch_trace_pid (pid, prio) ({-ENOSYS;})#define xnarch_trace_panic_freeze () ({-ENOSYS;})#define xnarch_trace_panic_dump () ({-ENOSYS;})
ksrc/nucleus/pod.c +24 -0
@@ -673,6 +673,10 @@ static inline void xnpod_switch_zombie(x xnthread_archtcb(threadin));#if defined(__KERNEL__) && defined(CONFIG_XENO_OPT_PERVASIVE ) +xnarch_trace_pid (xnthread_user_task (threadin) ? +xnarch_user_pid (xnthread_archtcb (threadin)) :-1 , +xnthread_current_priority (threadin)); +if (shadow)/* Reap the user-space mate of a deleted real-time shadow. The Linux task has resumed into the Linux domain at the
@@ -1534,6 +1538,9 @@ void xnpod_resume_thread(xnthread_t *thrxnlock_get_irqsave (&nklock, s);xnltt_log_event (xeno_ev_thresume, thread->name, mask); +xnarch_trace_pid (xnthread_user_task (thread) ? +xnarch_user_pid (xnthread_archtcb (thread)) :-1 , +xnthread_current_priority (thread)); sched = thread->sched;
@@ -2114,6 +2121,8 @@ void xnpod_welcome_thread(xnthread_t *thxnltt_log_event (xeno_ev_thrboot, thread->name); +xnarch_trace_pid (-1 ,xnthread_current_priority (thread)); +if (xnthread_signaled_p (thread))xnpod_dispatch_signals ();
@@ -2315,6 +2324,10 @@ void xnpod_schedule(void) sched =xnpod_current_sched (); runthread = sched->runthread; +xnarch_trace_pid (xnthread_user_task (runthread) ? +xnarch_user_pid (xnthread_archtcb (runthread)) :-1 , +xnthread_current_priority (runthread)); +#if defined(CONFIG_SMP ) || defined(CONFIG_XENO_OPT_DEBUG ) need_resched =xnsched_tst_resched (sched);#endif
@@ -2426,6 +2439,10 @@ void xnpod_schedule(void)* relaxed/hardened transitions. */ runthread = sched->runthread; +xnarch_trace_pid (xnthread_user_task (runthread) ? +xnarch_user_pid (xnthread_archtcb (runthread)) :-1 , +xnthread_current_priority (runthread)); +#if defined(__KERNEL__) && defined(CONFIG_XENO_OPT_PERVASIVE )/* Test whether we are relaxing a thread. In such a case, we are here the epilogue of Linux' schedule, and should skip xnpod_schedule epilogue. */
@@ -2498,6 +2515,9 @@ void xnpod_schedule_runnable(xnthread_txnthread_t *runthread = sched->runthread, *threadin;xnltt_log_event (xeno_ev_fastsched); +xnarch_trace_pid (xnthread_user_task (runthread) ? +xnarch_user_pid (xnthread_archtcb (runthread)) :-1 , +xnthread_current_priority (runthread));if (thread != runthread) {sched_removepq (&sched->readyq, &thread->rlink);
@@ -2582,6 +2602,10 @@ void xnpod_schedule_runnable(xnthread_txnarch_switch_to (xnthread_archtcb (runthread),xnthread_archtcb (threadin)); +xnarch_trace_pid (xnthread_user_task (runthread) ? +xnarch_user_pid (xnthread_archtcb (runthread)) :-1 , +xnthread_current_priority (runthread)); +#ifdef CONFIG_SMP /* If runthread migrated while suspended, sched is no longer correct. */ sched =xnpod_current_sched ();
include/nucleus/trace.h +1 -0
@@ -43,6 +43,7 @@#define xntrace_user_freeze (v, once) xnarch_trace_user_freeze(v, once)#define xntrace_special (id, v) xnarch_trace_special(id, v)#define xntrace_special_u64 (id, v) xnarch_trace_special_u64(id, v) +#define xntrace_pid (pid, prio) xnarch_trace_pid(pid, prio)#define xntrace_panic_freeze () xnarch_trace_panic_freeze()#define xntrace_panic_dump () xnarch_trace_panic_dump()
ksrc/nucleus/shadow.c +7 -2
@@ -794,7 +794,7 @@ int xnshadow_map(xnthread_t *thread, xnc {xnarch_cpumask_t affinity;unsigned muxid, magic; -int mode, prio; +int mode, prio, err;/* Increment the interface reference count. */ magic =xnthread_get_magic (thread);
@@ -849,7 +849,12 @@ int xnshadow_map(xnthread_t *thread, xnc mode = thread->rrperiod !=XN_INFINITE ?XNRRB :0 ;xnpod_start_thread (thread, mode,0 , affinity,NULL ,NULL ); -return xnshadow_harden (); + err =xnshadow_harden (); + +xnarch_trace_pid (xnarch_user_pid (xnthread_archtcb (thread)), +xnthread_current_priority (thread)); + +return err; }void xnshadow_unmap (xnthread_t *thread)
--- kernel/ipipe/tracer.c | 34 +++++++++++++++++++++++++--------- 1 file changed, 25 insertions(+), 9 deletions(-)
kernel/ipipe/tracer.c +25 -9
@@ -52,7 +52,15 @@#define IPIPE_TFLG_HWIRQ_OFF 0x0100#define IPIPE_TFLG_FREEZING 0x0200 +#define IPIPE_TFLG_CURRDOM_SHIFT 10/* bits 10..11: current domain */ +#define IPIPE_TFLG_CURRDOM_MASK 0x0C00#define IPIPE_TFLG_DOMSTATE_SHIFT 12/* bits 12..15: domain stalled? */ +#define IPIPE_TFLG_DOMSTATE_BITS 3 + +#define IPIPE_TFLG_DOMAIN_STALLED (point, n) \ + (point->flags & (1 << (n + IPIPE_TFLG_DOMSTATE_SHIFT))) +#define IPIPE_TFLG_CURRENT_DOMAIN (point) \ + ((point->flags & IPIPE_TFLG_CURRDOM_MASK) >> IPIPE_TFLG_CURRDOM_SHIFT)struct ipipe_trace_point {
@@ -137,17 +145,20 @@ static void __ipipe_print_symname(struct static notrace void__ipipe_store_domain_states (struct ipipe_trace_point *point,int cpu_id) { -int i =IPIPE_TFLG_DOMSTATE_SHIFT ;struct list_head *pos; +int i =0 ;list_for_each_prev (pos, &__ipipe_pipeline) {struct ipipe_domain *ipd = list_entry(pos, struct ipipe_domain, p_link);if (test_bit (IPIPE_STALL_FLAG , &ipd->cpudata[cpu_id].status)) - point->flags |= (1 << i); + point->flags |=1 << (i +IPIPE_TFLG_DOMSTATE_SHIFT ); + +if (ipd == ipipe_percpu_domain[cpu_id]) + point->flags |= i <<IPIPE_TFLG_CURRDOM_SHIFT ; -if (++i >IPIPE_TFLG_DOMSTATE_SHIFT +3 ) +if (++i >IPIPE_TFLG_DOMSTATE_BITS )break ; } }
@@ -720,11 +731,15 @@ __ipipe_print_pathmark(struct seq_file *seq_printf (m,"%c%c" , mark, (point->flags &IPIPE_TFLG_HWIRQ_OFF ) ?'|' :' ' ); -if (verbose_trace) -for (i =IPIPE_TFLG_DOMSTATE_SHIFT +3 ; - i >=IPIPE_TFLG_DOMSTATE_SHIFT ; i--) -seq_printf (m,"%c" , - (point->flags & (1 << i)) ?'*' :' ' ); +if (!verbose_trace) +return ; + +for (i =IPIPE_TFLG_DOMSTATE_BITS ; i >=0 ; i--) +seq_printf (m,"%c" , + (IPIPE_TFLG_CURRENT_DOMAIN (point) == i) ? + (IPIPE_TFLG_DOMAIN_STALLED (point, i) ? +'#' :'+' ) : + (IPIPE_TFLG_DOMAIN_STALLED (point, i) ?'*' :' ' )); } static void
@@ -836,7 +851,8 @@ static void __ipipe_print_headline(struc" Type User Val. Time Delay Function " "(Parent)\n" , name[3 ], name[2 ], name[1 ], name[0 ], - name[0 ] ?" ('*': domain stalled)" :"" , + name[0 ] ?" ('*': domain stalled, '+': current, " +"'#': current+stalled)" :"" , IPIPE_DELAY_NOTE/1000, IPIPE_DELAY_WARN/1000); }else seq_printf(m,
Grmbl, off-by-one bug. That buffer requires 12 bytes. Add-on patch to fix this:
kernel/ipipe/tracer.c +1 -1
@@ -612,7 +612,7 @@ void ipipe_trace_panic_dump(void) {int cnt = back_trace;int start, pos; -char task_info[11 ]; +char task_info[12 ];printk ("I-pipe tracer log (%d points):\n" , cnt);