Documention for trace records (trec).
Subsystems (3)
PATCH 1/3 Documention for trace records (trec).
In reply to a message not in the archiveTrec is a light weight tracing mechanism that places trace information into a buffer. The contents of the buffer is dumped when errors occurs or when enabled via SYSRQ commands.
--- Documentation/trec.txt | 87 ++++++++++++++++++++++++++++++++++++++++++++++++ 1 files changed, 87 insertions(+), 0 deletions(-) create mode 100644 Documentation/trec.txt
Documentation/trec.txt new file +87 -0
@@ -0,0 +1,87 @@ +Title : Trace Records +Authors : Wink Saville <[email protected]> + +CONTENTS + +1. Concepts +2. Architectures Supported +3. Configuring +4. API Reference +5. Overhead +6. TODO + + +1. Concepts + +Trace records are a light weight tracing technique that time stamps +small amounts of information and stores them in a buffer. TREC's are +light enough that they may be sprinkled most anywhere in the kernel +and have very little performance impact. + +For instance they can be placed in the scheduler and ISR's to watch +the interaction between ISR's and the scheduler. They can be placed +in memory handling routines to determine how and when memory is +allocated and freed. + +In the current default configuration the trec's are dumped by calling +trec_print_snapshot when die() or panic() are called as well as when +the kernel itself page faults in do_page_fault. + +If CONFIG_MAGIC_SYSRQ is the 'y' command will execute trec_snapshot +and the 'z' command will print the current snapshot. + +A general macro TREC allows trec_write to be invoked as a macro and +TRECC allows it to be invoked conditionally. See include/linux/trec.h +for the current set of macros. + +2. Architectures Supported + +Should support all architectures has been tested only on: + +- X86_64 + + +3. Configuring + +Since trec's are implemented as a device driver they are configured +by enabling support in the "Device Drivers" section of as they could +be used early being a module is not supported. + + +4. API Reference + +Trec supports the following API: + +void trec_init(void): + + Initialize the module, this may be called before the driver is loaded + if it is desired to use trec's early. + +void trec_write(unsigned long pc, int pid, unsigned long v1, unsigned long v2); + + This is the routine used to write into the buffer. pc is the program counter + pid is the process id and v1 and v2 are two parameters. + +void trec_snapshot(void); + + Calling this function takes a snapshot of the current trec buffer so that it + will not be modified. This is called prior to printing the snapshot via + trec_print_snapshot. + +void trec_print_snapshot(void); + + Print the snapshot. + +5. Overhead + +Measured on a 2.4GHZ Core 2 Duo the readings between two TREC's is +270 tics of the rdtsc or about 0.1us. No attempt has been made to +optimize and less information can be collected if the overhead +is still to high. + + +6. TODO + +a. Add code to dump trec to user space +b. Enhance to allow runtime registration and runtime enable disable. +
Signature · 1 line
1.5.0.rc2
PATCH 3/3 Initialize and use trec_snapshot and trec_print_snapshot.
In reply to Wink SavilleTrec's are initialized early in main.c and then dump trec's in die(), panic() and do_page_fault().
--- arch/x86_64/kernel/traps.c | 5 +++++ arch/x86_64/mm/fault.c | 6 ++++++ init/main.c | 4 ++++ kernel/panic.c | 5 +++++ 4 files changed, 20 insertions(+), 0 deletions(-)
arch/x86_64/kernel/traps.c +5 -0
@@ -32,6 +32,7 @@#include <linux/unwind.h> #include <linux/uaccess.h> #include <linux/bug.h> +#include <linux/trec.h> #include <asm/system.h> #include <asm/io.h>
@@ -547,9 +548,13 @@ void die(const char * str, struct pt_regs * regs, long err) {unsigned long flags =oops_begin (); +trec_snapshot (); +if (!user_mode (regs))report_bug (regs->rip); +trec_print_snapshot (); +__die (str, regs, err);oops_end (flags);do_exit (SIGSEGV );
arch/x86_64/mm/fault.c +6 -0
@@ -24,6 +24,7 @@#include <linux/module.h> #include <linux/kprobes.h> #include <linux/uaccess.h> +#include <linux/trec.h> #include <asm/system.h> #include <asm/pgalloc.h>
@@ -535,6 +536,8 @@ no_context: flags =oops_begin (); +trec_snapshot (); +if (address <PAGE_SIZE )printk (KERN_ALERT "Unable to handle kernel NULL pointer dereference" );else
@@ -548,6 +551,9 @@ no_context:__die ("Oops" , regs, error_code);/* Executive summary in case the body of the oops scrolled away */ printk (KERN_EMERG "CR2: %016lx\n" , address); + +trec_print_snapshot (); +oops_end (flags);do_exit (SIGKILL );
init/main.c +4 -0
@@ -54,6 +54,7 @@#include <linux/lockdep.h> #include <linux/pid_namespace.h> #include <linux/device.h> +#include <linux/trec.h> #include <asm/io.h> #include <asm/bugs.h>
@@ -517,6 +518,9 @@ asmlinkage void __init start_kernel(void)early_boot_irqs_off ();early_init_irq_lock_class (); +trec_init (); +TREC0 (); +/* * Interrupts are still disabled. Do necessary setups, then * enable them
kernel/panic.c +5 -0
@@ -19,6 +19,7 @@#include <linux/nmi.h> #include <linux/kexec.h> #include <linux/debug_locks.h> +#include <linux/trec.h> int panic_on_oops;int tainted;
@@ -66,6 +67,8 @@ NORET_TYPE void panic(const char * fmt, ...)unsigned long caller = (unsigned long )__builtin_return_address (0 );#endif +trec_snapshot (); +/* * It's possible to come here directly from a panic-assertion and not * have preempt disabled. Some functions called from here want
@@ -96,6 +99,8 @@ NORET_TYPE void panic(const char * fmt, ...)smp_send_stop ();#endif +trec_print_snapshot (); +atomic_notifier_call_chain (&panic_notifier_list,0 , buf);if (!panic_blink)
Signature · 1 line
1.5.0.rc2
PATCH 2/3 Trec driver.
In reply to Wink SavilleThis is the Trec driver, Makefile, header files.
Enable trec in Kernel hacking configuration menu.
--- drivers/Makefile | 1 + drivers/trec/Makefile | 5 + drivers/trec/trec.c | 404 ++++++++++++++++++++++++++++++++++++++++++++ include/asm-generic/trec.h | 17 ++ include/asm-i386/trec.h | 33 ++++ include/asm-x86_64/trec.h | 13 ++ include/linux/trec.h | 75 ++++++++ lib/Kconfig.debug | 7 + 8 files changed, 555 insertions(+), 0 deletions(-) create mode 100644 drivers/trec/Makefile create mode 100644 drivers/trec/trec.c create mode 100644 include/asm-generic/trec.h create mode 100644 include/asm-i386/trec.h create mode 100644 include/asm-x86_64/trec.h create mode 100644 include/linux/trec.h
drivers/Makefile +1 -0
@@ -81,3 +81,4 @@ obj-$(CONFIG_GENERIC_TIME) += clocksource/ obj-$(CONFIG_DMA_ENGINE) += dma/ obj-$(CONFIG_HID) += hid/ obj-$(CONFIG_PPC_PS3) += ps3/ +obj-$(CONFIG_TREC) += trec/
drivers/trec/Makefile new file +5 -0
@@ -0,0 +1,5 @@ +# +# Makefile for Trace records. +# + +obj-$(CONFIG_TREC) += trec.o
drivers/trec/trec.c new file +404 -0
404 lines not on this page · raw message
include/asm-generic/trec.h new file +17 -0
@@ -0,0 +1,17 @@ +/* +* Copyright (C) 2007 Saville Software, Inc. +* +* This code may be used for any purpose whatsoever, but +* no warranty of any kind is provided. +*/ + +#ifndef _ASM_GENERIC_TREC_H +#define _ASM_GENERIC_TREC_H + +#ifndef __ASSEMBLY__ + +#define rd_tsc () (0) + +#endif /* !__ASSEMBLY__ */ + +#endif /* _ASM_GENERIC_TREC_H */
include/asm-i386/trec.h new file +33 -0
@@ -0,0 +1,33 @@ +/* +* Copyright (C) 2007 Saville Software, Inc. +* +* This code may be used for any purpose whatsoever, but +* no warranty of any kind is provided. +*/ + +#ifndef _ASM_I386_TREC_H +#define _ASM_I386_TREC_H + +#ifndef __ASSEMBLY__ + +#if CONFIG_X86_TSC +#include <asm/msr.h> + +/* +* read x86 time stamp counter. +*/ +static uint64_t inline rd_tsc (void ) +{ +volatile uint64_t value; + +rdtscll (value); + +return value; +} +#else +#define rd_tsc () (0) +#endif /* CONFIG_X86_TSC */ + +#endif /* !__ASSEMBLY__ */ + +#endif /* _ASM_I386_TREC_H */
include/asm-x86_64/trec.h new file +13 -0
@@ -0,0 +1,13 @@ +/* +* Copyright (C) 2007 Saville Software, Inc. +* +* This code may be used for any purpose whatsoever, but +* no warranty of any kind is provided. +*/ + +#ifndef _ASM_X86_64_TREC_H +#define _ASM_X86_64_TREC_H + +#include <asm-i386/trec.h> + +#endif /* _ASM_X86_64_TREC_H */
include/linux/trec.h new file +75 -0
75 lines not on this page · raw message
lib/Kconfig.debug +7 -0
@@ -303,6 +303,13 @@ config STACKTRACE depends on DEBUG_KERNEL depends on STACKTRACE_SUPPORT +config TREC + def_bool n + bool "Trace record support" + depends on DEBUG_KERNEL + help + Trace records are a light weight tracing facility + config DEBUG_KOBJECT bool "kobject debugging" depends on DEBUG_KERNEL
Signature · 1 line
1.5.0.rc2
Re: [PATCH 2/3] Trec driver.
Quoted 12 lines · On Sat, 24 Mar 2007 16:51:38 -0700 Wink Saville wrote:
This is the Trec driver,
Makefile, header files.
Enable trec in Kernel hacking configuration menu.Signed-off-by Wink Saville <[email protected]>---
drivers/trec/trec.c new file +0
-0@@ -0,0 +1,404 @@
...
drivers/trec/trec.c line 27
Quoted 4 lines
drivers/trec/trec.c new file +4
-0+ +struct trec_dev_struct +{ +struct cdev cdev;/* Character device structure */
Fit in 80 columns, please.
Quoted 37 lines
drivers/trec/trec.c new file +37
-0+}; + +MODULE_AUTHOR ("Wink Saville" ); +MODULE_LICENSE ("Dual BSD/GPL" ); + +/* +* Module parameters +*/ +int major =240 ;/* 240 a "local/expermental" device number for the moment */ +int minor =1 ; + +module_param (major, int,S_IRUGO ); +module_param (minor, int,S_IRUGO ); + +/* +* Forward declarations +*/ +static int trec_open (struct inode *inode,struct file *file); +static int trec_release (struct inode *inode,struct file *file); + +/* +* File operations +*/ +struct file_operations trec_f_ops = { + .owner =THIS_MODULE , + .open = trec_open, + .release = trec_release, +}; + +struct trec_struct { +uint64_t tsc; +unsigned long pc; +unsigned long tsk; +unsigned int pid; +unsigned long v1; +unsigned long v2; +};
drivers/trec/trec.c line 74
Quoted 10 lines
drivers/trec/trec.c new file +10
-0+ +/* +* Change trec_buffer_struct.data to be a pointer to a PAGE in the future +*/ +#define TREC_DATA_SIZE 0x200 +struct trec_buffer_struct { +struct trec_buffer_struct * next; +struct trec_struct * cur; +struct trec_struct * end; +struct trec_struct data[TREC_DATA_SIZE ];
Kernel style is:
sturct trec_struct *cur;
drivers/trec/trec.c line 87
Quoted 13 lines
drivers/trec/trec.c new file +13
-0+}; + +/* +* Number of buffers must be a multiple of two so we can +* snapshot the buffers and the minimum should be 4. +*/ +#define TREC_COUNT 2 +struct trec_buffer_struct trec_buffers[2 ][TREC_COUNT ]; +int trec_idx =0 ; +spinlock_t trec_lock =SPIN_LOCK_UNLOCKED ; + +struct trec_buffer_struct * trec_buffer_cur =NULL ; +struct trec_buffer_struct * trec_buffer_snapshot =NULL ;
Pointer style. + don't init to NULL.
drivers/trec/trec.c line 94
Quoted 7 lines
drivers/trec/trec.c new file +7
-0+ +struct trec_dev_struct trec_dev; + +/** +* Print an address symbol if available to the buffer +* this is from traps.c +*/
Don't use kernel-doc indicator ("/**") unless the function block comment is actually in kernel-doc syntax (see Documentation/kernel-doc-nano-HOWTO.txt).
Quoted 29 lines
drivers/trec/trec.c new file +28
-0+static int snprint_address (char *b,int bsize,unsigned long address) +{ +#ifdef CONFIG_KALLSYMS +unsigned long offset =0 , symsize; +const char *symname; +char *modname; +char *delim =":" ; +int n; +char namebuf[128 ]; + + symname =kallsyms_lookup (address, &symsize, &offset, &modname, namebuf); +if (!symname) { + n =0 ; + }else { +if (!modname) + modname = delim ="" ; + n =snprintf (b, bsize,"0x%016lx %s%s%s%s+0x%lx/0x%lx" , + address, delim, modname, delim, symname, offset, symsize); + } +return n; +#else +return snprintf (b, bsize,"0x%016lx" , address); +#endif +} +/* +* Snapshot the current trecs +*/ +void trec_snapshot (void )
drivers/trec/trec.c line 188
Quoted 10 lines
drivers/trec/trec.c new file +10
-0+{ +int flags; + +spin_lock_irqsave (&trec_lock, flags); + trec_buffer_snapshot = trec_buffer_cur; + trec_idx = trec_idx ^1 ; + trec_buffer_cur = &trec_buffers[trec_idx][0 ]; +spin_unlock_irqrestore (&trec_lock, flags); +} +EXPORT_SYMBOL (trec_snapshot);
What (if anything) prevents multiple snapshots from overwriting each other, or from being overwritten while one is printing?
Quoted 52 lines
drivers/trec/trec.c new file +52
-0+/* +* trec_print +*/ +void trec_print_snapshot (void ) +{ +int i; +struct trec_buffer_struct *cur_buffer; + +if (!trec_buffer_snapshot) +trec_snapshot (); + + cur_buffer = trec_buffer_snapshot->next; +for (i =0 ; i <TREC_COUNT ; i++) { +struct trec_struct *cur; + +for (cur = &cur_buffer->data[0 ]; + cur < cur_buffer->cur; + cur +=1 ) { +char b[256 ]; +int n; + + n =snprintf (b,sizeof (b),KERN_ERR "t=%20llu pid=%5u tsk=%p v1=%p v2=%p " , + cur->tsc, cur->pid, (void *)cur->tsk, (void *)cur->v1, (void *)cur->v2); + n +=snprint_address (b+n,sizeof (b)-n, cur->pc); +printk ("%s\n" , b); + } + cur_buffer = cur_buffer->next; + } +} +EXPORT_SYMBOL (trec_print_snapshot); + +/* +* Sysrq support for trec's +*/ +#ifdef CONFIG_MAGIC_SYSRQ +/* +* trec print snapshot handler for sysrq +*/ +static void sysrq_handle_trec_print_snapshot (int key,struct tty_struct *tty) +{ +trec_print_snapshot (); +} + +/* +* trec snapshot handler for sysrq +*/ +static void sysrq_handle_trec_snapshot (int key,struct tty_struct *tty) +{ +trec_snapshot (); +} + +static struct sysrq_key_op sysrq_trec_snapshot_op =
drivers/trec/trec.c line 251
Quoted 10 lines
drivers/trec/trec.c new file +10
-0+{ + .handler = sysrq_handle_trec_snapshot, + .help_msg = "Trec snapshot(Y)", + .action_msg = "Trec snapshot", +}; + +static struct sysrq_key_op sysrq_trec_print_snapshot_op = +{ + .handler = sysrq_handle_trec_print_snapshot, + .help_msg ="Print current trec snapshot" ,
(Z) should be on the help_msg, not on the action_msg.
drivers/trec/trec.c line 256
Quoted 5 lines
drivers/trec/trec.c new file +5
-0+ .action_msg = "Trec print snapshot(Z)", +}; + +/* +* Initilize trec sysrq support
Initialize
Quoted 50 lines
+ */
drivers/trec/trec.c new file +48
-0+static int __init setup_trec_sysrq (void ) +{ +DPK ("setup_trec_sysrq y=trec_snapshot z=trec_print_snapshot\n" ); +register_sysrq_key ('y' , &sysrq_trec_snapshot_op); +register_sysrq_key ('z' , &sysrq_trec_print_snapshot_op); +return 0 ; +} +__initcall (setup_trec_sysrq); +#endif /* CONFIG_MAGIC_SYSRQ */ +/* +* Init routine for the trec device +*/ +static int __init trec_device_init (void ) +{ +int result; +dev_t dev_number =0 ; +static struct class *trec_class; + +DPK ("trec_device_init: E\n" ); + +if (major) { + dev_number =MKDEV (major, minor); + result =register_chrdev_region (dev_number,1 ,"trec" ); +DPK ("trec_device_init: static major result=%d\n" , result); + }else { + result =alloc_chrdev_region (&dev_number, minor,1 ,"trec" ); + major =MAJOR (dev_number); +DPK ("trec_device_init: dynamic major result=%d\n" , result); + } + +if (result <0 ) { +printk (KERN_WARNING "trec: can't get major %d\n" , major); + goto err; + } + +cdev_init (&trec_dev.cdev, &trec_f_ops); + trec_dev.cdev.owner =THIS_MODULE ; + trec_dev.cdev.ops = &trec_f_ops; + + result =cdev_add (&trec_dev.cdev, dev_number,1 ); +if (result) + { +DPK ("trec_device_init: cdev_add failed\n" ); + goto err; + } + +/* +* Make a trec class and create the device
drivers/trec/trec.c line 346
Quoted 10 lines
drivers/trec/trec.c new file +10
-0+*/ + trec_class =class_create (THIS_MODULE ,"trec" ); +class_device_create (trec_class,NULL , dev_number,NULL ,"trec" ); + +/* +* If the trec buffers have not been initialized do so +*/ +if (trec_buffer_cur ==NULL ) { +trec_init (); + }
Style: no braces on one-statement "blocks".
Quoted 26 lines
drivers/trec/trec.c new file +1
-0+}include/asm-i386/trec.h new file +18
-0@@ -0,0 +1,33 @@ +/* +* Copyright (C) 2007 Saville Software, Inc. +* +* This code may be used for any purpose whatsoever, but +* no warranty of any kind is provided. +*/ + +#ifndef _ASM_I386_TREC_H +#define _ASM_I386_TREC_H + +#ifndef __ASSEMBLY__ + +#if CONFIG_X86_TSC +#include <asm/msr.h> + +/* +* read x86 time stamp counter. +*/
include/asm-i386/trec.h line 28
Quoted 10 lines
include/asm-i386/trec.h new file +10
-0+static uint64_t inline rd_tsc (void ) +{ +volatile uint64_t value; + +rdtscll (value); + +return value; +} +#else +#define rd_tsc () (0)
We don't typically indent (#if/) #else block contents like this.
Quoted 11 lines
include/asm-i386/trec.h new file +5
-0+#endif /* CONFIG_X86_TSC */ + +#endif /* !__ASSEMBLY__ */ + +#endif /* _ASM_I386_TREC_H */ lib/Kconfig.debug +0
-0
lib/Kconfig.debug config STACKTRACE · line 311
Quoted 9 lines
lib/Kconfig.debug +6
-0@@ -303,6 +303,13 @@ config STACKTRACE depends on DEBUG_KERNEL depends on STACKTRACE_SUPPORT +config TREC + def_bool n + bool "Trace record support" + depends on DEBUG_KERNEL + help + Trace records are a light weight tracing facility
lightweight ... facility.
lib/Kconfig.debug config STACKTRACE · line 315
Quoted 4 lines
lib/Kconfig.debug +1
-0+ config DEBUG_KOBJECT bool "kobject debugging" depends on DEBUG_KERNEL
---
~Randy
*** Remember to use Documentation/SubmitChecklist when testing your code ***
Quoted 23 lines · On Sat, 24 Mar 2007 16:51:37 -0700 Wink Saville wrote:
Trec is a light weight tracing mechanism that places trace information into a buffer. The contents of the buffer is dumped when errors occurs or when enabled via SYSRQ commands.
Signed-off-by Wink Saville <[email protected]>--- Documentation/trec.txt | 87 ++++++++++++++++++++++++++++++++++++++++++++++++ 1 files changed, 87 insertions(+), 0 deletions(-) create mode 100644 Documentation/trec.txtDocumentation/trec.txt new file +6
-0@@ -0,0 +1,87 @@ +Title : Trace Records +Authors : Wink Saville <[email protected]> + +CONTENTS + +1. Concepts
Documentation/trec.txt line 16
Quoted 10 lines
Documentation/trec.txt new file +10
-0+2. Architectures Supported +3. Configuring +4. API Reference +5. Overhead +6. TODO + + +1. Concepts + +Trace records are a light weight tracing technique that time stamps
lightweight ... timestamps
Documentation/trec.txt line 27
Quoted 11 lines
Documentation/trec.txt new file +11
-0+small amounts of information and stores them in a buffer. TREC's are +light enough that they may be sprinkled most anywhere in the kernel +and have very little performance impact. + +For instance they can be placed in the scheduler and ISR's to watch +the interaction between ISR's and the scheduler. They can be placed +in memory handling routines to determine how and when memory is +allocated and freed. + +In the current default configuration the trec's are dumped by calling +trec_print_snapshot when die() or panic() are called as well as when
is
Documentation/trec.txt line 30
Quoted 3 lines
Documentation/trec.txt new file +3
-0+the kernel itself page faults in do_page_fault. + +If CONFIG_MAGIC_SYSRQ is the 'y' command will execute trec_snapshot
^
enabled,Documentation/trec.txt line 39
Quoted 9 lines
Documentation/trec.txt new file +9
-0+and the 'z' command will print the current snapshot. + +A general macro TREC allows trec_write to be invoked as a macro and +TRECC allows it to be invoked conditionally. See include/linux/trec.h +for the current set of macros. + +2. Architectures Supported + +Should support all architectures has been tested only on:
^
;Documentation/trec.txt line 47
Quoted 8 lines
Documentation/trec.txt new file +8
-0+ +- X86_64 + + +3. Configuring + +Since trec's are implemented as a device driver they are configured +by enabling support in the "Device Drivers" section of as they could
^
??Documentation/trec.txt line 57
Quoted 10 lines
Documentation/trec.txt new file +10
-0+be used early being a module is not supported. + + +4. API Reference + +Trec supports the following API: + +void trec_init(void): + + Initialize the module, this may be called before the driver is loaded
^
; (or maybe :)Documentation/trec.txt line 62
Quoted 5 lines
Documentation/trec.txt new file +5
-0+ if it is desired to use trec's early. + +void trec_write(unsigned long pc, int pid, unsigned long v1, unsigned long v2); + + This is the routine used to write into the buffer. pc is the program counter
end above line with ,
Quoted 8 lines
Documentation/trec.txt new file +8
-0+ pid is the process id and v1 and v2 are two parameters. + +void trec_snapshot(void); + + Calling this function takes a snapshot of the current trec buffer so that it + will not be modified. This is called prior to printing the snapshot via + trec_print_snapshot. +
Documentation/trec.txt line 80
Quoted 10 lines
Documentation/trec.txt new file +10
-0+void trec_print_snapshot(void); + + Print the snapshot. + +5. Overhead + +Measured on a 2.4GHZ Core 2 Duo the readings between two TREC's is +270 tics of the rdtsc or about 0.1us. No attempt has been made to +optimize and less information can be collected if the overhead +is still to high.
too
Documentation/trec.txt line 87
Quoted 7 lines
Documentation/trec.txt new file +7
-0+ + +6. TODO + +a. Add code to dump trec to user space +b. Enhance to allow runtime registration and runtime enable disable. +
---
~Randy
*** Remember to use Documentation/SubmitChecklist when testing your code ***
Quoted 4 lines · Wink Saville
Trec is a light weight tracing mechanism that places trace information into a buffer. The contents of the buffer is dumped when errors occurs or when enabled via SYSRQ commands.
relayfs was supposed to fill this purpose. Why do you want another one?
It doesn't work for sysrq dumping on fatal errors, but in my experience on such cases custom tracing is best.
-Andi