dynamic debug - adding ring buffer storage support
Subsystems (3)
RFC PATCH 1/1 dynamic debug - adding ring buffer storage support
Hi,
as I use dynamic debug sometimes, I thought it could be useful having the possibility to store the output somewhere else than dmesg.
The attached patch implements support for storing dynamic debug messages to the ring buffer.
The dynamic debug allows simple addition of new flags, so I added 'r' flag for ring buffer storage.
I used the ring buffer implementation from trace framework.
hopefuly this could be any use for others as well... plz let me know what you think,
jirka
--- Documentation/dynamic-debug-howto.txt | 33 ++- include/linux/dynamic_debug.h | 37 ++- lib/Kconfig.debug | 9 + lib/Makefile | 1 + lib/dynamic_debug.c | 15 + lib/dynamic_debug_rb.c | 501 +++++++++++++++++++++++++++++++++ 6 files changed, 588 insertions(+), 8 deletions(-)
Documentation/dynamic-debug-howto.txt +31 -2
@@ -208,9 +208,34 @@ The flags are: p Causes a printk() message to be emitted to dmesg -Note the regexp ^[-+=][scp]+$ matches a flags specification. +r + Causes messages to be emitted to the ring buffer + (see 'Ring buffer output' section) + +Note the regexp ^[-+=][pr]+$ matches a flags specification. Note also that there is no convenient syntax to remove all -the flags at once, you need to use "-psc". +the flags at once, you need to use "-pr". + +Ring buffer output +================== + +The ring buffer output ('r' flag) is enabled by config option +CONFIG_DYNAMIC_DEBUG_RB. Having this option set 3 files are +created under the '<debugfs>/dynamic_debug' directory: + +'trace' - displays the current content of the ring buffer + +'trace_pipe' - displays and consumes all the current content of + the ring buffer. + +'rb_size_kb' - displays/set the ring buffer size in kB + + +The 'trace_pipe' works in a similar manner like in the tracing +framework. You can use 'cat trace_pipe' as a blocking reader +for the messages. + +The ring buffer output can be enabled together with dmesg output. Examples ========
@@ -238,3 +263,7 @@ nullarbor:~ # echo -n 'func svc_process -p' > // enable messages for NFS calls READ, READLINK, READDIR and READDIR+. nullarbor:~ # echo -n 'format "nfsd: READ" +p' > <debugfs>/dynamic_debug/control + +// enable all compiled messages output to ring buffer and dmesg +nullarbor:~ # echo -n '+pr' > + <debugfs>/dynamic_debug/control
include/linux/dynamic_debug.h +31 -6
@@ -31,6 +31,7 @@ struct _ddebug {* writes commands to <debugfs>/dynamic_debug/ddebug */ #define _DPRINTK_FLAGS_PRINT (1<<0)/* printk() a message using the format */ +#define _DPRINTK_FLAGS_RB (1<<1)/* store message to the ring buffer */ #define _DPRINTK_FLAGS_DEFAULT 0 unsigned int flags:8; } __attribute__((aligned(8)));
@@ -50,15 +51,32 @@ extern int ddebug_remove_module(char *mod_name); __ret =1 ; \ __ret; }) +#ifdef CONFIG_DYNAMIC_DEBUG_RB +struct dentry ; +extern int dynamic_debug_rb_init (struct dentry *dir); +extern int dynamic_debug_rb_store (const char *fmt, ...); +#else +static inline int dynamic_debug_rb_store (const char *fmt, ...) +{ +return 0 ; +} +#endif +#define dynamic_pr_debug (fmt, ...) do { \ static struct _ddebug descriptor \ __used \ __attribute__((section("__verbose"), aligned(8))) = \ { KBUILD_MODNAME, __func__, __FILE__, fmt, DEBUG_HASH, \ DEBUG_HASH2, __LINE__, _DPRINTK_FLAGS_DEFAULT }; \ - if (__dynamic_dbg_enabled(descriptor)) \ - printk(KERN_DEBUG pr_fmt(fmt), ##__VA_ARGS__); \ - } while (0) + if (__dynamic_dbg_enabled(descriptor)) { \ + if (descriptor.flags & _DPRINTK_FLAGS_PRINT) \ + printk(KERN_DEBUG pr_fmt(fmt), ##__VA_ARGS__); \ + if (descriptor.flags & _DPRINTK_FLAGS_RB) \ + dynamic_debug_rb_store(KBUILD_MODNAME ":" \ + pr_fmt(fmt), \ + ##__VA_ARGS__); \ + } \ +} while (0)#define dynamic_dev_dbg (dev, fmt, ...) do { \
@@ -67,9 +85,16 @@ extern int ddebug_remove_module(char *mod_name); __attribute__((section("__verbose"), aligned(8))) = \ {KBUILD_MODNAME , __func__, __FILE__, fmt,DEBUG_HASH , \ DEBUG_HASH2, __LINE__, _DPRINTK_FLAGS_DEFAULT }; \ -if (__dynamic_dbg_enabled (descriptor)) \ - dev_printk(KERN_DEBUG, dev, fmt, ##__VA_ARGS__); \ - }while (0 ) +if (__dynamic_dbg_enabled (descriptor)) { \ +if (descriptor.flags & _DPRINTK_FLAGS_PRINT) \ + dev_printk(KERN_DEBUG, dev, fmt, ##__VA_ARGS__);\ +if (descriptor.flags & _DPRINTK_FLAGS_RB) \ +dynamic_debug_rb_store ("%s %s: " fmt, \ +dev_driver_string (dev), \ +dev_name (dev), \ + ##__VA_ARGS__); \ + } \ +}while (0 ) #else
lib/Kconfig.debug +9 -0
@@ -1056,6 +1056,15 @@ config DYNAMIC_DEBUG See Documentation/dynamic-debug-howto.txt for additional information. +config DYNAMIC_DEBUG_RB + bool "Use ring buffer for output" + default n + depends on DYNAMIC_DEBUG + help + Enables the support for storing DYNAMIC_DEBUG messages into the + ring buffer. The ring_buffer is accesable by 'trace' or 'trace_pipe' + files (see Documentation/dynamic-debug-howto.txt). + config DMA_API_DEBUG bool "Enable debugging of DMA-API usage" depends on HAVE_DMA_API_DEBUG
lib/Makefile +1 -0
@@ -88,6 +88,7 @@ obj-$(CONFIG_HAVE_LMB) += lmb.o obj-$(CONFIG_HAVE_ARCH_TRACEHOOK) += syscall.o obj-$(CONFIG_DYNAMIC_DEBUG) += dynamic_debug.o +obj-$(CONFIG_DYNAMIC_DEBUG_RB) += dynamic_debug_rb.o obj-$(CONFIG_NLATTR) += nlattr.o
lib/dynamic_debug.c +15 -0
@@ -79,6 +79,10 @@ static char *ddebug_describe_flags(struct _ddebug *dp, char *buf,BUG_ON (maxlen <4 );if (dp->flags & _DPRINTK_FLAGS_PRINT) *p++ ='p' ; +#ifdef CONFIG_DYNAMIC_DEBUG_RB +if (dp->flags & _DPRINTK_FLAGS_RB) + *p++ ='r' ; +#endif if (p == buf) *p++ ='-' ; *p ='\0' ;
@@ -398,6 +402,11 @@ static int ddebug_parse_flags(const char *str, unsigned int *flagsp,case 'p' : flags |= _DPRINTK_FLAGS_PRINT;break ; +#ifdef CONFIG_DYNAMIC_DEBUG_RB +case 'r' : + flags |= _DPRINTK_FLAGS_RB; +break ; +#endif default :return -EINVAL ; }
@@ -758,6 +767,12 @@ static int __init dynamic_debug_init(void) } ret =ddebug_add_module (iter_start, n, modname); } + +#ifdef CONFIG_DYNAMIC_DEBUG_RB +if (!ret) + ret =dynamic_debug_rb_init (dir); +#endif + out_free:if (ret) {ddebug_remove_all_tables ();
lib/dynamic_debug_rb.c new file +501 -0
501 lines not on this page · raw message
Quoted 8 lines · Jiri Olsa
Hi,
as I use dynamic debug sometimes, I thought it could be useful having the possibility to store the output somewhere else than dmesg.
The attached patch implements support for storing dynamic debug messages to the ring buffer.
Quoted 7 lines
The dynamic debug allows simple addition of new flags, so I added 'r' flag for ring buffer storage.
I used the ring buffer implementation from trace framework.
hopefuly this could be any use for others as well... plz let me know what you think,
The basic idea sounds good, but doing it all inline is a horrible idea. That will bloat the kernel considerably because all users have to expand this code.
I would suggest to implement an out of line dyndebug_printk() which does all of this out of line.
-Andi
Signature · 1 line
[email protected] -- Speaking for myself only.
Quoted 18 lines · Andi Kleen
Quoted 8 lines · Jiri Olsa
Hi,
as I use dynamic debug sometimes, I thought it could be useful having the possibility to store the output somewhere else than dmesg.
The attached patch implements support for storing dynamic debug messages to the ring buffer.
Quoted 7 lines
The dynamic debug allows simple addition of new flags, so I added 'r' flag for ring buffer storage.
I used the ring buffer implementation from trace framework.
hopefuly this could be any use for others as well... plz let me know what you think,
Quoted 6 lines
The basic idea sounds good, but doing it all inline is a horrible idea. That will bloat the kernel considerably because all users have to expand this code.
I would suggest to implement an out of line
dyndebug_printk()which does all of this out of line.
right, I'll try to squeeze it
thanks,
jirka
Quoted 4 lines
-Andi
Signature · 1 line
[email protected] -- Speaking for myself only.
Quoted 11 lines · Jiri Olsa
Hi,
as I use dynamic debug sometimes, I thought it could be useful having the possibility to store the output somewhere else than dmesg.
The attached patch implements support for storing dynamic debug messages to the ring buffer.
The dynamic debug allows simple addition of new flags, so I added 'r' flag for ring buffer storage.
Quoted 7 lines
I used the ring buffer implementation from trace framework.
hopefuly this could be any use for others as well... plz let me know what you think,
jirka
hi,
I like the idea of using the ring buffer for output. However, I'm not sure why we need a separate one for this. I like adding the 'r' for ring buffer storage, by why not just make this path call 'trace_printk()', and store the string in the existing kernel tracing ring buffer? That way it can interleave with other trace data as well.
thanks,
-Jason
Quoted 20 lines · Jason Baron
Quoted 11 lines · Jiri Olsa
Hi,
as I use dynamic debug sometimes, I thought it could be useful having the possibility to store the output somewhere else than dmesg.
The attached patch implements support for storing dynamic debug messages to the ring buffer.
The dynamic debug allows simple addition of new flags, so I added 'r' flag for ring buffer storage.
Quoted 7 lines
I used the ring buffer implementation from trace framework.
hopefuly this could be any use for others as well... plz let me know what you think,
jirka
Quoted 7 lines
hi,
I like the idea of using the ring buffer for output. However, I'm not sure why we need a separate one for this. I like adding the 'r' for ring buffer storage, by why not just make this path call '
trace_printk()', and store the string in the existing kernel tracing ring buffer? That way it can interleave with other trace data as well.
that way you need to enable tracing as well... but thats ok I guess :)
I was investigating trace events for this, but did not find a way to put variable length argument inside... and I overlooked the trace_printk, I'll look on it and see how it fits, thanks
also having separate ring buffer makes the 'trace'/'trace_pipe' code really simple (suprissingly) compared to ftrace, and I thought on this place it could last for some time.. ;)
jirka
Quoted 4 lines
thanks,
-Jason
Quoted 31 lines · Jiri Olsa
Quoted 20 lines · Jason Baron
Quoted 11 lines · Jiri Olsa
Hi,
as I use dynamic debug sometimes, I thought it could be useful having the possibility to store the output somewhere else than dmesg.
The attached patch implements support for storing dynamic debug messages to the ring buffer.
The dynamic debug allows simple addition of new flags, so I added 'r' flag for ring buffer storage.
Quoted 7 lines
I used the ring buffer implementation from trace framework.
hopefuly this could be any use for others as well... plz let me know what you think,
jirka
Quoted 7 lines
hi,
I like the idea of using the ring buffer for output. However, I'm not sure why we need a separate one for this. I like adding the 'r' for ring buffer storage, by why not just make this path call '
trace_printk()', and store the string in the existing kernel tracing ring buffer? That way it can interleave with other trace data as well.that way you need to enable tracing as well... but thats ok I guess :)
Quoted 7 lines
I was investigating trace events for this, but did not find a way to put variable length argument inside... and I overlooked the
trace_printk, I'll look on it and see how it fits, thanksalso having separate ring buffer makes the 'trace'/'trace_pipe' code really simple (suprissingly) compared to ftrace, and I thought on this place it could last for some time.. ;)
I think what we want is a unified channel of events, of which printk (and dynamic-printk) is one form. I.e. we should add printk events and dynamic-printk events as well, which would show up in /debug/tracing/events/ in a standard ftrace event form and would be accessible to tooling that way.
For printk a single event would be enough i suspect (we dont want a separate event for every printk), and for dynamic-printk we want to map the existing dyn-printk topologies into /debug/tracing/events, to preserve the distinctions and controls available there.
This way in the long run we'd have one unified facility.
Ingo
Quoted 15 lines · Ingo Molnar
Quoted 9 lines
that way you need to enable tracing as well... but thats ok I guess :)
I was investigating trace events for this, but did not find a way to put variable length argument inside... and I overlooked the
trace_printk, I'll look on it and see how it fits, thanksalso having separate ring buffer makes the 'trace'/'trace_pipe' code really simple (suprissingly) compared to ftrace, and I thought on this place it could last for some time.. ;)
I think what we want is a unified channel of events, of which printk (and dynamic-printk) is one form. I.e. we should add printk events and dynamic-printk events as well, which would show up in /debug/tracing/events/ in a standard ftrace event form and would be accessible to tooling that way.
Quoted 8 lines
For printk a single event would be enough i suspect (we dont want a separate event for every printk), and for dynamic-printk we want to map the existing dyn-printk topologies into /debug/tracing/events, to preserve the distinctions and controls available there.
This way in the long run we'd have one unified facility.
Ingo
That said, I sometimes dream about one event per printk.
Having, say:
/debug/tracing/events/printk/
|
---- kernel/
| |
| ------- time/
| | |
| | ---- clocksource.c
| | |
| | --- clocksource_unstable:218/
| | | |
| | | ---- format
| | | |
| | | ---- filter
| | | |
| | | ---- enable
| | --- [...]
| ------- [...]
|
---- drivers/
| |
| ---- [...]
|
---- [...]
That would give a total control over every printk, trace_printk, etc...
Too bad that would bloat the memory.
Well, that could be wrapped in a single, wildly implemented (understand:
not using TRACE_EVENT macro) trace event, something able to walk through every calls of printk, trace_printk, early_printk, etc... and imitate a per printk event granularity.
But still it needs to be useful...
Quoted 31 lines · Frederic Weisbecker
That said, I sometimes dream about one event per printk.
Having, say:
/debug/tracing/events/printk/
|
----kernel/
| |
| ------- time/
| | |
| | ----clocksource.c
| | |
| | --- clocksource_unstable:218/
| | | |
| | | ---- format
| | | |
| | | ---- filter
| | | |
| | | ---- enable
| | --- [...]
| ------- [...]
|
----drivers/
| |
| ---- [...]
|
---- [...]That would give a total control over every printk,
trace_printk, etc...
Quoted 7 lines
Too bad that would bloat the memory.
Well, that could be wrapped in a single, wildly implemented (understand:
not usingTRACE_EVENTmacro) trace event, something able to walk through every calls of printk,trace_printk,early_printk, etc... and imitate a per printk event granularity.But still it needs to be useful...
I think we can do the above without bloating memory. Yes we would not need the TRACE_EVENT macro for this. The TRACE_EVENT macro is just for generic tracing, but we could easily come up with something specific for the printk's that will not bloat the kernel as much.
When I get some time, I may try to play with this idea.
-- Steve
Quoted 41 lines · Steven Rostedt
Quoted 32 lines
That said, I sometimes dream about one event per printk.
Having, say:
/debug/tracing/events/printk/
|
----kernel/
| |
| ------- time/
| | |
| | ----clocksource.c
| | |
| | --- clocksource_unstable:218/
| | | |
| | | ---- format
| | | |
| | | ---- filter
| | | |
| | | ---- enable
| | --- [...]
| ------- [...]
|
----drivers/
| |
| ---- [...]
|
---- [...]That would give a total control over every printk,
trace_printk, etc...Quoted 7 lines
Too bad that would bloat the memory.
Well, that could be wrapped in a single, wildly implemented (understand:
not usingTRACE_EVENTmacro) trace event, something able to walk through every calls of printk,trace_printk,early_printk, etc... and imitate a per printk event granularity.But still it needs to be useful...
Quoted 9 lines
I think we can do the above without bloating memory. Yes we would not need the
TRACE_EVENTmacro for this. TheTRACE_EVENTmacro is just for generic tracing, but we could easily come up with something specific for the printk's that will not bloat the kernel as much.When I get some time, I may try to play with this idea.
-- Steve
I agree with this direction...in terms of the implementation I was thinking it could be very similar to the tracepoint optimization i've been working on. Where we basically end up with just a 'nop' in place of the printk and then when we enable it we patch the 'nop' with a jump to the proper printk location...
-Jason
Quoted 21 lines · Frederic Weisbecker
Quoted 20 lines · Ingo Molnar
Quoted 9 lines
that way you need to enable tracing as well... but thats ok I guess :)
I was investigating trace events for this, but did not find a way to put variable length argument inside... and I overlooked the
trace_printk, I'll look on it and see how it fits, thanksalso having separate ring buffer makes the 'trace'/'trace_pipe' code really simple (suprissingly) compared to ftrace, and I thought on this place it could last for some time.. ;)
I think what we want is a unified channel of events, of which printk (and dynamic-printk) is one form. I.e. we should add printk events and dynamic-printk events as well, which would show up in /debug/tracing/events/ in a standard ftrace event form and would be accessible to tooling that way.
For printk a single event would be enough i suspect (we dont want a separate event for every printk), and for dynamic-printk we want to map the existing dyn-printk topologies into /debug/tracing/events, to preserve the distinctions and controls available there.
Quoted 6 lines
Quoted 3 lines
This way in the long run we'd have one unified facility.
Ingo
That said, I sometimes dream about one event per printk.
Yeah - but it's only really useful if we could properly encode/extract the record format as well.
The one person's printk would become another person's programmable tracepoint.
Quoted 1 line
Too bad that would bloat the memory.
Should be optional of course, and then developers/distros pick instrumentation landscape winners/losers. To most people memory overhead is not a big issue, if the result is sufficiently useful.
Ingo