Skip to contentSkip to conversation

dynamic debug - adding ring buffer storage support

RFC PATCH In review Jiri Olsa 6 participants 10 messages Dec 22, 2009 – Jan 5, 2010

Subsystems (3)
lore
Share & tools
lore

Jiri Olsa

author
#0 lore raw

RFC PATCH 1/1 dynamic debug - adding ring buffer storage support

Hi,

as I use dynamic debug sometimes, I thought it could be useful having the possibility to store the output somewhere else than dmesg.

The attached patch implements support for storing dynamic debug messages to the ring buffer.

The dynamic debug allows simple addition of new flags, so I added 'r' flag for ring buffer storage.

I used the ring buffer implementation from trace framework.

hopefuly this could be any use for others as well... plz let me know what you think,

jirka

---
 Documentation/dynamic-debug-howto.txt |   33 ++-
 include/linux/dynamic_debug.h         |   37 ++-
 lib/Kconfig.debug                     |    9 +
 lib/Makefile                          |    1 +
 lib/dynamic_debug.c                   |   15 +
 lib/dynamic_debug_rb.c                |  501 +++++++++++++++++++++++++++++++++
 6 files changed, 588 insertions(+), 8 deletions(-)
Documentation/dynamic-debug-howto.txt +31 -2
index 674c566..aa05764 100644
@@ -208,9 +208,34 @@ The flags are:
 p
     Causes a printk() message to be emitted to dmesg
 
-Note the regexp ^[-+=][scp]+$ matches a flags specification.
+r
+    Causes messages to be emitted to the ring buffer
+    (see 'Ring buffer output' section)
+
+Note the regexp ^[-+=][pr]+$ matches a flags specification.
 Note also that there is no convenient syntax to remove all
-the flags at once, you need to use "-psc".
+the flags at once, you need to use "-pr".
+
+Ring buffer output
+==================
+
+The ring buffer output ('r' flag) is enabled by config option
+CONFIG_DYNAMIC_DEBUG_RB. Having this option set 3 files are
+created under the '<debugfs>/dynamic_debug' directory:
+
+'trace'      - displays the current content of the ring buffer
+
+'trace_pipe' - displays and consumes all the current content of
+               the ring buffer.
+
+'rb_size_kb' - displays/set the ring buffer size in kB
+
+
+The 'trace_pipe' works in a similar manner like in the tracing
+framework. You can use 'cat trace_pipe' as a blocking reader
+for the messages.
+
+The ring buffer output can be enabled together with dmesg output.
 
 Examples
 ========
@@ -238,3 +263,7 @@ nullarbor:~ # echo -n 'func svc_process -p' >
 // enable messages for NFS calls READ, READLINK, READDIR and READDIR+.
 nullarbor:~ # echo -n 'format "nfsd: READ" +p' >
 				<debugfs>/dynamic_debug/control
+
+// enable all compiled messages output to ring buffer and dmesg
+nullarbor:~ # echo -n '+pr' >
+				<debugfs>/dynamic_debug/control
include/linux/dynamic_debug.h +31 -6
index f8c2e17..79701ac 100644
@@ -31,6 +31,7 @@ struct _ddebug {
  	 * writes commands to <debugfs>/dynamic_debug/ddebug
 	 */
 #define _DPRINTK_FLAGS_PRINT   (1<<0)  /* printk() a message using the format */
+#define _DPRINTK_FLAGS_RB      (1<<1)  /* store message to the ring buffer */
 #define _DPRINTK_FLAGS_DEFAULT 0
 	unsigned int flags:8;
 } __attribute__((aligned(8)));
@@ -50,15 +51,32 @@ extern int ddebug_remove_module(char *mod_name);
 					__ret = 1;			     \
 	__ret; })
 
+#ifdef CONFIG_DYNAMIC_DEBUG_RB
+struct dentry;
+extern int dynamic_debug_rb_init(struct dentry *dir);
+extern int dynamic_debug_rb_store(const char *fmt, ...);
+#else
+static inline int dynamic_debug_rb_store(const char *fmt, ...)
+{
+	return 0;
+}
+#endif
+
 #define dynamic_pr_debug(fmt, ...) do {					\
 	static struct _ddebug descriptor				\
 	__used								\
 	__attribute__((section("__verbose"), aligned(8))) =		\
 	{ KBUILD_MODNAME, __func__, __FILE__, fmt, DEBUG_HASH,	\
 		DEBUG_HASH2, __LINE__, _DPRINTK_FLAGS_DEFAULT };	\
-	if (__dynamic_dbg_enabled(descriptor))				\
-		printk(KERN_DEBUG pr_fmt(fmt),	##__VA_ARGS__);		\
-	} while (0)
+	if (__dynamic_dbg_enabled(descriptor)) {			\
+		if (descriptor.flags & _DPRINTK_FLAGS_PRINT)		\
+			printk(KERN_DEBUG pr_fmt(fmt),	##__VA_ARGS__);	\
+		if (descriptor.flags & _DPRINTK_FLAGS_RB)		\
+			dynamic_debug_rb_store(KBUILD_MODNAME ":"	\
+						pr_fmt(fmt),		\
+						##__VA_ARGS__);		\
+	}								\
+} while (0)
 
 
 #define dynamic_dev_dbg(dev, fmt, ...) do {				\
@@ -67,9 +85,16 @@ extern int ddebug_remove_module(char *mod_name);
 	__attribute__((section("__verbose"), aligned(8))) =		\
 	{ KBUILD_MODNAME, __func__, __FILE__, fmt, DEBUG_HASH,	\
 		DEBUG_HASH2, __LINE__, _DPRINTK_FLAGS_DEFAULT };	\
-	if (__dynamic_dbg_enabled(descriptor))				\
-		dev_printk(KERN_DEBUG, dev, fmt, ##__VA_ARGS__);	\
-	} while (0)
+	if (__dynamic_dbg_enabled(descriptor)) {			\
+		if (descriptor.flags & _DPRINTK_FLAGS_PRINT)		\
+			dev_printk(KERN_DEBUG, dev, fmt, ##__VA_ARGS__);\
+		if (descriptor.flags & _DPRINTK_FLAGS_RB)		\
+			dynamic_debug_rb_store("%s %s: " fmt,		\
+					dev_driver_string(dev),		\
+					dev_name(dev),			\
+					##__VA_ARGS__);			\
+	}								\
+} while (0)
 
 #else
 
lib/Kconfig.debug +9 -0
index 9943601..eb0fdee 100644
@@ -1056,6 +1056,15 @@ config DYNAMIC_DEBUG
 
 	  See Documentation/dynamic-debug-howto.txt for additional information.
 
+config DYNAMIC_DEBUG_RB
+	bool "Use ring buffer for output"
+	default n
+	depends on DYNAMIC_DEBUG
+	help
+	  Enables the support for storing DYNAMIC_DEBUG messages into the
+	  ring buffer. The ring_buffer is accesable by 'trace' or 'trace_pipe'
+	  files (see Documentation/dynamic-debug-howto.txt).
+
 config DMA_API_DEBUG
 	bool "Enable debugging of DMA-API usage"
 	depends on HAVE_DMA_API_DEBUG
lib/Makefile +1 -0
index 347ad8d..80ad152 100644
@@ -88,6 +88,7 @@ obj-$(CONFIG_HAVE_LMB) += lmb.o
 obj-$(CONFIG_HAVE_ARCH_TRACEHOOK) += syscall.o
 
 obj-$(CONFIG_DYNAMIC_DEBUG) += dynamic_debug.o
+obj-$(CONFIG_DYNAMIC_DEBUG_RB) += dynamic_debug_rb.o
 
 obj-$(CONFIG_NLATTR) += nlattr.o
 
lib/dynamic_debug.c +15 -0
index f935029..4c322f2 100644
@@ -79,6 +79,10 @@ static char *ddebug_describe_flags(struct _ddebug *dp, char *buf,
 	BUG_ON(maxlen < 4);
 	if (dp->flags & _DPRINTK_FLAGS_PRINT)
 		*p++ = 'p';
+#ifdef CONFIG_DYNAMIC_DEBUG_RB
+	if (dp->flags & _DPRINTK_FLAGS_RB)
+		*p++ = 'r';
+#endif
 	if (p == buf)
 		*p++ = '-';
 	*p = '\0';
@@ -398,6 +402,11 @@ static int ddebug_parse_flags(const char *str, unsigned int *flagsp,
 		case 'p':
 			flags |= _DPRINTK_FLAGS_PRINT;
 			break;
+#ifdef CONFIG_DYNAMIC_DEBUG_RB
+		case 'r':
+			flags |= _DPRINTK_FLAGS_RB;
+			break;
+#endif
 		default:
 			return -EINVAL;
 		}
@@ -758,6 +767,12 @@ static int __init dynamic_debug_init(void)
 		}
 		ret = ddebug_add_module(iter_start, n, modname);
 	}
+
+#ifdef CONFIG_DYNAMIC_DEBUG_RB
+	if (!ret)
+		ret = dynamic_debug_rb_init(dir);
+#endif
+
 out_free:
 	if (ret) {
 		ddebug_remove_all_tables();
lib/dynamic_debug_rb.c new file +501 -0

501 lines not on this page · raw message

#1 lore raw
Quoted 8 lines · Jiri Olsa

Hi,

as I use dynamic debug sometimes, I thought it could be useful having the possibility to store the output somewhere else than dmesg.

The attached patch implements support for storing dynamic debug messages to the ring buffer.

Quoted 7 lines

The dynamic debug allows simple addition of new flags, so I added 'r' flag for ring buffer storage.

I used the ring buffer implementation from trace framework.

hopefuly this could be any use for others as well... plz let me know what you think,

The basic idea sounds good, but doing it all inline is a horrible idea. That will bloat the kernel considerably because all users have to expand this code.

I would suggest to implement an out of line dyndebug_printk() which does all of this out of line.

-Andi

Signature · 1 line
[email protected] -- Speaking for myself only.
author
#2 lore raw
Quoted 18 lines · Andi Kleen
Quoted 8 lines · Jiri Olsa

Hi,

as I use dynamic debug sometimes, I thought it could be useful having the possibility to store the output somewhere else than dmesg.

The attached patch implements support for storing dynamic debug messages to the ring buffer.

Quoted 7 lines

The dynamic debug allows simple addition of new flags, so I added 'r' flag for ring buffer storage.

I used the ring buffer implementation from trace framework.

hopefuly this could be any use for others as well... plz let me know what you think,

Quoted 6 lines

The basic idea sounds good, but doing it all inline is a horrible idea. That will bloat the kernel considerably because all users have to expand this code.

I would suggest to implement an out of line dyndebug_printk() which does all of this out of line.

right, I'll try to squeeze it

thanks,
jirka

Quoted 4 lines

-Andi

Signature · 1 line
[email protected] -- Speaking for myself only.
#3 lore raw
In reply to Jiri Olsa
Quoted 11 lines · Jiri Olsa

Hi,

as I use dynamic debug sometimes, I thought it could be useful having the possibility to store the output somewhere else than dmesg.

The attached patch implements support for storing dynamic debug messages to the ring buffer.

The dynamic debug allows simple addition of new flags, so I added 'r' flag for ring buffer storage.

Quoted 7 lines

I used the ring buffer implementation from trace framework.

hopefuly this could be any use for others as well... plz let me know what you think,

jirka

hi,

I like the idea of using the ring buffer for output. However, I'm not sure why we need a separate one for this. I like adding the 'r' for ring buffer storage, by why not just make this path call 'trace_printk()', and store the string in the existing kernel tracing ring buffer? That way it can interleave with other trace data as well.

thanks,

-Jason

author
#4 lore raw
Quoted 20 lines · Jason Baron
Quoted 11 lines · Jiri Olsa

Hi,

as I use dynamic debug sometimes, I thought it could be useful having the possibility to store the output somewhere else than dmesg.

The attached patch implements support for storing dynamic debug messages to the ring buffer.

The dynamic debug allows simple addition of new flags, so I added 'r' flag for ring buffer storage.

Quoted 7 lines

I used the ring buffer implementation from trace framework.

hopefuly this could be any use for others as well... plz let me know what you think,

jirka

Quoted 7 lines

hi,

I like the idea of using the ring buffer for output. However, I'm not sure why we need a separate one for this. I like adding the 'r' for ring buffer storage, by why not just make this path call 'trace_printk()', and store the string in the existing kernel tracing ring buffer? That way it can interleave with other trace data as well.

that way you need to enable tracing as well... but thats ok I guess :)

I was investigating trace events for this, but did not find a way to put variable length argument inside... and I overlooked the trace_printk, I'll look on it and see how it fits, thanks

also having separate ring buffer makes the 'trace'/'trace_pipe' code really simple (suprissingly) compared to ftrace, and I thought on this place it could last for some time.. ;)

jirka

Quoted 4 lines

thanks,

-Jason

#5 lore raw
Quoted 31 lines · Jiri Olsa
Quoted 20 lines · Jason Baron
Quoted 11 lines · Jiri Olsa

Hi,

as I use dynamic debug sometimes, I thought it could be useful having the possibility to store the output somewhere else than dmesg.

The attached patch implements support for storing dynamic debug messages to the ring buffer.

The dynamic debug allows simple addition of new flags, so I added 'r' flag for ring buffer storage.

Quoted 7 lines

I used the ring buffer implementation from trace framework.

hopefuly this could be any use for others as well... plz let me know what you think,

jirka

Quoted 7 lines

hi,

I like the idea of using the ring buffer for output. However, I'm not sure why we need a separate one for this. I like adding the 'r' for ring buffer storage, by why not just make this path call 'trace_printk()', and store the string in the existing kernel tracing ring buffer? That way it can interleave with other trace data as well.

that way you need to enable tracing as well... but thats ok I guess :)

Quoted 7 lines

I was investigating trace events for this, but did not find a way to put variable length argument inside... and I overlooked the trace_printk, I'll look on it and see how it fits, thanks

also having separate ring buffer makes the 'trace'/'trace_pipe' code really simple (suprissingly) compared to ftrace, and I thought on this place it could last for some time.. ;)

I think what we want is a unified channel of events, of which printk (and dynamic-printk) is one form. I.e. we should add printk events and dynamic-printk events as well, which would show up in /debug/tracing/events/ in a standard ftrace event form and would be accessible to tooling that way.

For printk a single event would be enough i suspect (we dont want a separate event for every printk), and for dynamic-printk we want to map the existing dyn-printk topologies into /debug/tracing/events, to preserve the distinctions and controls available there.

This way in the long run we'd have one unified facility.

Ingo

#6 lore raw
Quoted 15 lines · Ingo Molnar
Quoted 9 lines

that way you need to enable tracing as well... but thats ok I guess :)

I was investigating trace events for this, but did not find a way to put variable length argument inside... and I overlooked the trace_printk, I'll look on it and see how it fits, thanks

also having separate ring buffer makes the 'trace'/'trace_pipe' code really simple (suprissingly) compared to ftrace, and I thought on this place it could last for some time.. ;)

I think what we want is a unified channel of events, of which printk (and dynamic-printk) is one form. I.e. we should add printk events and dynamic-printk events as well, which would show up in /debug/tracing/events/ in a standard ftrace event form and would be accessible to tooling that way.

Quoted 8 lines

For printk a single event would be enough i suspect (we dont want a separate event for every printk), and for dynamic-printk we want to map the existing dyn-printk topologies into /debug/tracing/events, to preserve the distinctions and controls available there.

This way in the long run we'd have one unified facility.

Ingo

That said, I sometimes dream about one event per printk.

Having, say:

/debug/tracing/events/printk/
|
---- kernel/
| |
| ------- time/
| | |
| | ---- clocksource.c
| | |
| | --- clocksource_unstable:218/
| | | |
| | | ---- format
| | | |
| | | ---- filter
| | | |
| | | ---- enable
| | --- [...]
| ------- [...]
|
---- drivers/
| |
| ---- [...]
|
---- [...]

That would give a total control over every printk, trace_printk, etc...

Too bad that would bloat the memory.
Well, that could be wrapped in a single, wildly implemented (understand:
not using TRACE_EVENT macro) trace event, something able to walk through every calls of printk, trace_printk, early_printk, etc... and imitate a per printk event granularity.

But still it needs to be useful...

#7 lore raw
Quoted 31 lines · Frederic Weisbecker

That said, I sometimes dream about one event per printk.

Having, say:

/debug/tracing/events/printk/
|
---- kernel/
| |
| ------- time/
| | |
| | ---- clocksource.c
| | |
| | --- clocksource_unstable:218/
| | | |
| | | ---- format
| | | |
| | | ---- filter
| | | |
| | | ---- enable
| | --- [...]
| ------- [...]
|
---- drivers/
| |
| ---- [...]
|
---- [...]

That would give a total control over every printk, trace_printk, etc...

Quoted 7 lines

Too bad that would bloat the memory.
Well, that could be wrapped in a single, wildly implemented (understand:
not using TRACE_EVENT macro) trace event, something able to walk through every calls of printk, trace_printk, early_printk, etc... and imitate a per printk event granularity.

But still it needs to be useful...

I think we can do the above without bloating memory. Yes we would not need the TRACE_EVENT macro for this. The TRACE_EVENT macro is just for generic tracing, but we could easily come up with something specific for the printk's that will not bloat the kernel as much.

When I get some time, I may try to play with this idea.

-- Steve

#8 lore raw
Quoted 41 lines · Steven Rostedt
Quoted 32 lines

That said, I sometimes dream about one event per printk.

Having, say:

/debug/tracing/events/printk/
|
---- kernel/
| |
| ------- time/
| | |
| | ---- clocksource.c
| | |
| | --- clocksource_unstable:218/
| | | |
| | | ---- format
| | | |
| | | ---- filter
| | | |
| | | ---- enable
| | --- [...]
| ------- [...]
|
---- drivers/
| |
| ---- [...]
|
---- [...]

That would give a total control over every printk, trace_printk, etc...

Quoted 7 lines

Too bad that would bloat the memory.
Well, that could be wrapped in a single, wildly implemented (understand:
not using TRACE_EVENT macro) trace event, something able to walk through every calls of printk, trace_printk, early_printk, etc... and imitate a per printk event granularity.

But still it needs to be useful...

Quoted 9 lines

I think we can do the above without bloating memory. Yes we would not need the TRACE_EVENT macro for this. The TRACE_EVENT macro is just for generic tracing, but we could easily come up with something specific for the printk's that will not bloat the kernel as much.

When I get some time, I may try to play with this idea.

-- Steve

I agree with this direction...in terms of the implementation I was thinking it could be very similar to the tracepoint optimization i've been working on. Where we basically end up with just a 'nop' in place of the printk and then when we enable it we patch the 'nop' with a jump to the proper printk location...

-Jason

#9 lore raw
In reply to Frederic Weisbecker
Quoted 21 lines · Frederic Weisbecker
Quoted 20 lines · Ingo Molnar
Quoted 9 lines

that way you need to enable tracing as well... but thats ok I guess :)

I was investigating trace events for this, but did not find a way to put variable length argument inside... and I overlooked the trace_printk, I'll look on it and see how it fits, thanks

also having separate ring buffer makes the 'trace'/'trace_pipe' code really simple (suprissingly) compared to ftrace, and I thought on this place it could last for some time.. ;)

I think what we want is a unified channel of events, of which printk (and dynamic-printk) is one form. I.e. we should add printk events and dynamic-printk events as well, which would show up in /debug/tracing/events/ in a standard ftrace event form and would be accessible to tooling that way.

For printk a single event would be enough i suspect (we dont want a separate event for every printk), and for dynamic-printk we want to map the existing dyn-printk topologies into /debug/tracing/events, to preserve the distinctions and controls available there.

Quoted 6 lines
Quoted 3 lines

This way in the long run we'd have one unified facility.

Ingo

That said, I sometimes dream about one event per printk.

Yeah - but it's only really useful if we could properly encode/extract the record format as well.

The one person's printk would become another person's programmable tracepoint.

Quoted 1 line

Too bad that would bloat the memory.

Should be optional of course, and then developers/distros pick instrumentation landscape winners/losers. To most people memory overhead is not a big issue, if the result is sufficiently useful.

Ingo