Skip to contentSkip to conversation

dynamic debug: allow printing to trace event

PATCH In review Vincent Whitchurch 3 participants 5 messages Aug 14, 2020 – Aug 25, 2020

Subsystems (4)
lore
Share & tools
lore
#0 lore raw

PATCH v2 1/1 dynamic debug: allow printing to trace event

When debugging device drivers, I've found it very useful to be able to redirect existing pr_debug()/dev_dbg() prints to the trace buffer instead of dmesg. Among the many advantages of the trace buffer is that it can be dynamically resized, allows these prints to combined with other trace events, and doesn't fill up system logs.

Since dynamic debug already has hooks in these call sites, getting these prints into the ftrace buffer is straightforward if we have dynamic debug do it.

Add an "x" flag to make the dynamic debug call site print to a new printk:dynamic trace event. The trace event can be emitted instead of or in addition to the printk().

The print buffer is statically allocated and managed using code borrowed from __ftrace_trace_stack() and is limited to 256 bytes (four of these are allocated per CPU to handle the various contexts); anything larger will be truncated.

---
v2:
- Remove stack buffer and use code similar to __ftrace_trace_stack()
- Use an event with the same class as printk:console

 .../admin-guide/dynamic-debug-howto.rst       |   1 +
 include/linux/dynamic_debug.h                 |   7 +-
 include/trace/events/printk.h                 |  12 +-
 lib/dynamic_debug.c                           | 155 +++++++++++++++---
 4 files changed, 146 insertions(+), 29 deletions(-)
Documentation/admin-guide/dynamic-debug-howto.rst +1 -0
index e5a8def45f3f..d2ebee464db7 100644
@@ -229,6 +229,7 @@ of the characters::
 The flags are::
 
   p    enables the pr_debug() callsite.
+  x    enables trace to the printk:dynamic event
   f    Include the function name in the printed message
   l    Include line number in the printed message
   m    Include module name in the printed message
include/linux/dynamic_debug.h +5 -2
index aa9ff9e1c0b3..f599ed21ecc5 100644
@@ -27,13 +27,16 @@ struct _ddebug {
 	 * writes commands to <debugfs>/dynamic_debug/control
 	 */
 #define _DPRINTK_FLAGS_NONE	0
-#define _DPRINTK_FLAGS_PRINT	(1<<0) /* printk() a message using the format */
+#define _DPRINTK_FLAGS_PRINTK	(1<<0) /* printk() a message using the format */
 #define _DPRINTK_FLAGS_INCL_MODNAME	(1<<1)
 #define _DPRINTK_FLAGS_INCL_FUNCNAME	(1<<2)
 #define _DPRINTK_FLAGS_INCL_LINENO	(1<<3)
 #define _DPRINTK_FLAGS_INCL_TID		(1<<4)
+#define _DPRINTK_FLAGS_TRACE		(1<<5)
+#define _DPRINTK_FLAGS_PRINT		(_DPRINTK_FLAGS_PRINTK | \
+					 _DPRINTK_FLAGS_TRACE)
 #if defined DEBUG
-#define _DPRINTK_FLAGS_DEFAULT _DPRINTK_FLAGS_PRINT
+#define _DPRINTK_FLAGS_DEFAULT _DPRINTK_FLAGS_PRINTK
 #else
 #define _DPRINTK_FLAGS_DEFAULT 0
 #endif
include/trace/events/printk.h +11 -1
index 13d405b2fd8b..6c89121a1669 100644
@@ -7,7 +7,7 @@
 
 #include <linux/tracepoint.h>
 
-TRACE_EVENT(console,
+DECLARE_EVENT_CLASS(printk,
 	TP_PROTO(const char *text, size_t len),
 
 	TP_ARGS(text, len),
@@ -31,6 +31,16 @@ TRACE_EVENT(console,
 
 	TP_printk("%s", __get_str(msg))
 );
+
+DEFINE_EVENT(printk, console,
+	TP_PROTO(const char *text, size_t len),
+	TP_ARGS(text, len)
+);
+
+DEFINE_EVENT(printk, dynamic,
+	TP_PROTO(const char *text, size_t len),
+	TP_ARGS(text, len)
+);
 #endif /* _TRACE_PRINTK_H */
 
 /* This part must be outside protection */
lib/dynamic_debug.c +129 -26

236 lines not on this page · raw message

Signature · 1 line
2.25.1
maintainer Reviewed-by on v2 1/1
#1 lore raw

include/linux/dynamic_debug.h struct _ddebug { · line 30

Quoted 9 lines · On Fri, 14 Aug 2020 15:31:51 +0200 Vincent Whitchurch…

index aa9ff9e1c0b3..f599ed21ecc5 100644

include/linux/dynamic_debug.h +1 -1
@@ -27,13 +27,16 @@ struct _ddebug {
 	 * writes commands to <debugfs>/dynamic_debug/control
 	 */
 #define _DPRINTK_FLAGS_NONE	0
-#define _DPRINTK_FLAGS_PRINT	(1<<0) /* printk() a message using the format */
+#define _DPRINTK_FLAGS_PRINTK	(1<<0) /* printk() a message using the format */

The above looks like a cleanup unrelated to this patch, and probably should be on its own.

Quoted 27 lines
include/linux/dynamic_debug.h +4 -1
 #define _DPRINTK_FLAGS_INCL_MODNAME	(1<<1)
 #define _DPRINTK_FLAGS_INCL_FUNCNAME	(1<<2)
 #define _DPRINTK_FLAGS_INCL_LINENO	(1<<3)
 #define _DPRINTK_FLAGS_INCL_TID		(1<<4)
+#define _DPRINTK_FLAGS_TRACE		(1<<5)	
+#define _DPRINTK_FLAGS_PRINT		(_DPRINTK_FLAGS_PRINTK | \
+					 _DPRINTK_FLAGS_TRACE)
 #if defined DEBUG
-#define _DPRINTK_FLAGS_DEFAULT _DPRINTK_FLAGS_PRINT
+#define _DPRINTK_FLAGS_DEFAULT _DPRINTK_FLAGS_PRINTK
 #else
 #define _DPRINTK_FLAGS_DEFAULT 0
 #endif
include/trace/events/printk.h +1 -1
index 13d405b2fd8b..6c89121a1669 100644
@@ -7,7 +7,7 @@
 
 #include <linux/tracepoint.h>
 
-TRACE_EVENT(console,
+DECLARE_EVENT_CLASS(printk,
 	TP_PROTO(const char *text, size_t len),
 
 	TP_ARGS(text, len),

include/trace/events/printk.h TRACE_EVENT(console, · line 40

Quoted 10 lines
include/trace/events/printk.h +7 -0
@@ -31,6 +31,16 @@ TRACE_EVENT(console,
 
 	TP_printk("%s", __get_str(msg))
 );
+
+DEFINE_EVENT(printk, console,
+	TP_PROTO(const char *text, size_t len),
+	TP_ARGS(text, len)
+);
+
+DEFINE_EVENT(printk, dynamic,

Can we call this "dynamic_printk" or "printk_dynamic", as trace_dynamic() is too generic.

Quoted 19 lines
include/trace/events/printk.h +3 -0
+	TP_PROTO(const char *text, size_t len),
+	TP_ARGS(text, len)
+);
 #endif /* _TRACE_PRINTK_H */
 
 /* This part must be outside protection */
lib/dynamic_debug.c +1 -0
index 1d012e597cc3..76fc3e33fe41 100644
@@ -36,6 +36,7 @@
 #include <linux/sched.h>
 #include <linux/device.h>
 #include <linux/netdevice.h>
+#include <trace/events/printk.h>
 
 #include <rdma/ib_verbs.h>
 

lib/dynamic_debug.c static inline const char *trim_prefix(const char *path) · line 88

Quoted 5 lines
lib/dynamic_debug.c +1 -1
@@ -84,11 +85,12 @@ static inline const char *trim_prefix(const char *path)
 }
 
 static struct { unsigned flag:8; char opt_char; } opt_array[] = {
-	{ _DPRINTK_FLAGS_PRINT, 'p' },
+	{ _DPRINTK_FLAGS_PRINTK, 'p' },

Again, this looks unrelated, and shouldn't be part of this patch.

lib/dynamic_debug.c static inline const char *trim_prefix(const char *path) · line 96

Quoted 8 lines
lib/dynamic_debug.c +1 -0
 	{ _DPRINTK_FLAGS_INCL_MODNAME, 'm' },
 	{ _DPRINTK_FLAGS_INCL_FUNCNAME, 'f' },
 	{ _DPRINTK_FLAGS_INCL_LINENO, 'l' },
 	{ _DPRINTK_FLAGS_INCL_TID, 't' },
+	{ _DPRINTK_FLAGS_TRACE, 'x' },
 	{ _DPRINTK_FLAGS_NONE, '_' },
 };
 

Other then the two comments above, I'm fine with this change.

-- Steve

maintainer
#2 lore raw
Quoted 14 lines · Steven Rostedt

include/linux/dynamic_debug.h struct _ddebug { · line 30

Quoted 9 lines · On Fri, 14 Aug 2020 15:31:51 +0200 Vincent Whitchurch…

index aa9ff9e1c0b3..f599ed21ecc5 100644

include/linux/dynamic_debug.h +1 -1
@@ -27,13 +27,16 @@ struct _ddebug {
 	 * writes commands to <debugfs>/dynamic_debug/control
 	 */
 #define _DPRINTK_FLAGS_NONE	0
-#define _DPRINTK_FLAGS_PRINT	(1<<0) /* printk() a message using the format */
+#define _DPRINTK_FLAGS_PRINTK	(1<<0) /* printk() a message using the format */

The above looks like a cleanup unrelated to this patch, and probably should be on its own.

I read it as we used to have this one thing called 'print', which really meant printk, but now that we also have the ability to output to the trace buffer, what does 'print' mean now? So I read it as being part of this change.

Quoted 8 lines

include/linux/dynamic_debug.h struct _ddebug { · line 37

Quoted 8 lines
include/linux/dynamic_debug.h +3 -0
 #define _DPRINTK_FLAGS_INCL_MODNAME	(1<<1)
 #define _DPRINTK_FLAGS_INCL_FUNCNAME	(1<<2)
 #define _DPRINTK_FLAGS_INCL_LINENO	(1<<3)
 #define _DPRINTK_FLAGS_INCL_TID		(1<<4)
+#define _DPRINTK_FLAGS_TRACE		(1<<5)	
+#define _DPRINTK_FLAGS_PRINT		(_DPRINTK_FLAGS_PRINTK | \
+					 _DPRINTK_FLAGS_TRACE)

Is _DPRINTK_FLAGS_PRINT actually used anywhere? Looks to me like it can be removed.

This is a feature I've wanted for dynamic debug for a while. Thanks for implementing it!

Dynamic can be enabled on the command line in order to print things early in boot (I think ftrace can as well), I want to make sure that there are no ordering issues here? And things wouldn't blow up if we enable printing to the ftrace buffer early on via dyanmic debug?

Thanks,

-Jason

#3 lore raw
Quoted 16 lines · Jason Baron
Quoted 14 lines · Steven Rostedt

include/linux/dynamic_debug.h struct _ddebug { · line 30

Quoted 9 lines · On Fri, 14 Aug 2020 15:31:51 +0200 Vincent Whitchurch…

index aa9ff9e1c0b3..f599ed21ecc5 100644

include/linux/dynamic_debug.h +1 -1
@@ -27,13 +27,16 @@ struct _ddebug {
 	 * writes commands to <debugfs>/dynamic_debug/control
 	 */
 #define _DPRINTK_FLAGS_NONE	0
-#define _DPRINTK_FLAGS_PRINT	(1<<0) /* printk() a message using the format */
+#define _DPRINTK_FLAGS_PRINTK	(1<<0) /* printk() a message using the format */

The above looks like a cleanup unrelated to this patch, and probably should be on its own.

Quoted 3 lines

I read it as we used to have this one thing called 'print', which really meant printk, but now that we also have the ability to output to the trace buffer, what does 'print' mean now? So I read it as being part of this change.

Yes, that's what was intended, but I think it makes sense to split it out as Steven suggested so I've done that now (and also renamed the combined flag to the less ambiguous _DPRINTK_FLAGS_ENABLE).

Quoted 13 lines
Quoted 9 lines

include/linux/dynamic_debug.h struct _ddebug { · line 37

Quoted 9 lines
include/linux/dynamic_debug.h +3 -0
 #define _DPRINTK_FLAGS_INCL_MODNAME	(1<<1)
 #define _DPRINTK_FLAGS_INCL_FUNCNAME	(1<<2)
 #define _DPRINTK_FLAGS_INCL_LINENO	(1<<3)
 #define _DPRINTK_FLAGS_INCL_TID		(1<<4)
+#define _DPRINTK_FLAGS_TRACE		(1<<5)	
+#define _DPRINTK_FLAGS_PRINT		(_DPRINTK_FLAGS_PRINTK | \
+					 _DPRINTK_FLAGS_TRACE)

Is _DPRINTK_FLAGS_PRINT actually used anywhere? Looks to me like it can be removed.

It's used from DYNAMIC_DEBUG_BRANCH() as well as from lib/dynamic_debug.c to check if the location is enabled.

Quoted 7 lines

This is a feature I've wanted for dynamic debug for a while. Thanks for implementing it!

Dynamic can be enabled on the command line in order to print things early in boot (I think ftrace can as well), I want to make sure that there are no ordering issues here? And things wouldn't blow up if we enable printing to the ftrace buffer early on via dyanmic debug?

I tried enabling all dynamic debug locations and tracing via the command line and that worked fine:

dyndbg="file * +x" trace_event=printk:*

#4 lore raw
In reply to Steven Rostedt
Quoted 14 lines · Steven Rostedt

include/linux/dynamic_debug.h struct _ddebug { · line 30

Quoted 9 lines · On Fri, 14 Aug 2020 15:31:51 +0200 Vincent Whitchurch…

index aa9ff9e1c0b3..f599ed21ecc5 100644

include/linux/dynamic_debug.h +1 -1
@@ -27,13 +27,16 @@ struct _ddebug {
 	 * writes commands to <debugfs>/dynamic_debug/control
 	 */
 #define _DPRINTK_FLAGS_NONE	0
-#define _DPRINTK_FLAGS_PRINT	(1<<0) /* printk() a message using the format */
+#define _DPRINTK_FLAGS_PRINTK	(1<<0) /* printk() a message using the format */

The above looks like a cleanup unrelated to this patch, and probably should be on its own.

I've moved this and the other renaming hunk out to a separate patch.

Quoted 28 lines
Quoted 28 lines
include/linux/dynamic_debug.h +4 -1
 #define _DPRINTK_FLAGS_INCL_MODNAME	(1<<1)
 #define _DPRINTK_FLAGS_INCL_FUNCNAME	(1<<2)
 #define _DPRINTK_FLAGS_INCL_LINENO	(1<<3)
 #define _DPRINTK_FLAGS_INCL_TID		(1<<4)
+#define _DPRINTK_FLAGS_TRACE		(1<<5)	
+#define _DPRINTK_FLAGS_PRINT		(_DPRINTK_FLAGS_PRINTK | \
+					 _DPRINTK_FLAGS_TRACE)
 #if defined DEBUG
-#define _DPRINTK_FLAGS_DEFAULT _DPRINTK_FLAGS_PRINT
+#define _DPRINTK_FLAGS_DEFAULT _DPRINTK_FLAGS_PRINTK
 #else
 #define _DPRINTK_FLAGS_DEFAULT 0
 #endif
include/trace/events/printk.h +1 -1
index 13d405b2fd8b..6c89121a1669 100644
@@ -7,7 +7,7 @@
 
 #include <linux/tracepoint.h>
 
-TRACE_EVENT(console,
+DECLARE_EVENT_CLASS(printk,
 	TP_PROTO(const char *text, size_t len),
 
 	TP_ARGS(text, len),
@@ -31,6 +31,16 @@ TRACE_EVENT(console,
 
Quoted 12 lines

include/trace/events/printk.h TRACE_EVENT(console, · line 40

Quoted 9 lines
include/trace/events/printk.h +7 -0
 	TP_printk("%s", __get_str(msg))
 );
+
+DEFINE_EVENT(printk, console,
+	TP_PROTO(const char *text, size_t len),
+	TP_ARGS(text, len)
+);
+
+DEFINE_EVENT(printk, dynamic,

Can we call this "dynamic_printk" or "printk_dynamic", as trace_dynamic() is too generic.

I went for "dyndbg" (printk:dyndbg / trace_dyndbg()) which is the documented name of the infrastructure.