drm: Funnel drm logs to tracepoints
Subsystems (3)
PATCH v3 1/1 drm: Funnel drm logs to tracepoints
From: Sean Paul <[email protected]>
For a long while now, we (ChromeOS) have been struggling getting any value out of user feedback reports of display failures (notably external displays not working). The problem is that all logging, even fatal errors (well, fatal in the sense that a display won't light up) are logged at DEBUG log level. So in order to extract these logs, users need to be able to turn on logging, and reproduce the issue with debug enabled. Unfortunately, this isn't really something we can ask CrOS users to do. I spoke with airlied about this and RHEL has similar issues. After a few more people piped up on previous versions of this patch, it is a Real Issue.
So why don't we just enable DRM_UT_BLAH? Here are the reasons in ascending order of severity:
1- People aren't consistent with their categories, so we'd have to
enable a bunch to get proper coverage
2- We don't want to overwhelm syslog with drm spam, others have to use
it too
3- Console logging is slow
So what we really want is a ringbuffer of the most recent logs (filtered by categories we're interested in) exposed via debugfs so the logs can be extracted when users file feedback.
It just so happens that there is something which does _exactly_ this! This patch dumps drm logs into tracepoints, which allows us to turn tracing on and off depending on which category is useful, and pull them from tracefs on demand.
What about trace_printk()? It doesn't give us the control we get from using tracepoints and it's not meant to be left sprinkled around in code.
Changes in v2:
- Went with a completely different approach: https://lists.freedesktop.org/archives/dri-devel/2019-November/243230.html
Changes in v3:
- Changed commit message to be a bit less RFC-y
- Make class_drm_category_log an actual trace class
---
Even though we don't want it to be, this is UAPI. So here's some userspace code which uses it:
https://chromium-review.googlesource.com/c/chromiumos/platform2/+/1965562
drivers/gpu/drm/drm_print.c | 143 ++++++++++++++++++++++++++----- include/trace/events/drm_print.h | 116 +++++++++++++++++++++++++ 2 files changed, 239 insertions(+), 20 deletions(-) create mode 100644 include/trace/events/drm_print.h
drivers/gpu/drm/drm_print.c +123 -20
@@ -27,11 +27,15 @@#include <stdarg.h> +#include <linux/bitops.h> #include <linux/io.h> #include <linux/moduleparam.h> #include <linux/seq_file.h> #include <linux/slab.h> +#define CREATE_TRACE_POINTS +#include <trace/events/drm_print.h> +#include <drm/drm.h> #include <drm/drm_drv.h> #include <drm/drm_print.h>
@@ -241,10 +245,10 @@ void drm_dev_printk(const struct device *dev, const char *level,struct va_format vaf;va_list args; -va_start (args, format); vaf.fmt = format; vaf.va = &args; +va_start (args, format);if (dev)dev_printk (level, dev,"[" DRM_NAME ":%ps] %pV" ,__builtin_return_address (0 ), &vaf);
@@ -253,49 +257,145 @@ void drm_dev_printk(const struct device *dev, const char *level, level, __builtin_return_address(0), &vaf);va_end (args); + +va_start (args, format); +trace_drm_log (level, dev, &vaf); +va_end (args); }EXPORT_SYMBOL (drm_dev_printk); +static unsigned int drm_trace_enabled (unsigned int category) +{ +unsigned int bit; + +for_each_set_bit (bit, (unsigned long *)&category,sizeof (category) *8 ) { +switch (BIT (bit)) { +case DRM_UT_NONE : +return trace_drm_dbg_none_enabled (); +case DRM_UT_CORE : +return trace_drm_dbg_core_enabled (); +case DRM_UT_DRIVER : +return trace_drm_dbg_driver_enabled (); +case DRM_UT_KMS : +return trace_drm_dbg_kms_enabled (); +case DRM_UT_PRIME : +return trace_drm_dbg_prime_enabled (); +case DRM_UT_ATOMIC : +return trace_drm_dbg_atomic_enabled (); +case DRM_UT_VBL : +return trace_drm_dbg_vbl_enabled (); +case DRM_UT_STATE : +return trace_drm_dbg_state_enabled (); +case DRM_UT_LEASE : +return trace_drm_dbg_lease_enabled (); +case DRM_UT_DP : +return trace_drm_dbg_dp_enabled (); +default : +return trace_drm_dbg_unknown_enabled (); + } + } +return false; +} + +static void drm_do_trace (const struct device *dev,unsigned int category, +struct va_format *vaf) +{ +WARN_ON (hweight32 (category) >1 ); + +switch (category) { +case DRM_UT_NONE : +trace_drm_dbg_none (dev, vaf); +break ; +case DRM_UT_CORE : +trace_drm_dbg_core (dev, vaf); +break ; +case DRM_UT_DRIVER : +trace_drm_dbg_driver (dev, vaf); +break ; +case DRM_UT_KMS : +trace_drm_dbg_kms (dev, vaf); +break ; +case DRM_UT_PRIME : +trace_drm_dbg_prime (dev, vaf); +break ; +case DRM_UT_ATOMIC : +trace_drm_dbg_atomic (dev, vaf); +break ; +case DRM_UT_VBL : +trace_drm_dbg_vbl (dev, vaf); +break ; +case DRM_UT_STATE : +trace_drm_dbg_state (dev, vaf); +break ; +case DRM_UT_LEASE : +trace_drm_dbg_lease (dev, vaf); +break ; +case DRM_UT_DP : +trace_drm_dbg_dp (dev, vaf); +break ; +default : +trace_drm_dbg_unknown (dev, vaf); +break ; + } +} +void drm_dev_dbg (const struct device *dev,unsigned int category,const char *format, ...) {struct va_format vaf; +unsigned int bit;va_list args; -if (!drm_debug_enabled (category)) -return ; - -va_start (args, format); vaf.fmt = format; vaf.va = &args; -if (dev) -dev_printk (KERN_DEBUG , dev,"[" DRM_NAME ":%ps] %pV" , -__builtin_return_address (0 ), &vaf); -else -printk (KERN_DEBUG "[" DRM_NAME ":%ps] %pV" , -__builtin_return_address (0 ), &vaf); +if (drm_debug_enabled (category)) { +va_start (args, format); +if (dev) +dev_printk (KERN_DEBUG , dev,"[" DRM_NAME ":%ps] %pV" , +__builtin_return_address (0 ), &vaf); +else +printk (KERN_DEBUG "[" DRM_NAME ":%ps] %pV" , +__builtin_return_address (0 ), &vaf); +va_end (args); + } + +if (!drm_trace_enabled (category)) +return ; + +for_each_set_bit (bit, (unsigned long *)&category,sizeof (category) *8 ) { +va_start (args, format); +drm_do_trace (dev,BIT (bit), &vaf); +va_end (args); + } -va_end (args); }EXPORT_SYMBOL (drm_dev_dbg);void drm_dbg (unsigned int category,const char *format, ...) {struct va_format vaf; +unsigned int bit;va_list args; -if (!drm_debug_enabled (category)) -return ; - -va_start (args, format); vaf.fmt = format; vaf.va = &args; -printk (KERN_DEBUG "[" DRM_NAME ":%ps] %pV" , -__builtin_return_address (0 ), &vaf); +if (drm_debug_enabled (category)) { +va_start (args, format); +printk (KERN_DEBUG "[" DRM_NAME ":%ps] %pV" , +__builtin_return_address (0 ), &vaf); +va_end (args); + } -va_end (args); +if (!drm_trace_enabled (category)) +return ; + +for_each_set_bit (bit, (unsigned long *)&category,sizeof (category) *8 ) { +va_start (args, format); +drm_do_trace (NULL ,BIT (bit), &vaf); +va_end (args); + } }EXPORT_SYMBOL (drm_dbg);
@@ -304,13 +404,16 @@ void drm_err(const char *format, ...)struct va_format vaf;va_list args; -va_start (args, format); vaf.fmt = format; vaf.va = &args; +va_start (args, format);printk (KERN_ERR "[" DRM_NAME ":%ps] *ERROR* %pV" ,__builtin_return_address (0 ), &vaf); +va_end (args); +va_start (args, format); +trace_drm_err (NULL , &vaf);va_end (args); }EXPORT_SYMBOL (drm_err);
include/trace/events/drm_print.h new file +116 -0
116 lines not on this page · raw message
Signature · 6 lines
Sean Paul, Software Engineer, Google / Chromium OS _______________________________________________ dri-devel mailing list [email protected] https://lists.freedesktop.org/mailman/listinfo/dri-devel
Steven Rostedt
maintainerQuoted 3 lines · On Thu, 12 Dec 2019 15:32:35 -0500 Sean Paul <[email protected]> wrote:
What about
trace_printk()? It doesn't give us the control we get from using tracepoints and it's not meant to be left sprinkled around in code.
Not to mention that trace_printk() is not for production use (only for developers debugging purposes).
Quoted 92 lines
Cc David Airlie <[email protected]>Cc Daniel Vetter <[email protected]>Cc Pekka Paalanen <[email protected]>Cc Joonas Lahtinen <[email protected]>Cc Thomas Zimmermann <[email protected]>Cc Rob Clark <[email protected]>Cc Ville Syrjälä <[email protected]>Acked-by Daniel Vetter <[email protected]>Signed-off-by Sean Paul <[email protected]>Changes in v2:
- Went with a completely different approach: https://lists.freedesktop.org/archives/dri-devel/2019-November/243230.htmlChanges in v3:
- Changed commit message to be a bit less RFC-y
- Makeclass_drm_category_logan actual trace class
---Even though we don't want it to be, this is UAPI. So here's some userspace code which uses it:
https://chromium-review.googlesource.com/c/chromiumos/platform2/+/1965562drivers/gpu/drm/drm_print.c | 143 ++++++++++++++++++++++++++----- include/trace/events/drm_print.h | 116 +++++++++++++++++++++++++ 2 files changed, 239 insertions(+), 20 deletions(-) create mode 100644 include/trace/events/drm_print.hdrivers/gpu/drm/drm_print.c +28
-1@@ -27,11 +27,15 @@#include <stdarg.h> +#include <linux/bitops.h> #include <linux/io.h> #include <linux/moduleparam.h> #include <linux/seq_file.h> #include <linux/slab.h> +#define CREATE_TRACE_POINTS +#include <trace/events/drm_print.h> +#include <drm/drm.h> #include <drm/drm_drv.h> #include <drm/drm_print.h> @@ -241,10 +245,10 @@ void drm_dev_printk(const struct device *dev, const char *level,struct va_format vaf;va_list args; -va_start (args, format); vaf.fmt = format; vaf.va = &args; +va_start (args, format);if (dev)dev_printk (level, dev,"[" DRM_NAME ":%ps] %pV" ,__builtin_return_address (0 ), &vaf);@@ -253,49 +257,145 @@ void drm_dev_printk(const struct device *dev, const char *level, level, __builtin_return_address(0), &vaf);va_end (args); + +va_start (args, format); +trace_drm_log (level, dev, &vaf); +va_end (args); }EXPORT_SYMBOL (drm_dev_printk); +static unsigned int drm_trace_enabled (unsigned int category) +{ +unsigned int bit; + +for_each_set_bit (bit, (unsigned long *)&category,sizeof (category) *8 ) { +switch (BIT (bit)) { +case DRM_UT_NONE : +return trace_drm_dbg_none_enabled (); +case DRM_UT_CORE : +return trace_drm_dbg_core_enabled (); +case DRM_UT_DRIVER : +return trace_drm_dbg_driver_enabled (); +case DRM_UT_KMS : +return trace_drm_dbg_kms_enabled (); +case DRM_UT_PRIME : +return trace_drm_dbg_prime_enabled (); +case DRM_UT_ATOMIC : +return trace_drm_dbg_atomic_enabled (); +case DRM_UT_VBL :
drivers/gpu/drm/drm_print.c void drm_dev_printk(const struct device *dev, const char *level, · line 295
Quoted 10 lines
drivers/gpu/drm/drm_print.c +10
-0+return trace_drm_dbg_vbl_enabled (); +case DRM_UT_STATE : +return trace_drm_dbg_state_enabled (); +case DRM_UT_LEASE : +return trace_drm_dbg_lease_enabled (); +case DRM_UT_DP : +return trace_drm_dbg_dp_enabled (); +default : +return trace_drm_dbg_unknown_enabled (); + }
Why this double loop?
Quoted 33 lines
drivers/gpu/drm/drm_print.c +33
-0+ } +return false; +} + +static void drm_do_trace (const struct device *dev,unsigned int category, +struct va_format *vaf) +{ +WARN_ON (hweight32 (category) >1 ); + +switch (category) { +case DRM_UT_NONE : +trace_drm_dbg_none (dev, vaf); +break ; +case DRM_UT_CORE : +trace_drm_dbg_core (dev, vaf); +break ; +case DRM_UT_DRIVER : +trace_drm_dbg_driver (dev, vaf); +break ; +case DRM_UT_KMS : +trace_drm_dbg_kms (dev, vaf); +break ; +case DRM_UT_PRIME : +trace_drm_dbg_prime (dev, vaf); +break ; +case DRM_UT_ATOMIC : +trace_drm_dbg_atomic (dev, vaf); +break ; +case DRM_UT_VBL : +trace_drm_dbg_vbl (dev, vaf); +break ; +case DRM_UT_STATE : +trace_drm_dbg_state (dev, vaf);
drivers/gpu/drm/drm_print.c void drm_dev_printk(const struct device *dev, const char *level, · line 338
Quoted 10 lines
drivers/gpu/drm/drm_print.c +10
-0+break ; +case DRM_UT_LEASE : +trace_drm_dbg_lease (dev, vaf); +break ; +case DRM_UT_DP : +trace_drm_dbg_dp (dev, vaf); +break ; +default : +trace_drm_dbg_unknown (dev, vaf); +break ;
Why is there a separate trace event for each of these?
Quoted 31 lines
drivers/gpu/drm/drm_print.c +12
-10+ } +} +void drm_dev_dbg (const struct device *dev,unsigned int category,const char *format, ...) {struct va_format vaf; +unsigned int bit;va_list args; -if (!drm_debug_enabled (category)) -return ; - -va_start (args, format); vaf.fmt = format; vaf.va = &args; -if (dev) -dev_printk (KERN_DEBUG , dev,"[" DRM_NAME ":%ps] %pV" , -__builtin_return_address (0 ), &vaf); -else -printk (KERN_DEBUG "[" DRM_NAME ":%ps] %pV" , -__builtin_return_address (0 ), &vaf); +if (drm_debug_enabled (category)) { +va_start (args, format); +if (dev) +dev_printk (KERN_DEBUG , dev,"[" DRM_NAME ":%ps] %pV" , +__builtin_return_address (0 ), &vaf); +else +printk (KERN_DEBUG "[" DRM_NAME ":%ps] %pV" , +__builtin_return_address (0 ), &vaf);
drivers/gpu/drm/drm_print.c void drm_dev_printk(const struct device *dev, const char *level, · line 369
Quoted 10 lines
drivers/gpu/drm/drm_print.c +10
-0+va_end (args); + } + +if (!drm_trace_enabled (category)) +return ; + +for_each_set_bit (bit, (unsigned long *)&category,sizeof (category) *8 ) { +va_start (args, format); +drm_do_trace (dev,BIT (bit), &vaf); +va_end (args);
I'm thinking we could find a way to move the logic here to have a single loop.
Note, the current approach is racy. The state can change between the "drm_trace_enabled()" and this for loop.
-- Steve
Quoted 170 lines
drivers/gpu/drm/drm_print.c +20
-9+ } -va_end (args); }EXPORT_SYMBOL (drm_dev_dbg);void drm_dbg (unsigned int category,const char *format, ...) {struct va_format vaf; +unsigned int bit;va_list args; -if (!drm_debug_enabled (category)) -return ; - -va_start (args, format); vaf.fmt = format; vaf.va = &args; -printk (KERN_DEBUG "[" DRM_NAME ":%ps] %pV" , -__builtin_return_address (0 ), &vaf); +if (drm_debug_enabled (category)) { +va_start (args, format); +printk (KERN_DEBUG "[" DRM_NAME ":%ps] %pV" , +__builtin_return_address (0 ), &vaf); +va_end (args); + } -va_end (args); +if (!drm_trace_enabled (category)) +return ; + +for_each_set_bit (bit, (unsigned long *)&category,sizeof (category) *8 ) { +va_start (args, format); +drm_do_trace (NULL ,BIT (bit), &vaf); +va_end (args); + } }EXPORT_SYMBOL (drm_dbg);@@ -304,13 +404,16 @@ void drm_err(const char *format, ...)struct va_format vaf;va_list args; -va_start (args, format); vaf.fmt = format; vaf.va = &args; +va_start (args, format);printk (KERN_ERR "[" DRM_NAME ":%ps] *ERROR* %pV" ,__builtin_return_address (0 ), &vaf); +va_end (args); +va_start (args, format); +trace_drm_err (NULL , &vaf);va_end (args); }EXPORT_SYMBOL (drm_err);include/trace/events/drm_print.h new file +106
-0@@ -0,0 +1,116 @@ +// SPDX-License-Identifier: MIT +/* +* Copyright (C) 2019 Google, Inc. +* +* Authors: +* Sean Paul <[email protected]> +*/ +#undef TRACE_SYSTEM +#define TRACE_SYSTEM drm_print + +#if !defined(_TRACE_DRM_PRINT_H) || defined(TRACE_HEADER_MULTI_READ ) + +#include <linux/device.h> +#include <linux/tracepoint.h> + +#define DRM_PRINT_MAX 256 + +#define _TRACE_DRM_PRINT_H + +TRACE_EVENT (drm_log, + TP_PROTO(const char * level, const struct device *dev, + struct va_format *vaf), +TP_ARGS (level, dev, vaf), +TP_STRUCT__entry ( + __field(const char *, level ) + __field(const struct device *, dev ) + __dynamic_array(char, msg, DRM_PRINT_MAX ) +__field (int, len ) + ), +TP_fast_assign ( + __entry->level = level; + __entry->dev = dev; + __entry->len = vsnprintf(__get_str(msg), DRM_PRINT_MAX, + vaf->fmt, *vaf->va); + if (__entry->len > 0 && + __get_str(msg)[__entry->len - 1] == '\n') + __get_str(msg)[--__entry->len] = '\0'; + ), +TP_printk ("%s %s %s: %s" , __entry->level, + __entry->dev ?dev_driver_string (__entry->dev) :"(NULL):" , + __entry->dev ?dev_name (__entry->dev) :"" ,__get_str (msg)) +); + +DECLARE_EVENT_CLASS (class_drm_category_log, + TP_PROTO(const struct device *dev, struct va_format *vaf), +TP_ARGS (dev, vaf), +TP_STRUCT__entry ( + __field(const struct device *, dev ) + __dynamic_array(char, msg, DRM_PRINT_MAX ) +__field (int, len ) + ), +TP_fast_assign ( + __entry->dev = dev; + __entry->len = vsnprintf(__get_str(msg), DRM_PRINT_MAX, + vaf->fmt, *vaf->va); + if (__entry->len > 0 && + __get_str(msg)[__entry->len - 1] == '\n') + __get_str(msg)[--__entry->len] = '\0'; + ), +TP_printk ("%s %s%s%s" , + __entry->dev ?dev_driver_string (__entry->dev) :"" , + __entry->dev ?dev_name (__entry->dev) :"" , + __entry->dev ?": " :"" ,__get_str (msg)) +); +DEFINE_EVENT (class_drm_category_log, drm_err, + TP_PROTO(const struct device *dev, struct va_format *vaf), +TP_ARGS (dev, vaf) +); +DEFINE_EVENT (class_drm_category_log, drm_dbg_none, + TP_PROTO(const struct device *dev, struct va_format *vaf), +TP_ARGS (dev, vaf) +); +DEFINE_EVENT (class_drm_category_log, drm_dbg_core, + TP_PROTO(const struct device *dev, struct va_format *vaf), +TP_ARGS (dev, vaf) +); +DEFINE_EVENT (class_drm_category_log, drm_dbg_driver, + TP_PROTO(const struct device *dev, struct va_format *vaf), +TP_ARGS (dev, vaf) +); +DEFINE_EVENT (class_drm_category_log, drm_dbg_kms, + TP_PROTO(const struct device *dev, struct va_format *vaf), +TP_ARGS (dev, vaf) +); +DEFINE_EVENT (class_drm_category_log, drm_dbg_prime, + TP_PROTO(const struct device *dev, struct va_format *vaf), +TP_ARGS (dev, vaf) +); +DEFINE_EVENT (class_drm_category_log, drm_dbg_atomic, + TP_PROTO(const struct device *dev, struct va_format *vaf), +TP_ARGS (dev, vaf) +); +DEFINE_EVENT (class_drm_category_log, drm_dbg_vbl, + TP_PROTO(const struct device *dev, struct va_format *vaf), +TP_ARGS (dev, vaf) +); +DEFINE_EVENT (class_drm_category_log, drm_dbg_state, + TP_PROTO(const struct device *dev, struct va_format *vaf), +TP_ARGS (dev, vaf) +); +DEFINE_EVENT (class_drm_category_log, drm_dbg_lease, + TP_PROTO(const struct device *dev, struct va_format *vaf), +TP_ARGS (dev, vaf) +); +DEFINE_EVENT(class_drm_category_log, drm_dbg_dp, + TP_PROTO(const struct device *dev, struct va_format *vaf),
include/trace/events/drm_print.h line 116
Quoted 10 lines
include/trace/events/drm_print.h new file +10
-0+TP_ARGS (dev, vaf) +); +DEFINE_EVENT (class_drm_category_log, drm_dbg_unknown, + TP_PROTO(const struct device *dev, struct va_format *vaf), +TP_ARGS (dev, vaf) +); +#endif + +/* This part must be outside protection */ +#include <trace/define_trace.h>
_______________________________________________
dri-devel mailing list
[email protected]
https://lists.freedesktop.org/mailman/listinfo/dri-devel
Hi Steven,
Thanks for the feedback!
Quoted 10 lines · Steven Rostedt
Quoted 3 lines · On Thu, 12 Dec 2019 15:32:35 -0500 Sean Paul <[email protected]> wrote:
What about
trace_printk()? It doesn't give us the control we get from using tracepoints and it's not meant to be left sprinkled around in code.Not to mention that
trace_printk()is not for production use (only for developers debugging purposes).
Yep, I'll clarify this in the next version.
Quoted 96 lines
Quoted 96 lines
Cc David Airlie <[email protected]>Cc Daniel Vetter <[email protected]>Cc Pekka Paalanen <[email protected]>Cc Joonas Lahtinen <[email protected]>Cc Thomas Zimmermann <[email protected]>Cc Rob Clark <[email protected]>Cc Ville Syrjälä <[email protected]>Acked-by Daniel Vetter <[email protected]>Signed-off-by Sean Paul <[email protected]>Changes in v2:
- Went with a completely different approach: https://lists.freedesktop.org/archives/dri-devel/2019-November/243230.htmlChanges in v3:
- Changed commit message to be a bit less RFC-y
- Makeclass_drm_category_logan actual trace class
---Even though we don't want it to be, this is UAPI. So here's some userspace code which uses it:
https://chromium-review.googlesource.com/c/chromiumos/platform2/+/1965562drivers/gpu/drm/drm_print.c | 143 ++++++++++++++++++++++++++----- include/trace/events/drm_print.h | 116 +++++++++++++++++++++++++ 2 files changed, 239 insertions(+), 20 deletions(-) create mode 100644 include/trace/events/drm_print.hdrivers/gpu/drm/drm_print.c +32
-1@@ -27,11 +27,15 @@#include <stdarg.h> +#include <linux/bitops.h> #include <linux/io.h> #include <linux/moduleparam.h> #include <linux/seq_file.h> #include <linux/slab.h> +#define CREATE_TRACE_POINTS +#include <trace/events/drm_print.h> +#include <drm/drm.h> #include <drm/drm_drv.h> #include <drm/drm_print.h> @@ -241,10 +245,10 @@ void drm_dev_printk(const struct device *dev, const char *level,struct va_format vaf;va_list args; -va_start (args, format); vaf.fmt = format; vaf.va = &args; +va_start (args, format);if (dev)dev_printk (level, dev,"[" DRM_NAME ":%ps] %pV" ,__builtin_return_address (0 ), &vaf);@@ -253,49 +257,145 @@ void drm_dev_printk(const struct device *dev, const char *level, level, __builtin_return_address(0), &vaf);va_end (args); + +va_start (args, format); +trace_drm_log (level, dev, &vaf); +va_end (args); }EXPORT_SYMBOL (drm_dev_printk); +static unsigned int drm_trace_enabled (unsigned int category) +{ +unsigned int bit; + +for_each_set_bit (bit, (unsigned long *)&category,sizeof (category) *8 ) { +switch (BIT (bit)) { +case DRM_UT_NONE : +return trace_drm_dbg_none_enabled (); +case DRM_UT_CORE : +return trace_drm_dbg_core_enabled (); +case DRM_UT_DRIVER : +return trace_drm_dbg_driver_enabled (); +case DRM_UT_KMS : +return trace_drm_dbg_kms_enabled (); +case DRM_UT_PRIME : +return trace_drm_dbg_prime_enabled (); +case DRM_UT_ATOMIC : +return trace_drm_dbg_atomic_enabled (); +case DRM_UT_VBL : +return trace_drm_dbg_vbl_enabled (); +case DRM_UT_STATE : +return trace_drm_dbg_state_enabled (); +case DRM_UT_LEASE :
Quoted 9 lines
drivers/gpu/drm/drm_print.c void drm_dev_printk(const struct device *dev, const char *level, · line 295
Quoted 6 lines
drivers/gpu/drm/drm_print.c +6
-0+return trace_drm_dbg_lease_enabled (); +case DRM_UT_DP : +return trace_drm_dbg_dp_enabled (); +default : +return trace_drm_dbg_unknown_enabled (); + }Why this double loop?
category is technically a bitfield, but only 1 bit is ever set (grander plans when it was created, I suppose). As Jani mentions, it's being changed to an enum, so we won't need this loop.
Quoted 37 lines
Quoted 37 lines
drivers/gpu/drm/drm_print.c +37
-0+ } +return false; +} + +static void drm_do_trace (const struct device *dev,unsigned int category, +struct va_format *vaf) +{ +WARN_ON (hweight32 (category) >1 ); + +switch (category) { +case DRM_UT_NONE : +trace_drm_dbg_none (dev, vaf); +break ; +case DRM_UT_CORE : +trace_drm_dbg_core (dev, vaf); +break ; +case DRM_UT_DRIVER : +trace_drm_dbg_driver (dev, vaf); +break ; +case DRM_UT_KMS : +trace_drm_dbg_kms (dev, vaf); +break ; +case DRM_UT_PRIME : +trace_drm_dbg_prime (dev, vaf); +break ; +case DRM_UT_ATOMIC : +trace_drm_dbg_atomic (dev, vaf); +break ; +case DRM_UT_VBL : +trace_drm_dbg_vbl (dev, vaf); +break ; +case DRM_UT_STATE : +trace_drm_dbg_state (dev, vaf); +break ; +case DRM_UT_LEASE : +trace_drm_dbg_lease (dev, vaf); +break ;
Quoted 9 lines
drivers/gpu/drm/drm_print.c void drm_dev_printk(const struct device *dev, const char *level, · line 338
Quoted 6 lines
drivers/gpu/drm/drm_print.c +6
-0+case DRM_UT_DP : +trace_drm_dbg_dp (dev, vaf); +break ; +default : +trace_drm_dbg_unknown (dev, vaf); +break ;Why is there a separate trace event for each of these?
To make it easier on userspace to enable just a single drm category.
Quoted 36 lines
Quoted 36 lines
drivers/gpu/drm/drm_print.c +17
-10+ } +} +void drm_dev_dbg (const struct device *dev,unsigned int category,const char *format, ...) {struct va_format vaf; +unsigned int bit;va_list args; -if (!drm_debug_enabled (category)) -return ; - -va_start (args, format); vaf.fmt = format; vaf.va = &args; -if (dev) -dev_printk (KERN_DEBUG , dev,"[" DRM_NAME ":%ps] %pV" , -__builtin_return_address (0 ), &vaf); -else -printk (KERN_DEBUG "[" DRM_NAME ":%ps] %pV" , -__builtin_return_address (0 ), &vaf); +if (drm_debug_enabled (category)) { +va_start (args, format); +if (dev) +dev_printk (KERN_DEBUG , dev,"[" DRM_NAME ":%ps] %pV" , +__builtin_return_address (0 ), &vaf); +else +printk (KERN_DEBUG "[" DRM_NAME ":%ps] %pV" , +__builtin_return_address (0 ), &vaf); +va_end (args); + } + +if (!drm_trace_enabled (category)) +return ;
Quoted 8 lines
drivers/gpu/drm/drm_print.c void drm_dev_printk(const struct device *dev, const char *level, · line 369
Quoted 5 lines
drivers/gpu/drm/drm_print.c +5
-0+ +for_each_set_bit (bit, (unsigned long *)&category,sizeof (category) *8 ) { +va_start (args, format); +drm_do_trace (dev,BIT (bit), &vaf); +va_end (args);I'm thinking we could find a way to move the logic here to have a single loop.
Should be done with the enum conversion mentioned above.
Quoted 3 lines
Note, the current approach is racy. The state can change between the "
drm_trace_enabled()" and this for loop.
Yep, good catch. I think the race is ok since in that case the message will get dropped when we call drm_do_trace. The trace_enabled() call is just a fastpath for the majority case.
Sean
Quoted 185 lines
-- Steve
Quoted 180 lines
drivers/gpu/drm/drm_print.c +20
-9+ } -va_end (args); }EXPORT_SYMBOL (drm_dev_dbg);void drm_dbg (unsigned int category,const char *format, ...) {struct va_format vaf; +unsigned int bit;va_list args; -if (!drm_debug_enabled (category)) -return ; - -va_start (args, format); vaf.fmt = format; vaf.va = &args; -printk (KERN_DEBUG "[" DRM_NAME ":%ps] %pV" , -__builtin_return_address (0 ), &vaf); +if (drm_debug_enabled (category)) { +va_start (args, format); +printk (KERN_DEBUG "[" DRM_NAME ":%ps] %pV" , +__builtin_return_address (0 ), &vaf); +va_end (args); + } -va_end (args); +if (!drm_trace_enabled (category)) +return ; + +for_each_set_bit (bit, (unsigned long *)&category,sizeof (category) *8 ) { +va_start (args, format); +drm_do_trace (NULL ,BIT (bit), &vaf); +va_end (args); + } }EXPORT_SYMBOL (drm_dbg);@@ -304,13 +404,16 @@ void drm_err(const char *format, ...)struct va_format vaf;va_list args; -va_start (args, format); vaf.fmt = format; vaf.va = &args; +va_start (args, format);printk (KERN_ERR "[" DRM_NAME ":%ps] *ERROR* %pV" ,__builtin_return_address (0 ), &vaf); +va_end (args); +va_start (args, format); +trace_drm_err (NULL , &vaf);va_end (args); }EXPORT_SYMBOL (drm_err);include/trace/events/drm_print.h new file +116
-0@@ -0,0 +1,116 @@ +// SPDX-License-Identifier: MIT +/* +* Copyright (C) 2019 Google, Inc. +* +* Authors: +* Sean Paul <[email protected]> +*/ +#undef TRACE_SYSTEM +#define TRACE_SYSTEM drm_print + +#if !defined(_TRACE_DRM_PRINT_H) || defined(TRACE_HEADER_MULTI_READ ) + +#include <linux/device.h> +#include <linux/tracepoint.h> + +#define DRM_PRINT_MAX 256 + +#define _TRACE_DRM_PRINT_H + +TRACE_EVENT (drm_log, + TP_PROTO(const char * level, const struct device *dev, + struct va_format *vaf), +TP_ARGS (level, dev, vaf), +TP_STRUCT__entry ( + __field(const char *, level ) + __field(const struct device *, dev ) + __dynamic_array(char, msg, DRM_PRINT_MAX ) +__field (int, len ) + ), +TP_fast_assign ( + __entry->level = level; + __entry->dev = dev; + __entry->len = vsnprintf(__get_str(msg), DRM_PRINT_MAX, + vaf->fmt, *vaf->va); + if (__entry->len > 0 && + __get_str(msg)[__entry->len - 1] == '\n') + __get_str(msg)[--__entry->len] = '\0'; + ), +TP_printk ("%s %s %s: %s" , __entry->level, + __entry->dev ?dev_driver_string (__entry->dev) :"(NULL):" , + __entry->dev ?dev_name (__entry->dev) :"" ,__get_str (msg)) +); + +DECLARE_EVENT_CLASS (class_drm_category_log, + TP_PROTO(const struct device *dev, struct va_format *vaf), +TP_ARGS (dev, vaf), +TP_STRUCT__entry ( + __field(const struct device *, dev ) + __dynamic_array(char, msg, DRM_PRINT_MAX ) +__field (int, len ) + ), +TP_fast_assign ( + __entry->dev = dev; + __entry->len = vsnprintf(__get_str(msg), DRM_PRINT_MAX, + vaf->fmt, *vaf->va); + if (__entry->len > 0 && + __get_str(msg)[__entry->len - 1] == '\n') + __get_str(msg)[--__entry->len] = '\0'; + ), +TP_printk ("%s %s%s%s" , + __entry->dev ?dev_driver_string (__entry->dev) :"" , + __entry->dev ?dev_name (__entry->dev) :"" , + __entry->dev ?": " :"" ,__get_str (msg)) +); +DEFINE_EVENT (class_drm_category_log, drm_err, + TP_PROTO(const struct device *dev, struct va_format *vaf), +TP_ARGS (dev, vaf) +); +DEFINE_EVENT (class_drm_category_log, drm_dbg_none, + TP_PROTO(const struct device *dev, struct va_format *vaf), +TP_ARGS (dev, vaf) +); +DEFINE_EVENT (class_drm_category_log, drm_dbg_core, + TP_PROTO(const struct device *dev, struct va_format *vaf), +TP_ARGS (dev, vaf) +); +DEFINE_EVENT (class_drm_category_log, drm_dbg_driver, + TP_PROTO(const struct device *dev, struct va_format *vaf), +TP_ARGS (dev, vaf) +); +DEFINE_EVENT (class_drm_category_log, drm_dbg_kms, + TP_PROTO(const struct device *dev, struct va_format *vaf), +TP_ARGS (dev, vaf) +); +DEFINE_EVENT (class_drm_category_log, drm_dbg_prime, + TP_PROTO(const struct device *dev, struct va_format *vaf), +TP_ARGS (dev, vaf) +); +DEFINE_EVENT (class_drm_category_log, drm_dbg_atomic, + TP_PROTO(const struct device *dev, struct va_format *vaf), +TP_ARGS (dev, vaf) +); +DEFINE_EVENT (class_drm_category_log, drm_dbg_vbl, + TP_PROTO(const struct device *dev, struct va_format *vaf), +TP_ARGS (dev, vaf) +); +DEFINE_EVENT (class_drm_category_log, drm_dbg_state, + TP_PROTO(const struct device *dev, struct va_format *vaf), +TP_ARGS (dev, vaf) +); +DEFINE_EVENT (class_drm_category_log, drm_dbg_lease, + TP_PROTO(const struct device *dev, struct va_format *vaf), +TP_ARGS (dev, vaf) +); +DEFINE_EVENT (class_drm_category_log, drm_dbg_dp, + TP_PROTO(const struct device *dev, struct va_format *vaf), +TP_ARGS (dev, vaf) +); +DEFINE_EVENT (class_drm_category_log, drm_dbg_unknown, + TP_PROTO(const struct device *dev, struct va_format *vaf), +TP_ARGS (dev, vaf) +); +#endif + +/* This part must be outside protection */ +#include <trace/define_trace.h>
Signature · 5 lines
Sean Paul, Software Engineer, Google / Chromium OS _______________________________________________ dri-devel mailing list [email protected] https://lists.freedesktop.org/mailman/listinfo/dri-devel
Steven Rostedt
maintainerQuoted 5 lines · On Fri, 13 Dec 2019 11:47:03 -0500 Sean Paul <[email protected]> wrote:
Quoted 2 lines
Why is there a separate trace event for each of these?
To make it easier on userspace to enable just a single drm category.
But trace events can easily be filtered. Say you have a field called category, you could then just set in the trace event filter:
echo 'category == 1 || category == 5 || (category > 8 && category < 12)' > /sys/kernel/tracing/drm_print/drm_category_log/filter echo 1 > /sys/kernel/tracing/drm_print/drm_category_log/enable
-- Steve
_______________________________________________
dri-devel mailing list
[email protected]
https://lists.freedesktop.org/mailman/listinfo/dri-devel
Quoted 6 lines · Steven Rostedt
Quoted 3 lines · On Fri, 13 Dec 2019 11:47:03 -0500 Sean Paul <[email protected]> wrote:
Quoted 2 lines
Why is there a separate trace event for each of these?
Quoted 8 lines
Quoted 2 lines
To make it easier on userspace to enable just a single drm category.
But trace events can easily be filtered. Say you have a field called category, you could then just set in the trace event filter:
echo 'category == 1 || category == 5 || (category > 8 && category < 12)' >
/sys/kernel/tracing/drm_print/drm_category_log/filterecho 1 >/sys/kernel/tracing/drm_print/drm_category_log/enable
Hmm, true. I suppose the only downside would be that this would cause all logs to be processed/assigned as trace events before the filter is applied. That's a lot of extra work each frame.
Sean
Quoted 2 lines
-- Steve
Signature · 5 lines
Sean Paul, Software Engineer, Google / Chromium OS _______________________________________________ dri-devel mailing list [email protected] https://lists.freedesktop.org/mailman/listinfo/dri-devel
Quoted 57 lines · On Thu, 12 Dec 2019 15:32:35 -0500 Sean Paul <[email protected]> wrote:
From: Sean Paul <[email protected]>
For a long while now, we (ChromeOS) have been struggling getting any value out of user feedback reports of display failures (notably external displays not working). The problem is that all logging, even fatal errors (well, fatal in the sense that a display won't light up) are logged at DEBUG log level. So in order to extract these logs, users need to be able to turn on logging, and reproduce the issue with debug enabled. Unfortunately, this isn't really something we can ask CrOS users to do. I spoke with airlied about this and RHEL has similar issues. After a few more people piped up on previous versions of this patch, it is a Real Issue.
So why don't we just enable
DRM_UT_BLAH? Here are the reasons in ascending order of severity:
1- People aren't consistent with their categories, so we'd have to
enable a bunch to get proper coverage
2- We don't want to overwhelm syslog with drm spam, others have to use
it too
3- Console logging is slowSo what we really want is a ringbuffer of the most recent logs (filtered by categories we're interested in) exposed via debugfs so the logs can be extracted when users file feedback.
It just so happens that there is something which does _exactly_ this! This patch dumps drm logs into tracepoints, which allows us to turn tracing on and off depending on which category is useful, and pull them from tracefs on demand.
What about
trace_printk()? It doesn't give us the control we get from using tracepoints and it's not meant to be left sprinkled around in code.Cc David Airlie <[email protected]>Cc Daniel Vetter <[email protected]>Cc Pekka Paalanen <[email protected]>Cc Joonas Lahtinen <[email protected]>Cc Thomas Zimmermann <[email protected]>Cc Rob Clark <[email protected]>Cc Ville Syrjälä <[email protected]>Acked-by Daniel Vetter <[email protected]>Signed-off-by Sean Paul <[email protected]>Changes in v2:
- Went with a completely different approach: https://lists.freedesktop.org/archives/dri-devel/2019-November/243230.htmlChanges in v3:
- Changed commit message to be a bit less RFC-y
- Makeclass_drm_category_logan actual trace class
---Even though we don't want it to be, this is UAPI. So here's some userspace code which uses it:
https://chromium-review.googlesource.com/c/chromiumos/platform2/+/1965562
Quoted 4 lines
drivers/gpu/drm/drm_print.c | 143 ++++++++++++++++++++++++++----- include/trace/events/drm_print.h | 116 +++++++++++++++++++++++++ 2 files changed, 239 insertions(+), 20 deletions(-) create mode 100644 include/trace/events/drm_print.h
Hi,
reading the userspace patch is very enlightening, thanks.
It uses debugfs, and it uses the generic tracing UAPI. When all distributions will enable this debug logging like you do in your userspace patch (I really want that to be the end result, since otherwise we are back to asking people to manually enable debug and then reproduce the failure), does that scale?
What if V4L2 is the next one deciding they need a similar logging framework to debug camera issues? If the trace log is already flooded with DRM messages, it will be useless for them?
Or maybe someone else wants their piece and flood it even more aggressively than DRM, making the DRM messages disappear before they can be saved?
Is there a way to pull out messages
from /sys/kernel/debug/tracing/trace and filter them on reading instead of on writing?
Thanks,
pq
_______________________________________________
dri-devel mailing list
[email protected]
https://lists.freedesktop.org/mailman/listinfo/dri-devel
Steven Rostedt
maintainerQuoted 3 lines · On Fri, 13 Dec 2019 13:34:46 +0200 Pekka Paalanen <[email protected]> wrote:
Is there a way to pull out messages
from/sys/kernel/debug/tracing/traceand filter them on reading instead of on writing?
FYI, you can mount the tracing directory without having to mount the debug directory.
mount -t tracefs nodev /sys/kernel/tracing
This way you don't need to mount everyone's debugging features just to access tracing.
-- Steve
_______________________________________________
dri-devel mailing list
[email protected]
https://lists.freedesktop.org/mailman/listinfo/dri-devel
Quoted 79 lines · Pekka Paalanen
Quoted 57 lines · On Thu, 12 Dec 2019 15:32:35 -0500 Sean Paul <[email protected]> wrote:
From: Sean Paul <[email protected]>
For a long while now, we (ChromeOS) have been struggling getting any value out of user feedback reports of display failures (notably external displays not working). The problem is that all logging, even fatal errors (well, fatal in the sense that a display won't light up) are logged at DEBUG log level. So in order to extract these logs, users need to be able to turn on logging, and reproduce the issue with debug enabled. Unfortunately, this isn't really something we can ask CrOS users to do. I spoke with airlied about this and RHEL has similar issues. After a few more people piped up on previous versions of this patch, it is a Real Issue.
So why don't we just enable
DRM_UT_BLAH? Here are the reasons in ascending order of severity:
1- People aren't consistent with their categories, so we'd have to
enable a bunch to get proper coverage
2- We don't want to overwhelm syslog with drm spam, others have to use
it too
3- Console logging is slowSo what we really want is a ringbuffer of the most recent logs (filtered by categories we're interested in) exposed via debugfs so the logs can be extracted when users file feedback.
It just so happens that there is something which does _exactly_ this! This patch dumps drm logs into tracepoints, which allows us to turn tracing on and off depending on which category is useful, and pull them from tracefs on demand.
What about
trace_printk()? It doesn't give us the control we get from using tracepoints and it's not meant to be left sprinkled around in code.Cc David Airlie <[email protected]>Cc Daniel Vetter <[email protected]>Cc Pekka Paalanen <[email protected]>Cc Joonas Lahtinen <[email protected]>Cc Thomas Zimmermann <[email protected]>Cc Rob Clark <[email protected]>Cc Ville Syrjälä <[email protected]>Acked-by Daniel Vetter <[email protected]>Signed-off-by Sean Paul <[email protected]>Changes in v2:
- Went with a completely different approach: https://lists.freedesktop.org/archives/dri-devel/2019-November/243230.htmlChanges in v3:
- Changed commit message to be a bit less RFC-y
- Makeclass_drm_category_logan actual trace class
---Even though we don't want it to be, this is UAPI. So here's some userspace code which uses it:
https://chromium-review.googlesource.com/c/chromiumos/platform2/+/1965562Quoted 4 lines
drivers/gpu/drm/drm_print.c | 143 ++++++++++++++++++++++++++----- include/trace/events/drm_print.h | 116 +++++++++++++++++++++++++ 2 files changed, 239 insertions(+), 20 deletions(-) create mode 100644 include/trace/events/drm_print.hHi,
reading the userspace patch is very enlightening, thanks.
It uses debugfs, and it uses the generic tracing UAPI. When all distributions will enable this debug logging like you do in your userspace patch (I really want that to be the end result, since otherwise we are back to asking people to manually enable debug and then reproduce the failure), does that scale?
What if V4L2 is the next one deciding they need a similar logging framework to debug camera issues? If the trace log is already flooded with DRM messages, it will be useless for them?
Quoted 7 lines
Or maybe someone else wants their piece and flood it even more aggressively than DRM, making the DRM messages disappear before they can be saved?
Is there a way to pull out messages
from/sys/kernel/debug/tracing/traceand filter them on reading instead of on writing?
Hi Pekka,
Yep, there's also a trace_pipe output from tracefs. So you could pipe the output through a classifier in userspace and split off different subsystems.
I think if this type of patch proliferates this could be a problem. In that case, we'd probably have to become better citizens and reclassify some of our log messages such that we're not needlessly spamming the trace buffer.
I'm not too worried about this since we have a number of tools at our disposal.
Sean
Quoted 4 lines
Thanks,
pq
Signature · 5 lines
Sean Paul, Software Engineer, Google / Chromium OS _______________________________________________ dri-devel mailing list [email protected] https://lists.freedesktop.org/mailman/listinfo/dri-devel
Quoted 84 lines · On Fri, 13 Dec 2019 11:54:33 -0500 Sean Paul <[email protected]> wrote:
Quoted 83 lines · Pekka Paalanen
Quoted 57 lines · On Thu, 12 Dec 2019 15:32:35 -0500 Sean Paul <[email protected]> wrote:
From: Sean Paul <[email protected]>
For a long while now, we (ChromeOS) have been struggling getting any value out of user feedback reports of display failures (notably external displays not working). The problem is that all logging, even fatal errors (well, fatal in the sense that a display won't light up) are logged at DEBUG log level. So in order to extract these logs, users need to be able to turn on logging, and reproduce the issue with debug enabled. Unfortunately, this isn't really something we can ask CrOS users to do. I spoke with airlied about this and RHEL has similar issues. After a few more people piped up on previous versions of this patch, it is a Real Issue.
So why don't we just enable
DRM_UT_BLAH? Here are the reasons in ascending order of severity:
1- People aren't consistent with their categories, so we'd have to
enable a bunch to get proper coverage
2- We don't want to overwhelm syslog with drm spam, others have to use
it too
3- Console logging is slowSo what we really want is a ringbuffer of the most recent logs (filtered by categories we're interested in) exposed via debugfs so the logs can be extracted when users file feedback.
It just so happens that there is something which does _exactly_ this! This patch dumps drm logs into tracepoints, which allows us to turn tracing on and off depending on which category is useful, and pull them from tracefs on demand.
What about
trace_printk()? It doesn't give us the control we get from using tracepoints and it's not meant to be left sprinkled around in code.Cc David Airlie <[email protected]>Cc Daniel Vetter <[email protected]>Cc Pekka Paalanen <[email protected]>Cc Joonas Lahtinen <[email protected]>Cc Thomas Zimmermann <[email protected]>Cc Rob Clark <[email protected]>Cc Ville Syrjälä <[email protected]>Acked-by Daniel Vetter <[email protected]>Signed-off-by Sean Paul <[email protected]>Changes in v2:
- Went with a completely different approach: https://lists.freedesktop.org/archives/dri-devel/2019-November/243230.htmlChanges in v3:
- Changed commit message to be a bit less RFC-y
- Makeclass_drm_category_logan actual trace class
---Even though we don't want it to be, this is UAPI. So here's some userspace code which uses it:
https://chromium-review.googlesource.com/c/chromiumos/platform2/+/1965562Quoted 4 lines
drivers/gpu/drm/drm_print.c | 143 ++++++++++++++++++++++++++----- include/trace/events/drm_print.h | 116 +++++++++++++++++++++++++ 2 files changed, 239 insertions(+), 20 deletions(-) create mode 100644 include/trace/events/drm_print.hHi,
reading the userspace patch is very enlightening, thanks.
It uses debugfs, and it uses the generic tracing UAPI. When all distributions will enable this debug logging like you do in your userspace patch (I really want that to be the end result, since otherwise we are back to asking people to manually enable debug and then reproduce the failure), does that scale?
What if V4L2 is the next one deciding they need a similar logging framework to debug camera issues? If the trace log is already flooded with DRM messages, it will be useless for them?
Or maybe someone else wants their piece and flood it even more aggressively than DRM, making the DRM messages disappear before they can be saved?
Quoted 7 lines
Quoted 3 lines
Is there a way to pull out messages
from/sys/kernel/debug/tracing/traceand filter them on reading instead of on writing?Hi Pekka,
Yep, there's also atrace_pipeoutput from tracefs. So you could pipe the output through a classifier in userspace and split off different subsystems.
Hi,
IOW, one needs to run a userspace daemon to read the pipe, filter it, and keep a flight recorder buffer, instead of the kernel keeping it. Right?
I suppose keeping the flight recorder in userspace makes it much more flexible with things like dynamic sizing and remotes and whatnot. Is this where you would like to aim for, for general purpose desktop distributions to run one at all times?
Quoted 3 lines
I think if this type of patch proliferates this could be a problem. In that case, we'd probably have to become better citizens and reclassify some of our log messages such that we're not needlessly spamming the trace buffer.
How will you do that in a way that people who have already taken to bug reporting logging do not scream "UAPI broke, the logs are much less useful now"? :-)
Although I might worry more about people screaming "my tracing broke" if the buffers fill up with DRM chatter in the first place. After all, the aim is to have this enabled in desktop distributions by default, and if distributions get bug reports forcing them to disable it by default, then we're back to square "please enable debugging and reproduce the issue so we have something to look at" again.
FWIW, I'm very much in favour of your overall goal, but I'm worried it might not be sustainable or practical in the long term outside of very specialized distributions. Then again, I have no alternative suggestions that would have the same level of code and UAPI re-use.
Quoted 1 line
I'm not too worried about this since we have a number of tools at our disposal.
What tools would those be?
Thanks,
pq
_______________________________________________
dri-devel mailing list
[email protected]
https://lists.freedesktop.org/mailman/listinfo/dri-devel
Quoted 100 lines · Sean Paul
From: Sean Paul <[email protected]>
For a long while now, we (ChromeOS) have been struggling getting any value out of user feedback reports of display failures (notably external displays not working). The problem is that all logging, even fatal errors (well, fatal in the sense that a display won't light up) are logged at DEBUG log level. So in order to extract these logs, users need to be able to turn on logging, and reproduce the issue with debug enabled. Unfortunately, this isn't really something we can ask CrOS users to do. I spoke with airlied about this and RHEL has similar issues. After a few more people piped up on previous versions of this patch, it is a Real Issue.
So why don't we just enable
DRM_UT_BLAH? Here are the reasons in ascending order of severity:
1- People aren't consistent with their categories, so we'd have to
enable a bunch to get proper coverage
2- We don't want to overwhelm syslog with drm spam, others have to use
it too
3- Console logging is slowSo what we really want is a ringbuffer of the most recent logs (filtered by categories we're interested in) exposed via debugfs so the logs can be extracted when users file feedback.
It just so happens that there is something which does _exactly_ this! This patch dumps drm logs into tracepoints, which allows us to turn tracing on and off depending on which category is useful, and pull them from tracefs on demand.
What about
trace_printk()? It doesn't give us the control we get from using tracepoints and it's not meant to be left sprinkled around in code.Cc David Airlie <[email protected]>Cc Daniel Vetter <[email protected]>Cc Pekka Paalanen <[email protected]>Cc Joonas Lahtinen <[email protected]>Cc Thomas Zimmermann <[email protected]>Cc Rob Clark <[email protected]>Cc Ville Syrjälä <[email protected]>Acked-by Daniel Vetter <[email protected]>Signed-off-by Sean Paul <[email protected]>Changes in v2:
- Went with a completely different approach: https://lists.freedesktop.org/archives/dri-devel/2019-November/243230.htmlChanges in v3:
- Changed commit message to be a bit less RFC-y
- Makeclass_drm_category_logan actual trace class
---Even though we don't want it to be, this is UAPI. So here's some userspace code which uses it:
https://chromium-review.googlesource.com/c/chromiumos/platform2/+/1965562drivers/gpu/drm/drm_print.c | 143 ++++++++++++++++++++++++++----- include/trace/events/drm_print.h | 116 +++++++++++++++++++++++++ 2 files changed, 239 insertions(+), 20 deletions(-) create mode 100644 include/trace/events/drm_print.hdrivers/gpu/drm/drm_print.c +7
-1@@ -27,11 +27,15 @@#include <stdarg.h> +#include <linux/bitops.h> #include <linux/io.h> #include <linux/moduleparam.h> #include <linux/seq_file.h> #include <linux/slab.h> +#define CREATE_TRACE_POINTS +#include <trace/events/drm_print.h> +#include <drm/drm.h> #include <drm/drm_drv.h> #include <drm/drm_print.h> @@ -241,10 +245,10 @@ void drm_dev_printk(const struct device *dev, const char *level,struct va_format vaf;va_list args; -va_start (args, format); vaf.fmt = format; vaf.va = &args; +va_start (args, format);if (dev)dev_printk (level, dev,"[" DRM_NAME ":%ps] %pV" ,__builtin_return_address (0 ), &vaf);@@ -253,49 +257,145 @@ void drm_dev_printk(const struct device *dev, const char *level, level, __builtin_return_address(0), &vaf);va_end (args); + +va_start (args, format);
drivers/gpu/drm/drm_print.c void drm_dev_printk(const struct device *dev, const char *level, · line 271
Quoted 10 lines
drivers/gpu/drm/drm_print.c +7
-0+trace_drm_log (level, dev, &vaf); +va_end (args); }EXPORT_SYMBOL (drm_dev_printk); +static unsigned int drm_trace_enabled (unsigned int category) +{ +unsigned int bit; + +for_each_set_bit (bit, (unsigned long *)&category,sizeof (category) *8 ) {
You'll want to use BITS_PER_TYPE().
But wait, I've switched category to an enum upstream, and there should only ever be one bit set anyway?
Quoted 81 lines
drivers/gpu/drm/drm_print.c +71
-4+switch (BIT (bit)) { +case DRM_UT_NONE : +return trace_drm_dbg_none_enabled (); +case DRM_UT_CORE : +return trace_drm_dbg_core_enabled (); +case DRM_UT_DRIVER : +return trace_drm_dbg_driver_enabled (); +case DRM_UT_KMS : +return trace_drm_dbg_kms_enabled (); +case DRM_UT_PRIME : +return trace_drm_dbg_prime_enabled (); +case DRM_UT_ATOMIC : +return trace_drm_dbg_atomic_enabled (); +case DRM_UT_VBL : +return trace_drm_dbg_vbl_enabled (); +case DRM_UT_STATE : +return trace_drm_dbg_state_enabled (); +case DRM_UT_LEASE : +return trace_drm_dbg_lease_enabled (); +case DRM_UT_DP : +return trace_drm_dbg_dp_enabled (); +default : +return trace_drm_dbg_unknown_enabled (); + } + } +return false; +} + +static void drm_do_trace (const struct device *dev,unsigned int category, +struct va_format *vaf) +{ +WARN_ON (hweight32 (category) >1 ); + +switch (category) { +case DRM_UT_NONE : +trace_drm_dbg_none (dev, vaf); +break ; +case DRM_UT_CORE : +trace_drm_dbg_core (dev, vaf); +break ; +case DRM_UT_DRIVER : +trace_drm_dbg_driver (dev, vaf); +break ; +case DRM_UT_KMS : +trace_drm_dbg_kms (dev, vaf); +break ; +case DRM_UT_PRIME : +trace_drm_dbg_prime (dev, vaf); +break ; +case DRM_UT_ATOMIC : +trace_drm_dbg_atomic (dev, vaf); +break ; +case DRM_UT_VBL : +trace_drm_dbg_vbl (dev, vaf); +break ; +case DRM_UT_STATE : +trace_drm_dbg_state (dev, vaf); +break ; +case DRM_UT_LEASE : +trace_drm_dbg_lease (dev, vaf); +break ; +case DRM_UT_DP : +trace_drm_dbg_dp (dev, vaf); +break ; +default : +trace_drm_dbg_unknown (dev, vaf); +break ; + } +} +void drm_dev_dbg (const struct device *dev,unsigned int category,const char *format, ...) {struct va_format vaf; +unsigned int bit;va_list args; -if (!drm_debug_enabled (category)) -return ; - -va_start (args, format);
drivers/gpu/drm/drm_print.c void drm_dev_printk(const struct device *dev, const char *level, · line 352
Quoted 10 lines
drivers/gpu/drm/drm_print.c +1
-6vaf.fmt = format; vaf.va = &args; -if (dev) -dev_printk (KERN_DEBUG , dev,"[" DRM_NAME ":%ps] %pV" , -__builtin_return_address (0 ), &vaf); -else -printk (KERN_DEBUG "[" DRM_NAME ":%ps] %pV" , -__builtin_return_address (0 ), &vaf); +if (drm_debug_enabled (category)) {
Ville wants to move this check outside of the functions in the macro level in the header file. Apparently it's pretty bad for performance on some (older Atom) machines to do the call for nothing.
BR,
Jani.
Quoted 197 lines
drivers/gpu/drm/drm_print.c +37
-9+va_start (args, format); +if (dev) +dev_printk (KERN_DEBUG , dev,"[" DRM_NAME ":%ps] %pV" , +__builtin_return_address (0 ), &vaf); +else +printk (KERN_DEBUG "[" DRM_NAME ":%ps] %pV" , +__builtin_return_address (0 ), &vaf); +va_end (args); + } + +if (!drm_trace_enabled (category)) +return ; + +for_each_set_bit (bit, (unsigned long *)&category,sizeof (category) *8 ) { +va_start (args, format); +drm_do_trace (dev,BIT (bit), &vaf); +va_end (args); + } -va_end (args); }EXPORT_SYMBOL (drm_dev_dbg);void drm_dbg (unsigned int category,const char *format, ...) {struct va_format vaf; +unsigned int bit;va_list args; -if (!drm_debug_enabled (category)) -return ; - -va_start (args, format); vaf.fmt = format; vaf.va = &args; -printk (KERN_DEBUG "[" DRM_NAME ":%ps] %pV" , -__builtin_return_address (0 ), &vaf); +if (drm_debug_enabled (category)) { +va_start (args, format); +printk (KERN_DEBUG "[" DRM_NAME ":%ps] %pV" , +__builtin_return_address (0 ), &vaf); +va_end (args); + } -va_end (args); +if (!drm_trace_enabled (category)) +return ; + +for_each_set_bit (bit, (unsigned long *)&category,sizeof (category) *8 ) { +va_start (args, format); +drm_do_trace (NULL ,BIT (bit), &vaf); +va_end (args); + } }EXPORT_SYMBOL (drm_dbg);@@ -304,13 +404,16 @@ void drm_err(const char *format, ...)struct va_format vaf;va_list args; -va_start (args, format); vaf.fmt = format; vaf.va = &args; +va_start (args, format);printk (KERN_ERR "[" DRM_NAME ":%ps] *ERROR* %pV" ,__builtin_return_address (0 ), &vaf); +va_end (args); +va_start (args, format); +trace_drm_err (NULL , &vaf);va_end (args); }EXPORT_SYMBOL (drm_err);include/trace/events/drm_print.h new file +116
-0@@ -0,0 +1,116 @@ +// SPDX-License-Identifier: MIT +/* +* Copyright (C) 2019 Google, Inc. +* +* Authors: +* Sean Paul <[email protected]> +*/ +#undef TRACE_SYSTEM +#define TRACE_SYSTEM drm_print + +#if !defined(_TRACE_DRM_PRINT_H) || defined(TRACE_HEADER_MULTI_READ ) + +#include <linux/device.h> +#include <linux/tracepoint.h> + +#define DRM_PRINT_MAX 256 + +#define _TRACE_DRM_PRINT_H + +TRACE_EVENT (drm_log, + TP_PROTO(const char * level, const struct device *dev, + struct va_format *vaf), +TP_ARGS (level, dev, vaf), +TP_STRUCT__entry ( + __field(const char *, level ) + __field(const struct device *, dev ) + __dynamic_array(char, msg, DRM_PRINT_MAX ) +__field (int, len ) + ), +TP_fast_assign ( + __entry->level = level; + __entry->dev = dev; + __entry->len = vsnprintf(__get_str(msg), DRM_PRINT_MAX, + vaf->fmt, *vaf->va); + if (__entry->len > 0 && + __get_str(msg)[__entry->len - 1] == '\n') + __get_str(msg)[--__entry->len] = '\0'; + ), +TP_printk ("%s %s %s: %s" , __entry->level, + __entry->dev ?dev_driver_string (__entry->dev) :"(NULL):" , + __entry->dev ?dev_name (__entry->dev) :"" ,__get_str (msg)) +); + +DECLARE_EVENT_CLASS (class_drm_category_log, + TP_PROTO(const struct device *dev, struct va_format *vaf), +TP_ARGS (dev, vaf), +TP_STRUCT__entry ( + __field(const struct device *, dev ) + __dynamic_array(char, msg, DRM_PRINT_MAX ) +__field (int, len ) + ), +TP_fast_assign ( + __entry->dev = dev; + __entry->len = vsnprintf(__get_str(msg), DRM_PRINT_MAX, + vaf->fmt, *vaf->va); + if (__entry->len > 0 && + __get_str(msg)[__entry->len - 1] == '\n') + __get_str(msg)[--__entry->len] = '\0'; + ), +TP_printk ("%s %s%s%s" , + __entry->dev ?dev_driver_string (__entry->dev) :"" , + __entry->dev ?dev_name (__entry->dev) :"" , + __entry->dev ?": " :"" ,__get_str (msg)) +); +DEFINE_EVENT (class_drm_category_log, drm_err, + TP_PROTO(const struct device *dev, struct va_format *vaf), +TP_ARGS (dev, vaf) +); +DEFINE_EVENT (class_drm_category_log, drm_dbg_none, + TP_PROTO(const struct device *dev, struct va_format *vaf), +TP_ARGS (dev, vaf) +); +DEFINE_EVENT (class_drm_category_log, drm_dbg_core, + TP_PROTO(const struct device *dev, struct va_format *vaf), +TP_ARGS (dev, vaf) +); +DEFINE_EVENT (class_drm_category_log, drm_dbg_driver, + TP_PROTO(const struct device *dev, struct va_format *vaf), +TP_ARGS (dev, vaf) +); +DEFINE_EVENT (class_drm_category_log, drm_dbg_kms, + TP_PROTO(const struct device *dev, struct va_format *vaf), +TP_ARGS (dev, vaf) +); +DEFINE_EVENT (class_drm_category_log, drm_dbg_prime, + TP_PROTO(const struct device *dev, struct va_format *vaf), +TP_ARGS (dev, vaf) +); +DEFINE_EVENT (class_drm_category_log, drm_dbg_atomic, + TP_PROTO(const struct device *dev, struct va_format *vaf), +TP_ARGS (dev, vaf) +); +DEFINE_EVENT (class_drm_category_log, drm_dbg_vbl, + TP_PROTO(const struct device *dev, struct va_format *vaf), +TP_ARGS (dev, vaf) +); +DEFINE_EVENT (class_drm_category_log, drm_dbg_state, + TP_PROTO(const struct device *dev, struct va_format *vaf), +TP_ARGS (dev, vaf) +); +DEFINE_EVENT (class_drm_category_log, drm_dbg_lease, + TP_PROTO(const struct device *dev, struct va_format *vaf), +TP_ARGS (dev, vaf) +); +DEFINE_EVENT (class_drm_category_log, drm_dbg_dp, + TP_PROTO(const struct device *dev, struct va_format *vaf), +TP_ARGS (dev, vaf) +); +DEFINE_EVENT (class_drm_category_log, drm_dbg_unknown, + TP_PROTO(const struct device *dev, struct va_format *vaf), +TP_ARGS (dev, vaf) +); +#endif + +/* This part must be outside protection */ +#include <trace/define_trace.h>
Signature · 5 lines
Jani Nikula, Intel Open Source Graphics Center _______________________________________________ dri-devel mailing list [email protected] https://lists.freedesktop.org/mailman/listinfo/dri-devel
Quoted 201 lines · Jani Nikula
Quoted 100 lines · Sean Paul
From: Sean Paul <[email protected]>
For a long while now, we (ChromeOS) have been struggling getting any value out of user feedback reports of display failures (notably external displays not working). The problem is that all logging, even fatal errors (well, fatal in the sense that a display won't light up) are logged at DEBUG log level. So in order to extract these logs, users need to be able to turn on logging, and reproduce the issue with debug enabled. Unfortunately, this isn't really something we can ask CrOS users to do. I spoke with airlied about this and RHEL has similar issues. After a few more people piped up on previous versions of this patch, it is a Real Issue.
So why don't we just enable
DRM_UT_BLAH? Here are the reasons in ascending order of severity:
1- People aren't consistent with their categories, so we'd have to
enable a bunch to get proper coverage
2- We don't want to overwhelm syslog with drm spam, others have to use
it too
3- Console logging is slowSo what we really want is a ringbuffer of the most recent logs (filtered by categories we're interested in) exposed via debugfs so the logs can be extracted when users file feedback.
It just so happens that there is something which does _exactly_ this! This patch dumps drm logs into tracepoints, which allows us to turn tracing on and off depending on which category is useful, and pull them from tracefs on demand.
What about
trace_printk()? It doesn't give us the control we get from using tracepoints and it's not meant to be left sprinkled around in code.Cc David Airlie <[email protected]>Cc Daniel Vetter <[email protected]>Cc Pekka Paalanen <[email protected]>Cc Joonas Lahtinen <[email protected]>Cc Thomas Zimmermann <[email protected]>Cc Rob Clark <[email protected]>Cc Ville Syrjälä <[email protected]>Acked-by Daniel Vetter <[email protected]>Signed-off-by Sean Paul <[email protected]>Changes in v2:
- Went with a completely different approach: https://lists.freedesktop.org/archives/dri-devel/2019-November/243230.htmlChanges in v3:
- Changed commit message to be a bit less RFC-y
- Makeclass_drm_category_logan actual trace class
---Even though we don't want it to be, this is UAPI. So here's some userspace code which uses it:
https://chromium-review.googlesource.com/c/chromiumos/platform2/+/1965562drivers/gpu/drm/drm_print.c | 143 ++++++++++++++++++++++++++----- include/trace/events/drm_print.h | 116 +++++++++++++++++++++++++ 2 files changed, 239 insertions(+), 20 deletions(-) create mode 100644 include/trace/events/drm_print.hdrivers/gpu/drm/drm_print.c +7
-1@@ -27,11 +27,15 @@#include <stdarg.h> +#include <linux/bitops.h> #include <linux/io.h> #include <linux/moduleparam.h> #include <linux/seq_file.h> #include <linux/slab.h> +#define CREATE_TRACE_POINTS +#include <trace/events/drm_print.h> +#include <drm/drm.h> #include <drm/drm_drv.h> #include <drm/drm_print.h> @@ -241,10 +245,10 @@ void drm_dev_printk(const struct device *dev, const char *level,struct va_format vaf;va_list args; -va_start (args, format); vaf.fmt = format; vaf.va = &args; +va_start (args, format);if (dev)dev_printk (level, dev,"[" DRM_NAME ":%ps] %pV" ,__builtin_return_address (0 ), &vaf);@@ -253,49 +257,145 @@ void drm_dev_printk(const struct device *dev, const char *level, level, __builtin_return_address(0), &vaf);va_end (args); + +va_start (args, format);drivers/gpu/drm/drm_print.c void drm_dev_printk(const struct device *dev, const char *level, · line 271
Quoted 10 lines
drivers/gpu/drm/drm_print.c +7
-0+trace_drm_log (level, dev, &vaf); +va_end (args); }EXPORT_SYMBOL (drm_dev_printk); +static unsigned int drm_trace_enabled (unsigned int category) +{ +unsigned int bit; + +for_each_set_bit (bit, (unsigned long *)&category,sizeof (category) *8 ) {You'll want to use
BITS_PER_TYPE().But wait, I've switched category to an enum upstream, and there should only ever be one bit set anyway?
Quoted 84 lines
drivers/gpu/drm/drm_print.c +71
-4+switch (BIT (bit)) { +case DRM_UT_NONE : +return trace_drm_dbg_none_enabled (); +case DRM_UT_CORE : +return trace_drm_dbg_core_enabled (); +case DRM_UT_DRIVER : +return trace_drm_dbg_driver_enabled (); +case DRM_UT_KMS : +return trace_drm_dbg_kms_enabled (); +case DRM_UT_PRIME : +return trace_drm_dbg_prime_enabled (); +case DRM_UT_ATOMIC : +return trace_drm_dbg_atomic_enabled (); +case DRM_UT_VBL : +return trace_drm_dbg_vbl_enabled (); +case DRM_UT_STATE : +return trace_drm_dbg_state_enabled (); +case DRM_UT_LEASE : +return trace_drm_dbg_lease_enabled (); +case DRM_UT_DP : +return trace_drm_dbg_dp_enabled (); +default : +return trace_drm_dbg_unknown_enabled (); + } + } +return false; +} + +static void drm_do_trace (const struct device *dev,unsigned int category, +struct va_format *vaf) +{ +WARN_ON (hweight32 (category) >1 ); + +switch (category) { +case DRM_UT_NONE : +trace_drm_dbg_none (dev, vaf); +break ; +case DRM_UT_CORE : +trace_drm_dbg_core (dev, vaf); +break ; +case DRM_UT_DRIVER : +trace_drm_dbg_driver (dev, vaf); +break ; +case DRM_UT_KMS : +trace_drm_dbg_kms (dev, vaf); +break ; +case DRM_UT_PRIME : +trace_drm_dbg_prime (dev, vaf); +break ; +case DRM_UT_ATOMIC : +trace_drm_dbg_atomic (dev, vaf); +break ; +case DRM_UT_VBL : +trace_drm_dbg_vbl (dev, vaf); +break ; +case DRM_UT_STATE : +trace_drm_dbg_state (dev, vaf); +break ; +case DRM_UT_LEASE : +trace_drm_dbg_lease (dev, vaf); +break ; +case DRM_UT_DP : +trace_drm_dbg_dp (dev, vaf); +break ; +default : +trace_drm_dbg_unknown (dev, vaf); +break ; + } +} +void drm_dev_dbg (const struct device *dev,unsigned int category,const char *format, ...) {struct va_format vaf; +unsigned int bit;va_list args; -if (!drm_debug_enabled (category)) -return ; - -va_start (args, format); vaf.fmt = format; vaf.va = &args;
Quoted 11 lines
drivers/gpu/drm/drm_print.c void drm_dev_printk(const struct device *dev, const char *level, · line 352
Quoted 7 lines
drivers/gpu/drm/drm_print.c +1
-6-if (dev) -dev_printk (KERN_DEBUG , dev,"[" DRM_NAME ":%ps] %pV" , -__builtin_return_address (0 ), &vaf); -else -printk (KERN_DEBUG "[" DRM_NAME ":%ps] %pV" , -__builtin_return_address (0 ), &vaf); +if (drm_debug_enabled (category)) {Ville wants to move this check outside of the functions in the macro level in the header file. Apparently it's pretty bad for performance on some (older Atom) machines to do the call for nothing.
I've been meaning to measure if this actually matters. But haven't managed to motivate myself enough. Could be a good project for some random person to write a test that just does tons of different kinds of atomic TEST_ONLY commits to measure how badly we suck.
Signature · 6 lines
Ville Syrjälä Intel _______________________________________________ dri-devel mailing list [email protected] https://lists.freedesktop.org/mailman/listinfo/dri-devel