Skip to contentSkip to conversation

tracing: add error_report trace points

PATCH In review Alexander Potapenko 1 participant 5 messages Jan 13, 2021 1 message not in the archive

Subsystems (5)
lore
Share & tools
lore

A message that is not in the archive (sent off-list or not yet received): [email protected]

Alexander Potapenko

author maintainer
#1 lore raw

PATCH 1/4 tracing: add error_report trace points

Introduce error_report_start and error_report_end tracepoints. Those can be used in debugging tools like KASAN, KFENCE, etc. to provide extensions to the error reporting mechanisms (e.g. allow tests hook into error reporting, ease error report collection from production kernels).
Another benefit would be making use of ftrace for debugging or benchmarking the tools themselves.

Cc Andrew Morton <[email protected]>
Cc Andrey Konovalov <[email protected]>
Cc Dmitry Vyukov <[email protected]>
Cc Ingo Molnar <[email protected]>
Cc Marco Elver <[email protected]>
Cc Petr Mladek <[email protected]>
Cc Steven Rostedt <[email protected]>
Cc Sergey Senozhatsky <[email protected]>
---
 include/trace/events/error_report.h | 51 +++++++++++++++++++++++++++++
 kernel/trace/Makefile               |  1 +
 kernel/trace/error_report-traces.c  | 11 +++++++
 3 files changed, 63 insertions(+)
 create mode 100644 include/trace/events/error_report.h
 create mode 100644 kernel/trace/error_report-traces.c
include/trace/events/error_report.h new file +51 -0
new file mode 100644
index 000000000000..bae0129031f0
@@ -0,0 +1,51 @@
+/* SPDX-License-Identifier: GPL-2.0-only */
+#undef TRACE_SYSTEM
+#define TRACE_SYSTEM error_report
+
+#if !defined(_TRACE_ERROR_REPORT_H) || defined(TRACE_HEADER_MULTI_READ)
+#define _TRACE_ERROR_REPORT_H
+
+#include <linux/tracepoint.h>
+
+DECLARE_EVENT_CLASS(error_report_template,
+		    TP_PROTO(const char *error_detector, unsigned long id),
+		    TP_ARGS(error_detector, id),
+		    TP_STRUCT__entry(__field(const char *, error_detector)
+					     __field(unsigned long, id)),
+		    TP_fast_assign(__entry->error_detector = error_detector;
+				   __entry->id = id;),
+		    TP_printk("[%s] %lx", __entry->error_detector,
+			      __entry->id));
+
+/**
+ * error_report_start - called before printing the error report
+ * @error_detector:	short string describing the error detection tool
+ * @id:			pseudo-unique descriptor that can help distinguish reports
+ *			from one another. Depending on the tool, good examples
+ *			could be: memory access address, call site, allocation
+ *			site, etc.
+ *
+ * This event occurs right before a debugging tool starts printing the error
+ * report.
+ */
+DEFINE_EVENT(error_report_template, error_report_start,
+	     TP_PROTO(const char *error_detector, unsigned long id),
+	     TP_ARGS(error_detector, id));
+
+/**
+ * error_report_end - called after printing the error report
+ * @error_detector:	short string describing the error detection tool
+ * @id:			pseudo-unique descriptor, matches that passed to
+ *			error_report_start
+ *
+ * This event occurs right after a debugging tool finishes printing the error
+ * report.
+ */
+DEFINE_EVENT(error_report_template, error_report_end,
+	     TP_PROTO(const char *error_detector, unsigned long id),
+	     TP_ARGS(error_detector, id));
+
+#endif /* _TRACE_ERROR_REPORT_H */
+
+/* This part must be outside protection */
+#include <trace/define_trace.h>
kernel/trace/Makefile +1 -0
index 7e44cea89fdc..b28d3e5013cd 100644
@@ -81,6 +81,7 @@ obj-$(CONFIG_SYNTH_EVENTS) += trace_events_synth.o
 obj-$(CONFIG_HIST_TRIGGERS) += trace_events_hist.o
 obj-$(CONFIG_BPF_EVENTS) += bpf_trace.o
 obj-$(CONFIG_KPROBE_EVENTS) += trace_kprobe.o
+obj-$(CONFIG_TRACEPOINTS) += error_report-traces.o
 obj-$(CONFIG_TRACEPOINTS) += power-traces.o
 ifeq ($(CONFIG_PM),y)
 obj-$(CONFIG_TRACEPOINTS) += rpm-traces.o
kernel/trace/error_report-traces.c new file +11 -0
new file mode 100644
index 000000000000..80960c52c705
@@ -0,0 +1,11 @@
+// SPDX-License-Identifier: GPL-2.0
+/*
+ * Error reporting trace points
+ */
+
+#define CREATE_TRACE_POINTS
+#include <trace/events/error_report.h>
+
+EXPORT_TRACEPOINT_SYMBOL_GPL(error_report_start);
+EXPORT_TRACEPOINT_SYMBOL_GPL(error_report_end);
+
Signature · 1 line
2.30.0.284.gd98b1dd5eaa7-goog
author maintainer
#2 lore raw

PATCH 2/4 lib: add error_report_notify to collect debugging tools' reports

In reply to a message not in the archive

With the introduction of various production error-detection tools, such as MTE-based KASAN and KFENCE, the need arises to efficiently notify the userspace OS components about kernel errors. Currently, no facility exists to notify userspace about a kernel error from such bug-detection tools. The problem is obviously not restricted to the above bug detection tools, and applies to any error reporting mechanism that does not panic the kernel; this series, however, will only add support for KASAN and KFENCE reporting.

All such error reports appear in the kernel log. But, when such errors occur, userspace would normally need to read the entire kernel log and parse the relevant errors. This is error prone and inefficient, as userspace needs to continuously monitor the kernel log for error messages. On certain devices, this is unfortunately not acceptable. Therefore, we need to revisit how reports are propagated to userspace.

The library added, error_report_notify (CONFIG_ERROR_REPORT_NOTIFY), solves the above by using the error_report_start/error_report_end tracing events and exposing the last report and the total report count to the userspace via /sys/kernel/error_report/last_report and /sys/kernel/error_report/report_count.

Userspace apps can call poll(POLLPRI) on those files to get notified about the new reports without having to watch dmesg in a loop.

Cc Andrew Morton <[email protected]>
Cc Andrey Konovalov <[email protected]>
Cc Dmitry Vyukov <[email protected]>
Cc Ingo Molnar <[email protected]>
Cc Marco Elver <[email protected]>
Cc Petr Mladek <[email protected]>
Cc Steven Rostedt <[email protected]>
Cc Sergey Senozhatsky <[email protected]>
---
 lib/Kconfig.debug         |  14 ++
 lib/Makefile              |   2 +
 lib/error_report_notify.c | 278 ++++++++++++++++++++++++++++++++++++++
 3 files changed, 294 insertions(+)
 create mode 100644 lib/error_report_notify.c
lib/Kconfig.debug +14 -0
index 4b8f9e018f0f..c3f4aae9f19e 100644
@@ -209,6 +209,20 @@ config DEBUG_BUGVERBOSE
 	  of the BUG call as well as the EIP and oops trace.  This aids
 	  debugging but costs about 70-100K of memory.
 
+config ERROR_REPORT_NOTIFY
+	bool "Expose memory error reports to the userspace"
+	depends on TRACING
+	help
+	  When enabled, captures error reports from debugging tools (such as
+	  KFENCE or KASAN) using console tracing, and exposes reports in
+	  /sys/kernel/error_report/: the file last_report contains the last
+	  report (with maximum report length of PAGE_SIZE), and report_count,
+	  the total report count.
+
+	  Userspace programs can call poll(POLLPRI) on those files to get
+	  notified about the new reports without having to watch dmesg in a
+	  loop.
+
 endmenu # "printk and dmesg options"
 
 menu "Compile-time checks and compiler options"
lib/Makefile +2 -0
index afeff05fa8c5..96ed8bc0cc5b 100644
@@ -240,6 +240,8 @@ obj-$(CONFIG_CPU_RMAP) += cpu_rmap.o
 
 obj-$(CONFIG_DQL) += dynamic_queue_limits.o
 
+obj-$(CONFIG_ERROR_REPORT_NOTIFY) += error_report_notify.o
+
 obj-$(CONFIG_GLOB) += glob.o
 obj-$(CONFIG_GLOB_SELFTEST) += globtest.o
 
lib/error_report_notify.c new file +278 -0

278 lines not on this page · raw message

Signature · 1 line
2.30.0.284.gd98b1dd5eaa7-goog
author maintainer
#3 lore raw

Re: [PATCH 2/4] lib: add error_report_notify to collect debugging tools' reports

Sorry for the noise, had to resend with LKML in CC.

author maintainer
#4 lore raw

PATCH 3/4 kfence: use error_report_start and error_report_end tracepoints

In reply to a message not in the archive

Make it possible to trace KFENCE error reporting (in particular, for compatibility with CONFIG_ERROR_REPORT_NOTIFY).

Cc Andrew Morton <[email protected]>
Cc Andrey Konovalov <[email protected]>
Cc Dmitry Vyukov <[email protected]>
Cc Ingo Molnar <[email protected]>
Cc Marco Elver <[email protected]>
Cc Petr Mladek <[email protected]>
Cc Steven Rostedt <[email protected]>
Cc Sergey Senozhatsky <[email protected]>
---
 mm/kfence/report.c | 3 +++
 1 file changed, 3 insertions(+)
mm/kfence/report.c +3 -0
index 4dedc2ff8f28..5c063b5b2227 100644
@@ -9,6 +9,7 @@
 #include <linux/seq_file.h>
 #include <linux/stacktrace.h>
 #include <linux/string.h>
+#include <trace/events/error_report.h>
 
 #include <asm/kfence.h>
 
@@ -184,6 +185,7 @@ void kfence_report_error(unsigned long address, struct pt_regs *regs,
 	lockdep_off();
 
 	pr_err("==================================================================\n");
+	trace_error_report_start("kfence", address);
 	/* Print report header. */
 	switch (type) {
 	case KFENCE_ERROR_OOB: {
@@ -232,6 +234,7 @@ void kfence_report_error(unsigned long address, struct pt_regs *regs,
 		show_regs(regs);
 	else
 		dump_stack_print_info(KERN_ERR);
+	trace_error_report_end("kfence", address);
 	pr_err("==================================================================\n");
 
 	lockdep_on();
Signature · 1 line
2.30.0.284.gd98b1dd5eaa7-goog
author maintainer
#5 lore raw

PATCH 4/4 kasan: use error_report_start and error_report_end tracepoints

In reply to a message not in the archive

Make it possible to trace KASAN error reporting (in particular, for compatibility with CONFIG_ERROR_REPORT_NOTIFY).

Cc Andrew Morton <[email protected]>
Cc Andrey Konovalov <[email protected]>
Cc Dmitry Vyukov <[email protected]>
Cc Ingo Molnar <[email protected]>
Cc Marco Elver <[email protected]>
Cc Petr Mladek <[email protected]>
Cc Steven Rostedt <[email protected]>
Cc Sergey Senozhatsky <[email protected]>
---
 mm/kasan/report.c | 15 +++++++++------
 1 file changed, 9 insertions(+), 6 deletions(-)
mm/kasan/report.c +9 -6
index c0fb21797550..fd015dd5dd39 100644
@@ -25,6 +25,7 @@
 #include <linux/module.h>
 #include <linux/sched/task_stack.h>
 #include <linux/uaccess.h>
+#include <trace/events/error_report.h>
 
 #include <asm/sections.h>
 
@@ -74,7 +75,7 @@ static void print_error_description(struct kasan_access_info *info)
 
 static DEFINE_SPINLOCK(report_lock);
 
-static void start_report(unsigned long *flags)
+static void start_report(unsigned long *flags, unsigned long addr)
 {
 	/*
 	 * Make sure we don't end up in loop.
@@ -82,10 +83,12 @@ static void start_report(unsigned long *flags)
 	kasan_disable_current();
 	spin_lock_irqsave(&report_lock, *flags);
 	pr_err("==================================================================\n");
+	trace_error_report_start("kasan", addr);
 }
 
-static void end_report(unsigned long *flags)
+static void end_report(unsigned long *flags, unsigned long addr)
 {
+	trace_error_report_end("kasan", addr);
 	pr_err("==================================================================\n");
 	add_taint(TAINT_BAD_PAGE, LOCKDEP_NOW_UNRELIABLE);
 	spin_unlock_irqrestore(&report_lock, *flags);
@@ -348,14 +351,14 @@ void kasan_report_invalid_free(void *object, unsigned long ip)
 		kasan_update_kunit_status(current->kunit_test);
 #endif /* IS_ENABLED(CONFIG_KUNIT) */
 
-	start_report(&flags);
+	start_report(&flags, (unsigned long)object);
 	pr_err("BUG: KASAN: double-free or invalid-free in %pS\n", (void *)ip);
 	print_tags(tag, object);
 	pr_err("\n");
 	print_address_description(object, tag);
 	pr_err("\n");
 	print_memory_metadata(object);
-	end_report(&flags);
+	end_report(&flags, (unsigned long)object);
 }
 
 static void __kasan_report(unsigned long addr, size_t size, bool is_write,
@@ -385,7 +388,7 @@ static void __kasan_report(unsigned long addr, size_t size, bool is_write,
 	info.is_write = is_write;
 	info.ip = ip;
 
-	start_report(&flags);
+	start_report(&flags, addr);
 
 	print_error_description(&info);
 	if (addr_has_metadata(untagged_addr))
@@ -400,7 +403,7 @@ static void __kasan_report(unsigned long addr, size_t size, bool is_write,
 		dump_stack();
 	}
 
-	end_report(&flags);
+	end_report(&flags, addr);
 }
 
 bool kasan_report(unsigned long addr, size_t size, bool is_write,
Signature · 1 line
2.30.0.284.gd98b1dd5eaa7-goog