tracing: add error_report trace points
Subsystems (5)
A message that is not in the archive (sent off-list or not yet received): [email protected]
Alexander Potapenko
maintainerPATCH 1/4 tracing: add error_report trace points
Introduce error_report_start and error_report_end tracepoints. Those can be used in debugging tools like KASAN, KFENCE, etc. to provide extensions to the error reporting mechanisms (e.g. allow tests hook into error reporting, ease error report collection from production kernels).
Another benefit would be making use of ftrace for debugging or benchmarking the tools themselves.
--- include/trace/events/error_report.h | 51 +++++++++++++++++++++++++++++ kernel/trace/Makefile | 1 + kernel/trace/error_report-traces.c | 11 +++++++ 3 files changed, 63 insertions(+) create mode 100644 include/trace/events/error_report.h create mode 100644 kernel/trace/error_report-traces.c
include/trace/events/error_report.h new file +51 -0
@@ -0,0 +1,51 @@ +/* SPDX-License-Identifier: GPL-2.0-only */ +#undef TRACE_SYSTEM +#define TRACE_SYSTEM error_report + +#if !defined(_TRACE_ERROR_REPORT_H) || defined(TRACE_HEADER_MULTI_READ ) +#define _TRACE_ERROR_REPORT_H + +#include <linux/tracepoint.h> + +DECLARE_EVENT_CLASS (error_report_template, + TP_PROTO(const char *error_detector, unsigned long id), +TP_ARGS (error_detector, id), + TP_STRUCT__entry(__field(const char *, error_detector) + __field(unsigned long, id)), + TP_fast_assign(__entry->error_detector = error_detector; + __entry->id = id;), +TP_printk ("[%s] %lx" , __entry->error_detector, + __entry->id)); + +/** +* error_report_start - called before printing the error report +* @error_detector: short string describing the error detection tool +* @id: pseudo-unique descriptor that can help distinguish reports +* from one another. Depending on the tool, good examples +* could be: memory access address, call site, allocation +* site, etc. +* +* This event occurs right before a debugging tool starts printing the error +* report. +*/ +DEFINE_EVENT (error_report_template, error_report_start, + TP_PROTO(const char *error_detector, unsigned long id), +TP_ARGS (error_detector, id)); + +/** +* error_report_end - called after printing the error report +* @error_detector: short string describing the error detection tool +* @id: pseudo-unique descriptor, matches that passed to +* error_report_start +* +* This event occurs right after a debugging tool finishes printing the error +* report. +*/ +DEFINE_EVENT (error_report_template, error_report_end, + TP_PROTO(const char *error_detector, unsigned long id), +TP_ARGS (error_detector, id)); + +#endif /* _TRACE_ERROR_REPORT_H */ + +/* This part must be outside protection */ +#include <trace/define_trace.h>
kernel/trace/Makefile +1 -0
@@ -81,6 +81,7 @@ obj-$(CONFIG_SYNTH_EVENTS) += trace_events_synth.o obj-$(CONFIG_HIST_TRIGGERS) += trace_events_hist.o obj-$(CONFIG_BPF_EVENTS) += bpf_trace.o obj-$(CONFIG_KPROBE_EVENTS) += trace_kprobe.o +obj-$(CONFIG_TRACEPOINTS) += error_report-traces.o obj-$(CONFIG_TRACEPOINTS) += power-traces.o ifeq ($(CONFIG_PM),y) obj-$(CONFIG_TRACEPOINTS) += rpm-traces.o
kernel/trace/error_report-traces.c new file +11 -0
@@ -0,0 +1,11 @@ +// SPDX-License-Identifier: GPL-2.0 +/* +* Error reporting trace points +*/ + +#define CREATE_TRACE_POINTS +#include <trace/events/error_report.h> + +EXPORT_TRACEPOINT_SYMBOL_GPL (error_report_start); +EXPORT_TRACEPOINT_SYMBOL_GPL (error_report_end); +
Signature · 1 line
2.30.0.284.gd98b1dd5eaa7-goog
Alexander Potapenko
maintainerPATCH 2/4 lib: add error_report_notify to collect debugging tools' reports
In reply to a message not in the archiveWith the introduction of various production error-detection tools, such as MTE-based KASAN and KFENCE, the need arises to efficiently notify the userspace OS components about kernel errors. Currently, no facility exists to notify userspace about a kernel error from such bug-detection tools. The problem is obviously not restricted to the above bug detection tools, and applies to any error reporting mechanism that does not panic the kernel; this series, however, will only add support for KASAN and KFENCE reporting.
All such error reports appear in the kernel log. But, when such errors occur, userspace would normally need to read the entire kernel log and parse the relevant errors. This is error prone and inefficient, as userspace needs to continuously monitor the kernel log for error messages. On certain devices, this is unfortunately not acceptable. Therefore, we need to revisit how reports are propagated to userspace.
The library added, error_report_notify (CONFIG_ERROR_REPORT_NOTIFY), solves the above by using the error_report_start/error_report_end tracing events and exposing the last report and the total report count to the userspace via /sys/kernel/error_report/last_report and /sys/kernel/error_report/report_count.
Userspace apps can call poll(POLLPRI) on those files to get notified about the new reports without having to watch dmesg in a loop.
--- lib/Kconfig.debug | 14 ++ lib/Makefile | 2 + lib/error_report_notify.c | 278 ++++++++++++++++++++++++++++++++++++++ 3 files changed, 294 insertions(+) create mode 100644 lib/error_report_notify.c
lib/Kconfig.debug +14 -0
@@ -209,6 +209,20 @@ config DEBUG_BUGVERBOSE of the BUG call as well as the EIP and oops trace. This aids debugging but costs about 70-100K of memory. +config ERROR_REPORT_NOTIFY + bool "Expose memory error reports to the userspace" + depends on TRACING + help + When enabled, captures error reports from debugging tools (such as + KFENCE or KASAN) using console tracing, and exposes reports in + /sys/kernel/error_report/: the file last_report contains the last + report (with maximum report length of PAGE_SIZE), and report_count, + the total report count. + + Userspace programs can call poll(POLLPRI) on those files to get + notified about the new reports without having to watch dmesg in a + loop. + endmenu # "printk and dmesg options" menu "Compile-time checks and compiler options"
lib/Makefile +2 -0
@@ -240,6 +240,8 @@ obj-$(CONFIG_CPU_RMAP) += cpu_rmap.o obj-$(CONFIG_DQL) += dynamic_queue_limits.o +obj-$(CONFIG_ERROR_REPORT_NOTIFY) += error_report_notify.o + obj-$(CONFIG_GLOB) += glob.o obj-$(CONFIG_GLOB_SELFTEST) += globtest.o
lib/error_report_notify.c new file +278 -0
278 lines not on this page · raw message
Signature · 1 line
2.30.0.284.gd98b1dd5eaa7-goog
Alexander Potapenko
maintainerRe: [PATCH 2/4] lib: add error_report_notify to collect debugging tools' reports
Sorry for the noise, had to resend with LKML in CC.
Alexander Potapenko
maintainerPATCH 3/4 kfence: use error_report_start and error_report_end tracepoints
In reply to a message not in the archiveMake it possible to trace KFENCE error reporting (in particular, for compatibility with CONFIG_ERROR_REPORT_NOTIFY).
--- mm/kfence/report.c | 3 +++ 1 file changed, 3 insertions(+)
mm/kfence/report.c +3 -0
@@ -9,6 +9,7 @@#include <linux/seq_file.h> #include <linux/stacktrace.h> #include <linux/string.h> +#include <trace/events/error_report.h> #include <asm/kfence.h>
@@ -184,6 +185,7 @@ void kfence_report_error(unsigned long address, struct pt_regs *regs,lockdep_off ();pr_err ("==================================================================\n" ); +trace_error_report_start ("kfence" , address);/* Print report header. */ switch (type) {case KFENCE_ERROR_OOB : {
@@ -232,6 +234,7 @@ void kfence_report_error(unsigned long address, struct pt_regs *regs,show_regs (regs); elsedump_stack_print_info (KERN_ERR ); +trace_error_report_end ("kfence" , address);pr_err ("==================================================================\n" );lockdep_on ();
Signature · 1 line
2.30.0.284.gd98b1dd5eaa7-goog
Alexander Potapenko
maintainerPATCH 4/4 kasan: use error_report_start and error_report_end tracepoints
In reply to a message not in the archiveMake it possible to trace KASAN error reporting (in particular, for compatibility with CONFIG_ERROR_REPORT_NOTIFY).
--- mm/kasan/report.c | 15 +++++++++------ 1 file changed, 9 insertions(+), 6 deletions(-)
mm/kasan/report.c +9 -6
@@ -25,6 +25,7 @@#include <linux/module.h> #include <linux/sched/task_stack.h> #include <linux/uaccess.h> +#include <trace/events/error_report.h> #include <asm/sections.h>
@@ -74,7 +75,7 @@ static void print_error_description(struct kasan_access_info *info)static DEFINE_SPINLOCK (report_lock); -static void start_report (unsigned long *flags) +static void start_report (unsigned long *flags,unsigned long addr) {/* * Make sure we don't end up in loop.
@@ -82,10 +83,12 @@ static void start_report(unsigned long *flags)kasan_disable_current ();spin_lock_irqsave (&report_lock, *flags);pr_err ("==================================================================\n" ); +trace_error_report_start ("kasan" , addr); } -static void end_report (unsigned long *flags) +static void end_report (unsigned long *flags,unsigned long addr) { +trace_error_report_end ("kasan" , addr);pr_err ("==================================================================\n" );add_taint (TAINT_BAD_PAGE ,LOCKDEP_NOW_UNRELIABLE );spin_unlock_irqrestore (&report_lock, *flags);
@@ -348,14 +351,14 @@ void kasan_report_invalid_free(void *object, unsigned long ip)kasan_update_kunit_status (current->kunit_test);#endif /* IS_ENABLED(CONFIG_KUNIT) */ -start_report (&flags); +start_report (&flags, (unsigned long )object);pr_err ("BUG: KASAN: double-free or invalid-free in %pS\n" , (void *)ip);print_tags (tag, object);pr_err ("\n" );print_address_description (object, tag);pr_err ("\n" );print_memory_metadata (object); -end_report (&flags); +end_report (&flags, (unsigned long )object); } static void __kasan_report(unsigned long addr, size_t size, bool is_write,
@@ -385,7 +388,7 @@ static void __kasan_report(unsigned long addr, size_t size, bool is_write, info.is_write = is_write; info.ip = ip; -start_report (&flags); +start_report (&flags, addr);print_error_description (&info);if (addr_has_metadata (untagged_addr))
@@ -400,7 +403,7 @@ static void __kasan_report(unsigned long addr, size_t size, bool is_write,dump_stack (); } -end_report (&flags); +end_report (&flags, addr); } bool kasan_report(unsigned long addr, size_t size, bool is_write,
Signature · 1 line
2.30.0.284.gd98b1dd5eaa7-goog