Skip to contentSkip to conversation

RFC: x86/mce: Deprecate mcelog and other funsies

PATCH In review Borislav Petkov 3 participants 8 messages Jun 7, 2016 – Jun 13, 2016

Subsystems (2)
lore
Share & tools
lore

Borislav Petkov

author maintainer
#0 lore raw

cover RFC: x86/mce: Deprecate mcelog and other funsies

From: Borislav Petkov <[email protected]>

So this set cannot have a bigger "RFC" tag:

It attempts to revive the correctable errors collector. In this version, the thing gets counters in debugfs:

(debugfs)/ras/cec/
|-- array
`-- pfn

which give some stats on the CEC like how many CEs it caught and the number of pages it soft-offlined due to excessive amount of CEs.

$ cat ras/cec/array
{ n: 2
 000: [0000000000000012|001]
 001: [0000003735928559|e01]
}
Stats:
 CEs: 514
 offlined pages: 0
Flags: 0x0

Then, we stuck all the /dev/mcelog functionality behind a CONFIG_X86_MCELOG which people can disable if they don't need mcelog and are going to use the rasdaemon. This is probably going to happen sooner or later with the distros anyway.

And then, as a result of that move, mce-inject is being made independent from mcelog and gets the error injection debugfs nodes of mce_amd_inj which you can echo into and don't need special tools:

$ tree --charset=ascii /sys/kernel/debug/mce-inject/
/sys/kernel/debug/mce-inject/
|-- addr
|-- bank
|-- cpu
|-- flags
|-- misc
|-- README
`-- status

0 directories, 7 files

Comments are more than welcome, Tony and will be polishing on those but thought it would be a nice point in time to send them out to get other people's opinions.

Thanks.

Borislav Petkov (3):
RAS: Add a Corrected Errors Collector
x86/mce: Merge mce_amd_inj into mce-inject
x86/mce-inject: Use debugfs_remove_recursive()

Tony Luck (1):
x86/mce: Deprecate /dev/mcelog

 arch/x86/Kconfig                          |   8 +
 arch/x86/kernel/cpu/mcheck/Makefile       |   2 +
 arch/x86/kernel/cpu/mcheck/dev-mcelog.c   | 397 +++++++++++++++++++++++
 arch/x86/kernel/cpu/mcheck/mce-inject.c   | 519 +++++++++++++++++++++++++++++-
 arch/x86/kernel/cpu/mcheck/mce-internal.h |   8 +
 arch/x86/kernel/cpu/mcheck/mce.c          | 513 +++++------------------------
 arch/x86/ras/Kconfig                      |  11 +
 arch/x86/ras/Makefile                     |   2 -
 arch/x86/ras/mce_amd_inj.c                | 484 ----------------------------
 drivers/ras/Makefile                      |   3 +-
 drivers/ras/cec.c                         | 425 ++++++++++++++++++++++++
 drivers/ras/debugfs.c                     |   2 +-
 drivers/ras/debugfs.h                     |   8 +
 include/linux/ras.h                       |  12 +-
 14 files changed, 1467 insertions(+), 927 deletions(-)
 create mode 100644 arch/x86/kernel/cpu/mcheck/dev-mcelog.c
 delete mode 100644 arch/x86/ras/mce_amd_inj.c
 create mode 100644 drivers/ras/cec.c
 create mode 100644 drivers/ras/debugfs.h
Signature · 1 line
2.7.3
author maintainer
#1 lore raw

RFC PATCH 1/4 RAS: Add a Corrected Errors Collector

From: Borislav Petkov <[email protected]>

A simple data structure for collecting correctable errors along with accessors. More detailed description in the code itself.

---
 arch/x86/kernel/cpu/mcheck/mce.c | 120 +++++------
 arch/x86/ras/Kconfig             |  11 +
 drivers/ras/Makefile             |   3 +-
 drivers/ras/cec.c                | 425 +++++++++++++++++++++++++++++++++++++++
 drivers/ras/debugfs.c            |   2 +-
 drivers/ras/debugfs.h            |   8 +
 include/linux/ras.h              |  12 +-
 7 files changed, 521 insertions(+), 60 deletions(-)
 create mode 100644 drivers/ras/cec.c
 create mode 100644 drivers/ras/debugfs.h
arch/x86/kernel/cpu/mcheck/mce.c +63 -57
index 92e5e37d97bf..a6cc08a10cc3 100644
@@ -35,6 +35,7 @@
 #include <linux/poll.h>
 #include <linux/nmi.h>
 #include <linux/cpu.h>
+#include <linux/ras.h>
 #include <linux/smp.h>
 #include <linux/fs.h>
 #include <linux/mm.h>
@@ -151,14 +152,71 @@ static struct mce_log mcelog = {
 	.recordlen	= sizeof(struct mce),
 };
 
-void mce_log(struct mce *mce)
+static bool memory_error(struct mce *m)
+{
+	struct cpuinfo_x86 *c = &boot_cpu_data;
+
+	if (c->x86_vendor == X86_VENDOR_AMD) {
+		/* ErrCodeExt[20:16] */
+		u8 xec = (m->status >> 16) & 0x1f;
+
+		return (xec == 0x0 || xec == 0x8);
+	} else if (c->x86_vendor == X86_VENDOR_INTEL) {
+		/*
+		 * Intel SDM Volume 3B - 15.9.2 Compound Error Codes
+		 *
+		 * Bit 7 of the MCACOD field of IA32_MCi_STATUS is used for
+		 * indicating a memory error. Bit 8 is used for indicating a
+		 * cache hierarchy error. The combination of bit 2 and bit 3
+		 * is used for indicating a `generic' cache hierarchy error
+		 * But we can't just blindly check the above bits, because if
+		 * bit 11 is set, then it is a bus/interconnect error - and
+		 * either way the above bits just gives more detail on what
+		 * bus/interconnect error happened. Note that bit 12 can be
+		 * ignored, as it's the "filter" bit.
+		 */
+		return (m->status & 0xef80) == BIT(7) ||
+		       (m->status & 0xef00) == BIT(8) ||
+		       (m->status & 0xeffc) == 0xc;
+	}
+
+	return false;
+}
+
+/*
+ * Check if the address reported by the CPU is in a format we can parse.
+ * It would be possible to add code for most other cases, but all would
+ * be somewhat complicated (e.g. segment offset would require an instruction
+ * parser). So only support physical addresses up to page granuality for now.
+ */
+static int mce_usable_address(struct mce *m)
+{
+	if (!(m->status & MCI_STATUS_MISCV) || !(m->status & MCI_STATUS_ADDRV))
+		return 0;
+
+	/* Checks after this one are Intel-specific: */
+	if (boot_cpu_data.x86_vendor != X86_VENDOR_INTEL)
+		return 1;
+
+	if (MCI_MISC_ADDR_LSB(m->misc) > PAGE_SHIFT)
+		return 0;
+	if (MCI_MISC_ADDR_MODE(m->misc) != MCI_MISC_ADDR_PHYS)
+		return 0;
+	return 1;
+}
+
+void mce_log(struct mce *m)
 {
 	unsigned next, entry;
 
+	if (!in_atomic() && memory_error(m) && mce_usable_address(m))
+		if (!ce_add_elem(m->addr >> PAGE_SHIFT))
+			return;
+
 	/* Emit the trace record: */
-	trace_mce_record(mce);
+	trace_mce_record(m);
 
-	if (!mce_gen_pool_add(mce))
+	if (!mce_gen_pool_add(m))
 		irq_work_queue(&mce_irq_work);
 
 	wmb();
@@ -188,7 +246,7 @@ void mce_log(struct mce *mce)
 		if (cmpxchg(&mcelog.next, entry, next) == entry)
 			break;
 	}
-	memcpy(mcelog.entry + entry, mce, sizeof(struct mce));
+	memcpy(mcelog.entry + entry, m, sizeof(struct mce));
 	wmb();
 	mcelog.entry[entry].finished = 1;
 	wmb();
@@ -518,28 +576,6 @@ static void mce_report_event(struct pt_regs *regs)
 	irq_work_queue(&mce_irq_work);
 }
 
-/*
- * Check if the address reported by the CPU is in a format we can parse.
- * It would be possible to add code for most other cases, but all would
- * be somewhat complicated (e.g. segment offset would require an instruction
- * parser). So only support physical addresses up to page granuality for now.
- */
-static int mce_usable_address(struct mce *m)
-{
-	if (!(m->status & MCI_STATUS_MISCV) || !(m->status & MCI_STATUS_ADDRV))
-		return 0;
-
-	/* Checks after this one are Intel-specific: */
-	if (boot_cpu_data.x86_vendor != X86_VENDOR_INTEL)
-		return 1;
-
-	if (MCI_MISC_ADDR_LSB(m->misc) > PAGE_SHIFT)
-		return 0;
-	if (MCI_MISC_ADDR_MODE(m->misc) != MCI_MISC_ADDR_PHYS)
-		return 0;
-	return 1;
-}
-
 static int srao_decode_notifier(struct notifier_block *nb, unsigned long val,
 				void *data)
 {
@@ -582,37 +618,6 @@ static void mce_read_aux(struct mce *m, int i)
 	}
 }
 
-static bool memory_error(struct mce *m)
-{
-	struct cpuinfo_x86 *c = &boot_cpu_data;
-
-	if (c->x86_vendor == X86_VENDOR_AMD) {
-		/* ErrCodeExt[20:16] */
-		u8 xec = (m->status >> 16) & 0x1f;
-
-		return (xec == 0x0 || xec == 0x8);
-	} else if (c->x86_vendor == X86_VENDOR_INTEL) {
-		/*
-		 * Intel SDM Volume 3B - 15.9.2 Compound Error Codes
-		 *
-		 * Bit 7 of the MCACOD field of IA32_MCi_STATUS is used for
-		 * indicating a memory error. Bit 8 is used for indicating a
-		 * cache hierarchy error. The combination of bit 2 and bit 3
-		 * is used for indicating a `generic' cache hierarchy error
-		 * But we can't just blindly check the above bits, because if
-		 * bit 11 is set, then it is a bus/interconnect error - and
-		 * either way the above bits just gives more detail on what
-		 * bus/interconnect error happened. Note that bit 12 can be
-		 * ignored, as it's the "filter" bit.
-		 */
-		return (m->status & 0xef80) == BIT(7) ||
-		       (m->status & 0xef00) == BIT(8) ||
-		       (m->status & 0xeffc) == 0xc;
-	}
-
-	return false;
-}
-
 DEFINE_PER_CPU(unsigned, mce_poll_count);
 
 /*
@@ -2679,6 +2684,7 @@ static int __init mcheck_debugfs_init(void) { return -EINVAL; }
 static int __init mcheck_late_init(void)
 {
 	mcheck_debugfs_init();
+	ce_init();
 
 	/*
 	 * Flush out everything that has been logged during early boot, now that
arch/x86/ras/Kconfig +11 -0
index d957d5f21a86..513e8ffc9e08 100644
@@ -7,3 +7,14 @@ config MCE_AMD_INJ
 	  aspects of the MCE handling code.
 
 	  WARNING: Do not even assume this interface is staying stable!
+
+config RAS_CEC
+	bool "Correctable Errors Collector"
+	depends on X86_MCE && MEMORY_FAILURE && DEBUG_FS
+	---help---
+	  This is a small cache which collects correctable memory errors per 4K
+	  page PFN and counts their repeated occurrence. Once the counter for a
+	  PFN overflows, we try to soft-offline that page as we take it to mean
+	  that it has reached a relatively high error count and would probably
+	  be best if we don't use it anymore.
+
drivers/ras/Makefile +2 -1
index d7f73341ced3..7b26dd3aa5d0 100644
@@ -1 +1,2 @@
-obj-$(CONFIG_RAS) += ras.o debugfs.o
+obj-$(CONFIG_RAS)	+= ras.o debugfs.o
+obj-$(CONFIG_RAS_CEC)	+= cec.o
drivers/ras/cec.c new file +425 -0

425 lines not on this page · raw message

drivers/ras/debugfs.c +1 -1
index 0322acf67ea5..501603057dff 100644
@@ -1,6 +1,6 @@
 #include <linux/debugfs.h>
 
-static struct dentry *ras_debugfs_dir;
+struct dentry *ras_debugfs_dir;
 
 static atomic_t trace_count = ATOMIC_INIT(0);
 
drivers/ras/debugfs.h new file +8 -0
new file mode 100644
index 000000000000..db72e4513191
@@ -0,0 +1,8 @@
+#ifndef __RAS_DEBUGFS_H__
+#define __RAS_DEBUGFS_H__
+
+#include <linux/debugfs.h>
+
+extern struct dentry *ras_debugfs_dir;
+
+#endif /* __RAS_DEBUGFS_H__ */
include/linux/ras.h +11 -1
index 2aceeafd6fe5..a96762a10100 100644
@@ -1,14 +1,24 @@
 #ifndef __RAS_H__
 #define __RAS_H__
 
+#include <asm/errno.h>
+
 #ifdef CONFIG_DEBUG_FS
 int ras_userspace_consumers(void);
 void ras_debugfs_init(void);
 int ras_add_daemon_trace(void);
 #else
 static inline int ras_userspace_consumers(void) { return 0; }
-static inline void ras_debugfs_init(void) { return; }
+static inline void ras_debugfs_init(void) { }
 static inline int ras_add_daemon_trace(void) { return 0; }
 #endif
 
+#ifdef CONFIG_RAS_CEC
+void __init ce_init(void);
+int ce_add_elem(u64 pfn);
+#else
+static inline void __init ce_init(void)		{ }
+static inline int ce_add_elem(u64 pfn)		{ return -ENODEV; }
 #endif
+
+#endif /* __RAS_H__ */
Signature · 1 line
2.7.3
maintainer
#2 lore raw

Re: [RFC PATCH 1/4] RAS: Add a Corrected Errors Collector

Quoted 5 lines · Borislav Petkov
arch/x86/kernel/cpu/mcheck/mce.c +2 -0
+void mce_log(struct mce *m)
 {
 	unsigned next, entry;
 
+	if (!in_atomic() && memory_error(m) && mce_usable_address(m))

arch/x86/kernel/cpu/mcheck/mce.c static struct mce_log mcelog = { · line 220

Quoted 10 lines
arch/x86/kernel/cpu/mcheck/mce.c +5 -2
+		if (!ce_add_elem(m->addr >> PAGE_SHIFT))
+			return;
+
 	/* Emit the trace record: */
-	trace_mce_record(mce);
+	trace_mce_record(m);
 
-	if (!mce_gen_pool_add(mce))
+	if (!mce_gen_pool_add(m))
 		irq_work_queue(&mce_irq_work);

Is there a reason that we need to call the ce_add_elem() inline here instead of having it just register on the mce_notifier chain? This series just cleaned out all the /dev/mcelog special code from here, and you are adding something back before the ink is dry on that change.

I'm also strongly divided about whether this corrected error handler should be allowed to preempt anything else even seeing the error.

Argument for:
Lonely corrected errors are "No Big Deal"(TM). Just counting them and moving on is a good thing.

Arguments against:
1) We may miss out on a one-time opportunity to get extra information (from acpi_extlog.c).
2) I think this subverts our CMCI storm detection and mitigation code?

We could make the chain more caller friendly by adding a filter argument so users could say "just tell me about memory errors" (currently each of the EDAC drivers has inline code to do the same as "memory_error(m) && mce_usable_address(m)")

-Tony

author maintainer
#3 lore raw

Re: [RFC PATCH 1/4] RAS: Add a Corrected Errors Collector

Quoted 9 lines · Tony

Is there a reason that we need to call the ce_add_elem() inline here instead of having it just register on the mce_notifier chain? This series just cleaned out all the /dev/mcelog special code from here, and you are adding something back before the ink is dry on that change.

I'm also strongly divided about whether this corrected error handler should be allowed to preempt anything else even seeing the error.

Well, so this is the main reason for adding the CEC: not to disturb users with random CECC errors which might happen a couple of times due to alpha particles and then never again. I.e., address all those sporadic bursts of correctable errors which don't mean that the hw is going faulty.

If the CEC consumes the error and does the leaky bucket of "forgetting" about it after a while and after no more of that same PFN triggers errors, then we do that silently and do not scare users. Yeah, they think their hw is broken and whether they should start swapping things.

And then there's the aspect of soft-offlining PFNs when the error threshold has been reached. I don't think we have had any automatic recovery actions wrt errors so far without external agents.

Quoted 3 lines

Argument for:
Lonely corrected errors are "No Big Deal"(TM). Just counting them and moving on is a good thing.

Yap, exactly.

Quoted 4 lines

Arguments against:
1) We may miss out on a one-time opportunity to get extra information (from acpi_extlog.c).
2) I think this subverts our CMCI storm detection and mitigation code?

...and we can address that by adding "ras=cec_doesnt_consume_errors" or somesuch so that the rest of the chain sees them too.

I think we can be pretty flexible about it. And again, my main angle is the "do not disturb users unnecessarily".

Quoted 4 lines

We could make the chain more caller friendly by adding a filter argument so users could say "just tell me about memory errors" (currently each of the EDAC drivers has inline code to do the same as "memory_error(m) && mce_usable_address(m)")

Sure, that too.

And it can work on any system without the need for an EDAC driver.

Thanks.

Signature · 4 lines
Regards/Gruss,
    Boris.

ECO tip #101: Trim your mails when you reply.
author maintainer
#4 lore raw

RFC PATCH 2/4 x86/mce: Deprecate /dev/mcelog

In reply to Borislav Petkov

From: Tony Luck <[email protected]>

Move all code relating to /dev/mcelog to own source file. /dev/mcelog driver can now operate from the machine check notifier with lowest prio.

Boris:
* Move the mce_helper and trigger functionality behind CONFIG_X86_MCELOG.

---
 arch/x86/Kconfig                          |  10 +-
 arch/x86/kernel/cpu/mcheck/Makefile       |   2 +
 arch/x86/kernel/cpu/mcheck/dev-mcelog.c   | 397 +++++++++++++++++++++++++++++
 arch/x86/kernel/cpu/mcheck/mce-internal.h |   8 +
 arch/x86/kernel/cpu/mcheck/mce.c          | 405 +++---------------------------
 5 files changed, 445 insertions(+), 377 deletions(-)
 create mode 100644 arch/x86/kernel/cpu/mcheck/dev-mcelog.c
arch/x86/Kconfig +9 -1
index 0a7b885964ba..0bb289520207 100644
@@ -1004,6 +1004,14 @@ config X86_MCE
 	  The action the kernel takes depends on the severity of the problem,
 	  ranging from warning messages to halting the machine.
 
+config X86_MCELOG
+	bool "Support for deprecated /dev/mcelog character device"
+	depends on X86_MCE
+	---help---
+	  Enable support for /dev/mcelog which is needed by the old mcelog
+	  userspace logging daemon. Consider switching to the new generation
+	  rasdaemon solution.
+
 config X86_MCE_INTEL
 	def_bool y
 	prompt "Intel MCE features"
@@ -1033,7 +1041,7 @@ config X86_MCE_THRESHOLD
 	def_bool y
 
 config X86_MCE_INJECT
-	depends on X86_MCE
+	depends on X86_MCE && X86_MCELOG
 	tristate "Machine check injector support"
 	---help---
 	  Provide support for injecting machine checks for testing purposes.
arch/x86/kernel/cpu/mcheck/Makefile +2 -0
index a3311c886194..950e9ff602ea 100644
@@ -9,3 +9,5 @@ obj-$(CONFIG_X86_MCE_INJECT)	+= mce-inject.o
 obj-$(CONFIG_X86_THERMAL_VECTOR) += therm_throt.o
 
 obj-$(CONFIG_ACPI_APEI)		+= mce-apei.o
+
+obj-$(CONFIG_X86_MCELOG)	+= dev-mcelog.o
arch/x86/kernel/cpu/mcheck/dev-mcelog.c new file +397 -0

397 lines not on this page · raw message

arch/x86/kernel/cpu/mcheck/mce-internal.h +8 -0
index cd74a3f00aea..aa0deeb8bb69 100644
@@ -96,3 +96,11 @@ static inline bool mce_cmp(struct mce *m1, struct mce *m2)
 		m1->addr != m2->addr ||
 		m1->misc != m2->misc;
 }
+
+extern struct device_attribute dev_attr_trigger;
+
+#ifdef CONFIG_X86_MCELOG
+extern void mce_work_trigger(void);
+#else
+static inline void mce_work_trigger(void)	{ }
+#endif
arch/x86/kernel/cpu/mcheck/mce.c +29 -376

490 lines not on this page · raw message

Signature · 1 line
2.7.3
Acked-by on v1 2/4
#5 lore raw

Re: [RFC PATCH 2/4] x86/mce: Deprecate /dev/mcelog

Quoted 8 lines · Borislav Petkov

From: Tony Luck <[email protected]>

Move all code relating to /dev/mcelog to own source file. /dev/mcelog driver can now operate from the machine check notifier with lowest prio.

Boris:
* Move the mce_helper and trigger functionality behind CONFIG_X86_MCELOG.

author maintainer
#6 lore raw

RFC PATCH 3/4 x86/mce: Merge mce_amd_inj into mce-inject

In reply to Borislav Petkov

From: Borislav Petkov <[email protected]>

We're reusing mce_amd_inj's debugfs interface so that mce-inject can benefit from it too. The old functionality is still preserved under CONFIG_X86_MCELOG.

---
 arch/x86/Kconfig                        |   2 +-
 arch/x86/kernel/cpu/mcheck/mce-inject.c | 525 +++++++++++++++++++++++++++++++-
 arch/x86/ras/Makefile                   |   2 -
 arch/x86/ras/mce_amd_inj.c              | 484 -----------------------------
 4 files changed, 515 insertions(+), 498 deletions(-)
 delete mode 100644 arch/x86/ras/mce_amd_inj.c
arch/x86/Kconfig +1 -1
index 0bb289520207..c34e0fe23cf2 100644
@@ -1041,7 +1041,7 @@ config X86_MCE_THRESHOLD
 	def_bool y
 
 config X86_MCE_INJECT
-	depends on X86_MCE && X86_MCELOG
+	depends on X86_MCE
 	tristate "Machine check injector support"
 	---help---
 	  Provide support for injecting machine checks for testing purposes.
arch/x86/kernel/cpu/mcheck/mce-inject.c +514 -11

564 lines not on this page · raw message

arch/x86/ras/Makefile +0 -2
index 5f94546db280..e69de29bb2d1 100644
@@ -1,2 +0,0 @@
-obj-$(CONFIG_MCE_AMD_INJ)		+= mce_amd_inj.o
-
arch/x86/ras/mce_amd_inj.c deleted +0 -484

484 lines not on this page · raw message

Signature · 1 line
2.7.3
author maintainer
#7 lore raw

RFC PATCH 4/4 x86/mce-inject: Use debugfs_remove_recursive()

In reply to Borislav Petkov

From: Borislav Petkov <[email protected]>

... and simplify a bit.

No functionality change.

---
 arch/x86/kernel/cpu/mcheck/mce-inject.c | 30 ++++++++++++------------------
 1 file changed, 12 insertions(+), 18 deletions(-)
arch/x86/kernel/cpu/mcheck/mce-inject.c +12 -18
index f1d8d3f9ec1d..675d2da3d31f 100644
@@ -676,8 +676,8 @@ static struct dfs_node {
 
 static int __init debugfs_init(void)
 {
-	int i;
 	u64 cap;
+	int i;
 
 	rdmsrl(MSR_IA32_MCG_CAP, cap);
 	n_banks = cap & MCG_BANKCNT_MASK;
@@ -700,28 +700,12 @@ static int __init debugfs_init(void)
 	return 0;
 
 err_dfs_add:
-	while (--i >= 0)
-		debugfs_remove(dfs_fls[i].d);
-
-	debugfs_remove(dfs_inj);
+	debugfs_remove_recursive(dfs_inj);
 	dfs_inj = NULL;
 
 	return -ENOMEM;
 }
 
-static void __exit debugfs_exit(void)
-{
-	int i;
-
-	for (i = 0; i < ARRAY_SIZE(dfs_fls); i++)
-		debugfs_remove(dfs_fls[i].d);
-
-	memset(&dfs_fls, 0, sizeof(dfs_fls));
-
-	debugfs_remove(dfs_inj);
-	dfs_inj = NULL;
-}
-
 static int inject_init(void)
 {
 	int err;
@@ -752,6 +736,16 @@ module_init(inject_init);
  * guarantee all openers of mce_chrdev will get a reference to us.
  */
 #ifndef CONFIG_X86_MCELOG
+static void __exit debugfs_exit(void)
+{
+	debugfs_remove_recursive(dfs_inj);
+	dfs_inj = NULL;
+
+	unregister_nmi_handler(NMI_LOCAL, "mce_notify");
+
+	free_cpumask_var(mce_inject_cpumask);
+}
+
 module_exit(debugfs_exit);
 #endif
 
Signature · 1 line
2.7.3