Skip to contentSkip to conversation

edac_mce: Add an interface driver to report mce errors via edac

PATCH In review Mauro Carvalho Chehab 3 participants 10 messages Sep 24, 2009 – Sep 25, 2009

Subsystems (2)
lore
Share & tools
lore
#0 lore raw

PATCH 17/63 edac_mce: Add an interface driver to report mce errors via edac

edac_mce module is an interface module that gets mcelog data and forwards to any registered edac module that expects to receive data via mce.

---
 arch/x86/kernel/cpu/mcheck/mce.c |   12 ++++++++
 drivers/edac/Kconfig             |    8 ++++-
 drivers/edac/Makefile            |    3 +-
 drivers/edac/edac_mce.c          |   58 ++++++++++++++++++++++++++++++++++++++
 include/linux/edac_mce.h         |   31 ++++++++++++++++++++
 5 files changed, 110 insertions(+), 2 deletions(-)
 create mode 100644 drivers/edac/edac_mce.c
 create mode 100644 include/linux/edac_mce.h
arch/x86/kernel/cpu/mcheck/mce.c +12 -0
index 2f5aab2..fefbb7c 100644
@@ -35,6 +35,7 @@
 #include <linux/fs.h>
 #include <linux/mm.h>
 #include <linux/debugfs.h>
+#include <linux/edac_mce.h>
 
 #include <asm/processor.h>
 #include <asm/hw_irq.h>
@@ -135,6 +136,15 @@ void mce_log(struct mce *mce)
 		entry = rcu_dereference(mcelog.next);
 		for (;;) {
 			/*
+			 * If edac_mce is enabled, it will check the error type
+			 * and will process it, if it is a known error.
+			 * Otherwise, the error will be sent through mcelog
+			 * interface
+			 */
+			if (edac_mce_parse(mce))
+				return;
+
+			/*
 			 * When the buffer fills up discard new entries.
 			 * Assume that the earlier errors are the more
 			 * interesting ones:
@@ -193,6 +203,8 @@ static void print_mce(struct mce *m)
 			m->cpuvendor, m->cpuid, m->time, m->socketid,
 			m->apicid);
 
+	edac_mce_parse(m);
+
 	decode_mce(m);
 }
 
drivers/edac/Kconfig +7 -1
index 0c8ca30..a69f69e 100644
@@ -57,6 +57,9 @@ config EDAC_MM_EDAC
 	  occurred so that a particular failing memory module can be
 	  replaced.  If unsure, select 'Y'.
 
+config EDAC_MCE
+	tristate
+
 config EDAC_AMD64
 	tristate "AMD64 (Opteron, Athlon64) K8, F10h, F11h"
 	depends on EDAC_MM_EDAC && K8_NB && X86_64 && PCI && CPU_SUP_AMD
@@ -150,9 +153,12 @@ config EDAC_I5400
 config EDAC_I7CORE
 	tristate "Intel i7 Core (Nehalem) processors"
 	depends on EDAC_MM_EDAC && PCI && X86
+	select EDAC_MCE
 	help
 	  Support for error detection and correction the Intel
-	  i7 Core (Nehalem) Integrated Memory Controller
+	  i7 Core (Nehalem) Integrated Memory Controller that exists on
+	  newer processors like i7 Core, i7 Core Extreme, Xeon 35xx
+	  and Xeon 55xx processors.
 
 config EDAC_I82860
 	tristate "Intel 82860"
drivers/edac/Makefile +2 -1
index 6f0cfe2..5ec7e9c 100644
@@ -7,8 +7,9 @@
 #
 
 
-obj-$(CONFIG_EDAC)			:= edac_stub.o
+obj-$(CONFIG_EDAC)			+= edac_stub.o
 obj-$(CONFIG_EDAC_MM_EDAC)		+= edac_core.o
+obj-$(CONFIG_EDAC_MCE)			+= edac_mce.o
 
 edac_core-objs	:= edac_mc.o edac_device.o edac_mc_sysfs.o edac_pci_sysfs.o
 edac_core-objs	+= edac_module.o edac_device_sysfs.o
drivers/edac/edac_mce.c new file +58 -0

58 lines not on this page · raw message

include/linux/edac_mce.h new file +31 -0
new file mode 100644
index 0000000..f974fc0
@@ -0,0 +1,31 @@
+/* Provides edac interface to mcelog events
+ *
+ * This file may be distributed under the terms of the
+ * GNU General Public License version 2.
+ *
+ * Copyright (c) 2009 by:
+ *	 Mauro Carvalho Chehab <[email protected]>
+ *
+ * Red Hat Inc. http://www.redhat.com
+ */
+
+#if defined(CONFIG_EDAC_MCE) || \
+			(defined(CONFIG_EDAC_MCE_MODULE) && defined(MODULE))
+
+#include <asm/mce.h>
+#include <linux/list.h>
+
+struct edac_mce {
+	struct list_head list;
+
+	void *priv;
+	int (*check_error)(void *priv, struct mce *mce);
+};
+
+int edac_mce_register(struct edac_mce *edac_mce);
+void edac_mce_unregister(struct edac_mce *edac_mce);
+int edac_mce_parse(struct mce *mce);
+
+#else
+#define edac_mce_parse(mce) (0)
+#endif
Signature · 1 line
1.5.5.6
#1 lore raw

Mauro,

Quoted 28 lines · Mauro Carvalho Chehab

edac_mce module is an interface module that gets mcelog data and forwards to any registered edac module that expects to receive data via mce.

---
 arch/x86/kernel/cpu/mcheck/mce.c |   12 ++++++++
 drivers/edac/Kconfig             |    8 ++++-
 drivers/edac/Makefile            |    3 +-
 drivers/edac/edac_mce.c          |   58 ++++++++++++++++++++++++++++++++++++++
 include/linux/edac_mce.h         |   31 ++++++++++++++++++++
 5 files changed, 110 insertions(+), 2 deletions(-)
 create mode 100644 drivers/edac/edac_mce.c
 create mode 100644 include/linux/edac_mce.h
arch/x86/kernel/cpu/mcheck/mce.c +1 -0
index 2f5aab2..fefbb7c 100644
@@ -35,6 +35,7 @@
 #include <linux/fs.h>
 #include <linux/mm.h>
 #include <linux/debugfs.h>
+#include <linux/edac_mce.h>
 
 #include <asm/processor.h>
 #include <asm/hw_irq.h>

arch/x86/kernel/cpu/mcheck/mce.c void mce_log(struct mce *mce) · line 145

Quoted 10 lines
arch/x86/kernel/cpu/mcheck/mce.c +7 -0
@@ -135,6 +136,15 @@ void mce_log(struct mce *mce)
 		entry = rcu_dereference(mcelog.next);
 		for (;;) {
 			/*
+			 * If edac_mce is enabled, it will check the error type
+			 * and will process it, if it is a known error.
+			 * Otherwise, the error will be sent through mcelog
+			 * interface
+			 */
+			if (edac_mce_parse(mce))
+				return;

for the third time (!): this may run in NMI context and as such does not obey to normal kernel locking rules and you cannot safely use almost any kernel resources involving locking. This way, your hook calls into a module, which is a very bad idea. Please remove that hook and put in the polling routine or somewhere more appropriate.

Quoted 13 lines
arch/x86/kernel/cpu/mcheck/mce.c +4 -0
+
+			/*
 			 * When the buffer fills up discard new entries.
 			 * Assume that the earlier errors are the more
 			 * interesting ones:
@@ -193,6 +203,8 @@ static void print_mce(struct mce *m)
 			m->cpuvendor, m->cpuid, m->time, m->socketid,
 			m->apicid);
 
+	edac_mce_parse(m);
+
 	decode_mce(m);
 }
Signature · 8 lines
Regards/Gruss,
Boris.

Operating | Advanced Micro Devices GmbH
  System  | Karl-Hammerschmidt-Str. 34, 85609 Dornach b. München, Germany
 Research | Geschäftsführer: Andrew Bowd, Thomas M. McCoy, Giuliano Meroni
  Center  | Sitz: Dornach, Gemeinde Aschheim, Landkreis München
  (OSRC)  | Registergericht München, HRB Nr. 43632
#2 lore raw

Boris,

Em Fri, 25 Sep 2009 11:48:55 +0200
Borislav Petkov <[email protected]> escreveu:

Quoted 8 lines
Quoted 8 lines
arch/x86/kernel/cpu/mcheck/mce.c +5 -0
 		entry = rcu_dereference(mcelog.next);
 		for (;;) {
 			/*
+			 * If edac_mce is enabled, it will check the error type
+			 * and will process it, if it is a known error.
+			 * Otherwise, the error will be sent through mcelog
+			 * interface
+			 */
Quoted 8 lines

arch/x86/kernel/cpu/mcheck/mce.c void mce_log(struct mce *mce) · line 145

Quoted 2 lines
arch/x86/kernel/cpu/mcheck/mce.c +2 -0
+			if (edac_mce_parse(mce))
+				return;

for the third time (!): this may run in NMI context and as such does not obey to normal kernel locking rules and you cannot safely use almost any kernel resources involving locking. This way, your hook calls into a module, which is a very bad idea. Please remove that hook and put in the polling routine or somewhere more appropriate.

I had answered you already, but let me give a more complete explanation.

For sure all the code called at this point should be carefully analyzed. So, let's see the complete implementation:

1) edac_mce is not a module (see patch 18). So, just calling a routine on edac_mce should be safe, even at NMI;

2) While there's nobody registered on edac_mce, it will just return 0, letting the code proceed. There's no lock there. The called code is as simple as:

int edac_mce_parse(struct mce *mce)

{
	struct edac_mce *edac_mce;

	list_for_each_entry(edac_mce, &edac_mce_list, list) {
		if (edac_mce->check_error(edac_mce->priv, mce))
			return 1;
	}

	/* Nobody queued the error */
	return 0;
}

3) i7core_edac will only start handling mce events after being loaded on memory and registered on edac_mce. If an error occurs before it, normal mce handling will happen;

4) after registered, edac_mce will call this hook, at i7core_edac:

static int i7core_mce_check_error(void *priv, struct mce *mce)

{
	struct mem_ctl_info *mci = priv;
	struct i7core_pvt *pvt = mci->pvt_info;
	unsigned long flags;

	/*
	 * Just let mcelog handle it if the error is
	 * outside the memory controller
	 */
	if (((mce->status & 0xffff) >> 7) != 1)
		return 0;

	/* Bank 8 registers are the only ones that we know how to handle */
	if (mce->bank != 8)
		return 0;

	/* Only handle if it is the right mc controller */
	if (cpu_data(mce->cpu).phys_proc_id != pvt->i7core_dev->socket) {
		debugf0("mc%d: ignoring mce log for socket %d. "
			"Another mc should get it.\n",
			pvt->i7core_dev->socket,
			cpu_data(mce->cpu).phys_proc_id);
			return 0;
	}

	spin_lock_irqsave(&pvt->mce_lock, flags);
        if (pvt->mce_count < MCE_LOG_LEN) {
		memcpy(&pvt->mce_entry[pvt->mce_count], mce, sizeof(*mce));
		pvt->mce_count++;
	}
	spin_unlock_irqrestore(&pvt->mce_lock, flags);

	/* Handle fatal errors immediately */
	if (mce->mcgstatus & 1)
		i7core_check_error(mci);

	/* Advice mcelog that the error were handled */
	return 1;
}

It will basically check if the error is related to the MC, returning if not. If the error is for the mc, it will copy the error inside a buffer on i7core_mce.

In this part, it is currently using a spinlock. It might have used an RCU code instead, but I opted to use a simpler approach, since, on all tests, this worked fine.

Iff the error is a fatal error (Uncorrected Error), it will call the EDAC UE routine to to display the error. For all Corrected Errors, like the mce driver, the code that will parse the error will be called outside NMI, by edac polling code.

5) there are very few calls to kernel routines here: list_for_each_entry(), spin_lock_irqsave(), in_unlock_irqrestore(), memcpy(). I don't think any of them will have any problems on working even inside NMI context.

Signature · 2 lines
Cheers,
Mauro
#3 lore raw
Quoted 4 lines · On Fri, 25 Sep 2009 09:11:30 -0300 Mauro Carvalho Chehab <[email protected]>…

5) there are very few calls to kernel routines here:
list_for_each_entry(), spin_lock_irqsave(), in_unlock_irqrestore(), memcpy(). I don't think any of them will have any problems on working even inside NMI context.

spinlocks in NMI context don't do what you think they do.... _irqsave does not disable NMIs so this can deadlock.

You likely want to use trylock here....

Signature · 3 lines
Arjan van de Ven 	Intel Open Source Technology Centre
For development, discussion and tips for power savings, 
visit http://www.lesswatts.org
#4 lore raw

Em Fri, 25 Sep 2009 14:47:20 +0200
Arjan van de Ven <[email protected]> escreveu:

Quoted 8 lines
Quoted 4 lines · On Fri, 25 Sep 2009 09:11:30 -0300 Mauro Carvalho Chehab <[email protected]>…

5) there are very few calls to kernel routines here:
list_for_each_entry(), spin_lock_irqsave(), in_unlock_irqrestore(), memcpy(). I don't think any of them will have any problems on working even inside NMI context.

Quoted 4 lines

spinlocks in NMI context don't do what you think they do.... _irqsave does not disable NMIs so this can deadlock.

You likely want to use trylock here....

I see. I'll rewrite that part of the code using trylock or RCU.

Signature · 2 lines
Cheers,
Mauro
#5 lore raw
In reply to Mauro Carvalho Chehab

Hi,

Quoted 17 lines · Mauro Carvalho Chehab
Quoted 8 lines
Quoted 8 lines
arch/x86/kernel/cpu/mcheck/mce.c +5 -0
 		entry = rcu_dereference(mcelog.next);
 		for (;;) {
 			/*
+			 * If edac_mce is enabled, it will check the error type
+			 * and will process it, if it is a known error.
+			 * Otherwise, the error will be sent through mcelog
+			 * interface
+			 */
Quoted 8 lines

arch/x86/kernel/cpu/mcheck/mce.c void mce_log(struct mce *mce) · line 145

Quoted 2 lines
arch/x86/kernel/cpu/mcheck/mce.c +2 -0
+			if (edac_mce_parse(mce))
+				return;

for the third time (!): this may run in NMI context and as such does not obey to normal kernel locking rules and you cannot safely use almost any kernel resources involving locking. This way, your hook calls into a module, which is a very bad idea. Please remove that hook and put in the polling routine or somewhere more appropriate.

Quoted 7 lines

I had answered you already, but let me give a more complete explanation.

For sure all the code called at this point should be carefully analyzed. So, let's see the complete implementation:

1) edac_mce is not a module (see patch 18). So, just calling a routine on edac_mce should be safe, even at NMI;

no, I mean the ->check_error member - it could call into a module if i7core_edac is compiled as such.

<snip the obvious non-registered module case>

Quoted 23 lines

3) i7core_edac will only start handling mce events after being loaded on memory and registered on edac_mce. If an error occurs before it, normal mce handling will happen;

4) after registered, edac_mce will call this hook, at i7core_edac:

static int i7core_mce_check_error(void *priv, struct mce *mce)

{
	struct mem_ctl_info *mci = priv;
	struct i7core_pvt *pvt = mci->pvt_info;
	unsigned long flags;

	/*
	 * Just let mcelog handle it if the error is
	 * outside the memory controller
	 */
	if (((mce->status & 0xffff) >> 7) != 1)
		return 0;

	/* Bank 8 registers are the only ones that we know how to handle */
	if (mce->bank != 8)
		return 0;
Quoted 8 lines
	/* Only handle if it is the right mc controller */
	if (cpu_data(mce->cpu).phys_proc_id != pvt->i7core_dev->socket) {
		debugf0("mc%d: ignoring mce log for socket %d. "
			"Another mc should get it.\n",
			pvt->i7core_dev->socket,
			cpu_data(mce->cpu).phys_proc_id);
			return 0;
	}

One problem here is the debug call which is a printk() and you may deadlock while doing a printk in an NMI context. That's why you add MCEs to the lockless buffer in mce_log and decode them later - otherwise you could just as well printk them here.

Generally, you need to keep the NMI handlers as short as possible and postpone the parsing of the MCEs for later.

Signature · 8 lines
Regards/Gruss,
Boris.

Operating | Advanced Micro Devices GmbH
  System  | Karl-Hammerschmidt-Str. 34, 85609 Dornach b. München, Germany
 Research | Geschäftsführer: Andrew Bowd, Thomas M. McCoy, Giuliano Meroni
  Center  | Sitz: Dornach, Gemeinde Aschheim, Landkreis München
  (OSRC)  | Registergericht München, HRB Nr. 43632
#6 lore raw

On Fri, 25 Sep 2009 15:56:26 +0200

Quoted 5 lines
Quoted 2 lines

1) edac_mce is not a module (see patch 18). So, just calling a routine on edac_mce should be safe, even at NMI;

no, I mean the ->check_error member - it could call into a module if i7core_edac is compiled as such.

calling modular code from NMI is not a fatal event though.

calling printk or taking spinlocks otoh..

Signature · 3 lines
Arjan van de Ven 	Intel Open Source Technology Centre
For development, discussion and tips for power savings, 
visit http://www.lesswatts.org
#7 lore raw
Quoted 8 lines · Arjan van de Ven

On Fri, 25 Sep 2009 15:56:26 +0200

Quoted 5 lines
Quoted 2 lines

1) edac_mce is not a module (see patch 18). So, just calling a routine on edac_mce should be safe, even at NMI;

no, I mean the ->check_error member - it could call into a module if i7core_edac is compiled as such.

calling modular code from NMI is not a fatal event though.

No, not really. However, I remember Andi raising a stability concern one time whether it'd be such a good idea to allow modules to hook into MCE critical path when the system is already unstable and about to panic.

Therefore, we might want to decode critical MCEs in core kernel code and non-critical later, at a more appropriate time (aka in modules).

Hmm?

Signature · 8 lines
Regards/Gruss,
Boris.

Operating | Advanced Micro Devices GmbH
  System  | Karl-Hammerschmidt-Str. 34, 85609 Dornach b. München, Germany
 Research | Geschäftsführer: Andrew Bowd, Thomas M. McCoy, Giuliano Meroni
  Center  | Sitz: Dornach, Gemeinde Aschheim, Landkreis München
  (OSRC)  | Registergericht München, HRB Nr. 43632
#8 lore raw
Quoted 8 lines · On Fri, 25 Sep 2009 16:40:38 +0200 Borislav Petkov <[email protected]>…
Quoted 7 lines · Arjan van de Ven

On Fri, 25 Sep 2009 15:56:26 +0200

Quoted 5 lines
Quoted 2 lines

1) edac_mce is not a module (see patch 18). So, just calling a routine on edac_mce should be safe, even at NMI;

no, I mean the ->check_error member - it could call into a module if i7core_edac is compiled as such.

Quoted 6 lines
Quoted 1 line

calling modular code from NMI is not a fatal event though.

No, not really. However, I remember Andi raising a stability concern one time whether it'd be such a good idea to allow modules to hook into MCE critical path when the system is already unstable and about to panic.

module vs not does not make a difference here.

either it's good code or it's not, does not matter where it lives.

Quoted 4 lines

Therefore, we might want to decode critical MCEs in core kernel code and non-critical later, at a more appropriate time (aka in modules).

modules are also just an artificial carve-off of the kernel, nothing more.

Signature · 3 lines
Arjan van de Ven 	Intel Open Source Technology Centre
For development, discussion and tips for power savings, 
visit http://www.lesswatts.org
#9 lore raw
In reply to Borislav Petkov

Em Fri, 25 Sep 2009 15:56:26 +0200
Borislav Petkov <[email protected]> escreveu:

Quoted 22 lines

Hi,

Quoted 19 lines · Mauro Carvalho Chehab
Quoted 8 lines
Quoted 8 lines
arch/x86/kernel/cpu/mcheck/mce.c +5 -0
 		entry = rcu_dereference(mcelog.next);
 		for (;;) {
 			/*
+			 * If edac_mce is enabled, it will check the error type
+			 * and will process it, if it is a known error.
+			 * Otherwise, the error will be sent through mcelog
+			 * interface
+			 */
Quoted 8 lines

arch/x86/kernel/cpu/mcheck/mce.c void mce_log(struct mce *mce) · line 145

Quoted 2 lines
arch/x86/kernel/cpu/mcheck/mce.c +2 -0
+			if (edac_mce_parse(mce))
+				return;

for the third time (!): this may run in NMI context and as such does not obey to normal kernel locking rules and you cannot safely use almost any kernel resources involving locking. This way, your hook calls into a module, which is a very bad idea. Please remove that hook and put in the polling routine or somewhere more appropriate.

I had answered you already, but let me give a more complete explanation.

Quoted 8 lines
Quoted 5 lines

For sure all the code called at this point should be carefully analyzed. So, let's see the complete implementation:

1) edac_mce is not a module (see patch 18). So, just calling a routine on edac_mce should be safe, even at NMI;

no, I mean the ->check_error member - it could call into a module if i7core_edac is compiled as such.

Yes, but calling a code inside a module already loaded in memory should work just fine as calling a builtin code. As the module needs to be loaded first, in order to register on edac_mce, there's no problem here.

Quoted 34 lines

<snip the obvious non-registered module case>

Quoted 23 lines

3) i7core_edac will only start handling mce events after being loaded on memory and registered on edac_mce. If an error occurs before it, normal mce handling will happen;

4) after registered, edac_mce will call this hook, at i7core_edac:

static int i7core_mce_check_error(void *priv, struct mce *mce)

{
	struct mem_ctl_info *mci = priv;
	struct i7core_pvt *pvt = mci->pvt_info;
	unsigned long flags;

	/*
	 * Just let mcelog handle it if the error is
	 * outside the memory controller
	 */
	if (((mce->status & 0xffff) >> 7) != 1)
		return 0;

	/* Bank 8 registers are the only ones that we know how to handle */
	if (mce->bank != 8)
		return 0;
Quoted 8 lines
	/* Only handle if it is the right mc controller */
	if (cpu_data(mce->cpu).phys_proc_id != pvt->i7core_dev->socket) {
		debugf0("mc%d: ignoring mce log for socket %d. "
			"Another mc should get it.\n",
			pvt->i7core_dev->socket,
			cpu_data(mce->cpu).phys_proc_id);
			return 0;
	}
Quoted 4 lines

One problem here is the debug call which is a printk() and you may deadlock while doing a printk in an NMI context. That's why you add MCEs to the lockless buffer in mce_log and decode them later - otherwise you could just as well printk them here.

That debug code can just be dropped. Anyway, this code disaperars if EDAC_DEBUG is disabled.

Quoted 2 lines

Generally, you need to keep the NMI handlers as short as possible and postpone the parsing of the MCEs for later.

True. The parser is outside the NMI called routine (except for UE, since you may not have a chance of parsing the error outside it, as panic is called on mce code).

Signature · 2 lines
Cheers,
Mauro