Skip to contentSkip to conversation

x86, NMI, Add symbol definition for NMI magic constants

RFC In review Huang Ying 13 participants 64 messages Sep 10, 2010 – Sep 29, 2010

Subsystems (7)
lore
Share & tools
lore

Huang Ying

author
#0 lore raw

RFC 1/6 x86, NMI, Add symbol definition for NMI magic constants

Replace the NMI related magic numbers with symbol constants.

---
 arch/x86/include/asm/mach_traps.h |   12 +++++++++++-
 arch/x86/kernel/traps.c           |   18 +++++++++---------
 2 files changed, 20 insertions(+), 10 deletions(-)
arch/x86/include/asm/mach_traps.h +11 -1
@@ -7,9 +7,19 @@
 
 #include <asm/mc146818rtc.h>
 
+#define NMI_REASON_PORT		0x61
+
+#define NMI_REASON_MEMPAR	0x80
+#define NMI_REASON_IOCHK	0x40
+#define NMI_REASON_MASK		(NMI_REASON_MEMPAR | NMI_REASON_IOCHK)
+
+#define NMI_REASON_CLEAR_MEMPAR	0x04
+#define NMI_REASON_CLEAR_IOCHK	0x08
+#define NMI_REASON_CLEAR_MASK	0x0f
+
 static inline unsigned char get_nmi_reason(void)
 {
-	return inb(0x61);
+	return inb(NMI_REASON_PORT);
 }
 
 static inline void reassert_nmi(void)
arch/x86/kernel/traps.c +9 -9
@@ -323,8 +323,8 @@ mem_parity_error(unsigned char reason, s
 	printk(KERN_EMERG "Dazed and confused, but trying to continue\n");
 
 	/* Clear and disable the memory parity error line. */
-	reason = (reason & 0xf) | 4;
-	outb(reason, 0x61);
+	reason = (reason & NMI_REASON_CLEAR_MASK) | NMI_REASON_CLEAR_MEMPAR;
+	outb(reason, NMI_REASON_PORT);
 }
 
 static notrace __kprobes void
@@ -339,15 +339,15 @@ io_check_error(unsigned char reason, str
 		panic("NMI IOCK error: Not continuing");
 
 	/* Re-enable the IOCK line, wait for a few seconds */
-	reason = (reason & 0xf) | 8;
-	outb(reason, 0x61);
+	reason = (reason & NMI_REASON_CLEAR_MASK) | NMI_REASON_CLEAR_IOCHK;
+	outb(reason, NMI_REASON_PORT);
 
 	i = 2000;
 	while (--i)
 		udelay(1000);
 
-	reason &= ~8;
-	outb(reason, 0x61);
+	reason &= ~NMI_REASON_CLEAR_IOCHK;
+	outb(reason, NMI_REASON_PORT);
 }
 
 static notrace __kprobes void
@@ -388,7 +388,7 @@ static notrace __kprobes void default_do
 	if (!cpu)
 		reason = get_nmi_reason();
 
-	if (!(reason & 0xc0)) {
+	if (!(reason & NMI_REASON_MASK)) {
 		if (notify_die(DIE_NMI_IPI, "nmi_ipi", regs, reason, 2, SIGINT)
 								== NOTIFY_STOP)
 			return;
@@ -418,9 +418,9 @@ static notrace __kprobes void default_do
 		return;
 
 	/* AK: following checks seem to be broken on modern chipsets. FIXME */
-	if (reason & 0x80)
+	if (reason & NMI_REASON_MEMPAR)
 		mem_parity_error(reason, regs);
-	if (reason & 0x40)
+	if (reason & NMI_REASON_IOCHK)
 		io_check_error(reason, regs);
 #ifdef CONFIG_X86_32
 	/*
author
#1 lore raw

RFC 2/6 x86, NMI, Add touch_nmi_watchdog to io_check_error delay

Prevent the long delay in io_check_error make NMI watchdog timeout.

---
 arch/x86/kernel/traps.c |    8 +++++---
 1 file changed, 5 insertions(+), 3 deletions(-)
arch/x86/kernel/traps.c +5 -3
@@ -342,9 +342,11 @@ io_check_error(unsigned char reason, str
 	reason = (reason & NMI_REASON_CLEAR_MASK) | NMI_REASON_CLEAR_IOCHK;
 	outb(reason, NMI_REASON_PORT);
 
-	i = 2000;
-	while (--i)
-		udelay(1000);
+	i = 20000;
+	while (--i) {
+		touch_nmi_watchdog();
+		udelay(100);
+	}
 
 	reason &= ~NMI_REASON_CLEAR_IOCHK;
 	outb(reason, NMI_REASON_PORT);
author
#2 lore raw

RFC 3/6 x86, NMI, Rename memory parity error to PCI SERR error

In reply to Huang Ying

memory parity error is only valid for IBM PC-AT, newer machine use 7 bit (0x80) of 0x61 port for PCI SERR. While memory error is usually reported via MCE. So corresponding function name and kernel log string is changed.

---
 arch/x86/include/asm/mach_traps.h |    6 +++---
 arch/x86/kernel/traps.c           |   24 ++++++------------------
 2 files changed, 9 insertions(+), 21 deletions(-)
arch/x86/include/asm/mach_traps.h +3 -3
@@ -9,11 +9,11 @@
 
 #define NMI_REASON_PORT		0x61
 
-#define NMI_REASON_MEMPAR	0x80
+#define NMI_REASON_SERR		0x80
 #define NMI_REASON_IOCHK	0x40
-#define NMI_REASON_MASK		(NMI_REASON_MEMPAR | NMI_REASON_IOCHK)
+#define NMI_REASON_MASK		(NMI_REASON_SERR | NMI_REASON_IOCHK)
 
-#define NMI_REASON_CLEAR_MEMPAR	0x04
+#define NMI_REASON_CLEAR_SERR	0x04
 #define NMI_REASON_CLEAR_IOCHK	0x08
 #define NMI_REASON_CLEAR_MASK	0x0f
 
arch/x86/kernel/traps.c +6 -18
@@ -301,29 +301,17 @@ gp_in_kernel:
 }
 
 static notrace __kprobes void
-mem_parity_error(unsigned char reason, struct pt_regs *regs)
+pci_serr_error(unsigned char reason, struct pt_regs *regs)
 {
-	printk(KERN_EMERG
-		"Uhhuh. NMI received for unknown reason %02x on CPU %d.\n",
-			reason, smp_processor_id());
-
-	printk(KERN_EMERG
-		"You have some hardware problem, likely on the PCI bus.\n");
-
-#if defined(CONFIG_EDAC)
-	if (edac_handler_set()) {
-		edac_atomic_assert_error();
-		return;
-	}
-#endif
+	printk(KERN_EMERG "NMI: PCI system error (SERR).\n");
 
 	if (panic_on_unrecovered_nmi)
 		panic("NMI: Not continuing");
 
 	printk(KERN_EMERG "Dazed and confused, but trying to continue\n");
 
-	/* Clear and disable the memory parity error line. */
-	reason = (reason & NMI_REASON_CLEAR_MASK) | NMI_REASON_CLEAR_MEMPAR;
+	/* Clear and disable the PCI SERR error line. */
+	reason = (reason & NMI_REASON_CLEAR_MASK) | NMI_REASON_CLEAR_SERR;
 	outb(reason, NMI_REASON_PORT);
 }
 
@@ -420,8 +408,8 @@ static notrace __kprobes void default_do
 		return;
 
 	/* AK: following checks seem to be broken on modern chipsets. FIXME */
-	if (reason & NMI_REASON_MEMPAR)
-		mem_parity_error(reason, regs);
+	if (reason & NMI_REASON_SERR)
+		pci_serr_error(reason, regs);
 	if (reason & NMI_REASON_IOCHK)
 		io_check_error(reason, regs);
 #ifdef CONFIG_X86_32
#3 lore raw

Re: [RFC 3/6] x86, NMI, Rename memory parity error to PCI SERR error

Quoted 17 lines · Huang Ying

memory parity error is only valid for IBM PC-AT, newer machine use 7 bit (0x80) of 0x61 port for PCI SERR. While memory error is usually reported via MCE. So corresponding function name and kernel log string is changed.

---
 arch/x86/include/asm/mach_traps.h |    6 +++---
 arch/x86/kernel/traps.c           |   24 ++++++------------------
 2 files changed, 9 insertions(+), 21 deletions(-)
arch/x86/include/asm/mach_traps.h +0 -0
@@ -9,11 +9,11 @@
 
 #define NMI_REASON_PORT		0x61
 

arch/x86/include/asm/mach_traps.h line 18

Quoted 10 lines
arch/x86/include/asm/mach_traps.h +3 -3
-#define NMI_REASON_MEMPAR	0x80
+#define NMI_REASON_SERR		0x80
 #define NMI_REASON_IOCHK	0x40
-#define NMI_REASON_MASK		(NMI_REASON_MEMPAR | NMI_REASON_IOCHK)
+#define NMI_REASON_MASK		(NMI_REASON_SERR | NMI_REASON_IOCHK)
 
-#define NMI_REASON_CLEAR_MEMPAR	0x04
+#define NMI_REASON_CLEAR_SERR	0x04
 #define NMI_REASON_CLEAR_IOCHK	0x08
 #define NMI_REASON_CLEAR_MASK	0x0f

Haven't you introduced *_MEMPAR with patch 1/6?

-Robert

Signature · 2 lines
Advanced Micro Devices, Inc.
Operating System Research Center
author
#4 lore raw

Re: [RFC 3/6] x86, NMI, Rename memory parity error to PCI SERR error

Quoted 24 lines · Robert Richter
Quoted 23 lines · Huang Ying

memory parity error is only valid for IBM PC-AT, newer machine use 7 bit (0x80) of 0x61 port for PCI SERR. While memory error is usually reported via MCE. So corresponding function name and kernel log string is changed.

---
 arch/x86/include/asm/mach_traps.h |    6 +++---
 arch/x86/kernel/traps.c           |   24 ++++++------------------
 2 files changed, 9 insertions(+), 21 deletions(-)
arch/x86/include/asm/mach_traps.h +2 -2
@@ -9,11 +9,11 @@
 
 #define NMI_REASON_PORT		0x61
 
-#define NMI_REASON_MEMPAR	0x80
+#define NMI_REASON_SERR		0x80
 #define NMI_REASON_IOCHK	0x40
-#define NMI_REASON_MASK		(NMI_REASON_MEMPAR | NMI_REASON_IOCHK)
+#define NMI_REASON_MASK		(NMI_REASON_SERR | NMI_REASON_IOCHK)
 
Quoted 6 lines

arch/x86/include/asm/mach_traps.h line 18

Quoted 4 lines
arch/x86/include/asm/mach_traps.h +1 -1
-#define NMI_REASON_CLEAR_MEMPAR	0x04
+#define NMI_REASON_CLEAR_SERR	0x04
 #define NMI_REASON_CLEAR_IOCHK	0x08
 #define NMI_REASON_CLEAR_MASK	0x0f

Haven't you introduced *_MEMPAR with patch 1/6?

Yes. And I suggest to rename it to *_SERR in this patch to reflect the contemporary hardware better. I think they are two steps: naming the magic number and changing the name of constant and function.

Best Regards,
Huang Ying

#5 lore raw

Re: [RFC 3/6] x86, NMI, Rename memory parity error to PCI SERR error

Quoted 5 lines · Huang Ying
Quoted 1 line

Haven't you introduced *_MEMPAR with patch 1/6?

Yes. And I suggest to rename it to *_SERR in this patch to reflect the contemporary hardware better. I think they are two steps: naming the magic number and changing the name of constant and function.

Hmm, I would rather avoid introducing a name and then immediatly renaming it again.

-Robert

Signature · 2 lines
Advanced Micro Devices, Inc.
Operating System Research Center
author
#6 lore raw

Re: [RFC 3/6] x86, NMI, Rename memory parity error to PCI SERR error

Quoted 9 lines · Robert Richter
Quoted 5 lines · Huang Ying
Quoted 1 line

Haven't you introduced *_MEMPAR with patch 1/6?

Yes. And I suggest to rename it to *_SERR in this patch to reflect the contemporary hardware better. I think they are two steps: naming the magic number and changing the name of constant and function.

Hmm, I would rather avoid introducing a name and then immediatly renaming it again.

I split them into two patches because there is function named as mem_parity_error already. The 1/6 names constants as *_MEMPAR to make it consistent with mem_parity_error(). 3/6 rename both the constants, function name and some string/comments to serr. Do you think this is more clean?

Best Regards,
Huang Ying

#7 lore raw

Re: [RFC 3/6] x86, NMI, Rename memory parity error to PCI SERR error

Quoted 8 lines · Huang Ying
Quoted 7 lines · Robert Richter
Quoted 5 lines · Huang Ying
Quoted 1 line

Haven't you introduced *_MEMPAR with patch 1/6?

Yes. And I suggest to rename it to *_SERR in this patch to reflect the contemporary hardware better. I think they are two steps: naming the magic number and changing the name of constant and function.

Quoted 8 lines
Quoted 2 lines

Hmm, I would rather avoid introducing a name and then immediatly renaming it again.

I split them into two patches because there is function named as mem_parity_error already. The 1/6 names constants as *_MEMPAR to make it consistent with mem_parity_error(). 3/6 rename both the constants, function name and some string/comments to serr. Do you think this is more clean?

If I am not wrong the only real functional change is to rip out the edac handler. So, just introduce the final names with your renaming in patch #1 (and maybe make a comment in the commit message or change patch order).

-Robert

Quoted 11 lines

Best Regards,
Huang Ying

--
To unsubscribe from this list: send the line "unsubscribe linux-kernel" in the body of a message to [email protected]
More majordomo info at http://vger.kernel.org/majordomo-info.html
Please read the FAQ at http://www.tux.org/lkml/

Signature · 2 lines
Advanced Micro Devices, Inc.
Operating System Research Center
author
#8 lore raw

Re: [RFC 3/6] x86, NMI, Rename memory parity error to PCI SERR error

Quoted 18 lines · Robert Richter
Quoted 8 lines · Huang Ying
Quoted 7 lines · Robert Richter
Quoted 5 lines · Huang Ying
Quoted 1 line

Haven't you introduced *_MEMPAR with patch 1/6?

Yes. And I suggest to rename it to *_SERR in this patch to reflect the contemporary hardware better. I think they are two steps: naming the magic number and changing the name of constant and function.

Quoted 8 lines
Quoted 2 lines

Hmm, I would rather avoid introducing a name and then immediatly renaming it again.

I split them into two patches because there is function named as mem_parity_error already. The 1/6 names constants as *_MEMPAR to make it consistent with mem_parity_error(). 3/6 rename both the constants, function name and some string/comments to serr. Do you think this is more clean?

Quoted 4 lines

If I am not wrong the only real functional change is to rip out the edac handler. So, just introduce the final names with your renaming in patch #1 (and maybe make a comment in the commit message or change patch order).

If I merge the 1/6 and 3/6, can you suggest a patch subject, "Add symbol definition for NMI magic constants and rename memory parity to PCI SERR"?

Best Regards,
Huang Ying

#9 lore raw

Re: [RFC 3/6] x86, NMI, Rename memory parity error to PCI SERR error

Quoted 8 lines · Huang Ying
Quoted 4 lines

If I am not wrong the only real functional change is to rip out the edac handler. So, just introduce the final names with your renaming in patch #1 (and maybe make a comment in the commit message or change patch order).

If I merge the 1/6 and 3/6, can you suggest a patch subject, "Add symbol definition for NMI magic constants and rename memory parity to PCI SERR"?

Yes, maybe we make the edac change a separate one.

Thanks,

-Robert

Signature · 2 lines
Advanced Micro Devices, Inc.
Operating System Research Center
#10 lore raw

Re: [RFC 3/6] x86, NMI, Rename memory parity error to PCI SERR error

From: Robert Richter <[email protected]>
Date: Mon, Sep 20, 2010 at 10:00:10AM +0200

(adding Doug to CC)

Quoted 11 lines · Huang Ying
Quoted 8 lines
Quoted 4 lines

If I am not wrong the only real functional change is to rip out the edac handler. So, just introduce the final names with your renaming in patch #1 (and maybe make a comment in the commit message or change patch order).

If I merge the 1/6 and 3/6, can you suggest a patch subject, "Add symbol definition for NMI magic constants and rename memory parity to PCI SERR"?

Yes, maybe we make the edac change a separate one.

What is more, there are a bunch of edac drivers using the PCI SERR nmi as a means to check for PCI errors so we shouldn't be removing it now, should we?

Signature · 5 lines
Regards/Gruss,
Boris.

Operating Systems Research Center
Advanced Micro Devices, Inc.
author
#11 lore raw

Re: [RFC 3/6] x86, NMI, Rename memory parity error to PCI SERR error

Quoted 15 lines · Borislav Petkov

From: Robert Richter <[email protected]>
Date: Mon, Sep 20, 2010 at 10:00:10AM +0200

(adding Doug to CC)

Quoted 10 lines · Huang Ying
Quoted 8 lines
Quoted 4 lines

If I am not wrong the only real functional change is to rip out the edac handler. So, just introduce the final names with your renaming in patch #1 (and maybe make a comment in the commit message or change patch order).

If I merge the 1/6 and 3/6, can you suggest a patch subject, "Add symbol definition for NMI magic constants and rename memory parity to PCI SERR"?

Quoted 5 lines
Quoted 1 line

Yes, maybe we make the edac change a separate one.

What is more, there are a bunch of edac drivers using the PCI SERR nmi as a means to check for PCI errors so we shouldn't be removing it now, should we?

After checking the source, I found in mem_parity_error (will renamed to pci_serr_error), edac_atomic_assert_error() is called, which increase edac_err_assert, edac_err_assert is used in edac_mc_assert_error_check_and_clear(), which is used in edac_mc_workq_function for memory error only, not for PCI errors.

Do we really need PCI SERR NMI for memory errors?

Best Regards,
Huang Ying

#12 lore raw

Re: [RFC 3/6] x86, NMI, Rename memory parity error to PCI SERR error

From: Huang Ying <[email protected]>
Date: Mon, Sep 20, 2010 at 08:22:28PM -0400

(Forgot to add edac-devel to Cc)

Quoted 9 lines
Quoted 3 lines

What is more, there are a bunch of edac drivers using the PCI SERR nmi as a means to check for PCI errors so we shouldn't be removing it now, should we?

After checking the source, I found in mem_parity_error (will renamed to pci_serr_error), edac_atomic_assert_error() is called, which increase edac_err_assert, edac_err_assert is used in edac_mc_assert_error_check_and_clear(), which is used in edac_mc_workq_function for memory error only, not for PCI errors.

Yes, I suppose the edac part in the mem_parity_error() was originally meant for memory parity errors. Now, I understand your incentive of changing that to handle PCI SERR errors but by axing the edac part, you're practically disabling the mci->edac_check() call for edac drivers using NMIs for error reporting (I don't know how many do that, btw...) and almost every edac driver defines that function pointer to a driver-specific error checking function.

So if there are no more IBM PC-AT machines running Linux out, I think we can rip out the whole code around edac_err_assert and thus remove the edac_mc_assert_error_check_and_clear() part from the edac_mc_workq_function() which would make all edac drivers solely poll for mem errors.

What do the others think, Doug?

Signature · 8 lines
Regards/Gruss,
Boris.

Advanced Micro Devices GmbH
Einsteinring 24, 85609 Dornach
General Managers: Alberto Bozzo, Andrew Bowd
Registration: Dornach, Gemeinde Aschheim, Landkreis Muenchen
Registergericht Muenchen, HRB Nr. 43632
#13 lore raw

Re: [RFC 3/6] x86, NMI, Rename memory parity error to PCI SERR error

Quoted 9 lines · Borislav Petkov

From: Borislav Petkov <[email protected]>
Subject: Re: [RFC 3/6] x86, NMI, Rename memory parity error to PCI SERR error
To: "Huang Ying" <[email protected]>
Cc: "Richter, Robert" <[email protected]>, "Ingo Molnar" <[email protected]>, "H. Peter Anvin" <[email protected]>, "[email protected]" <[email protected]>, "Andi Kleen" <[email protected]>, "Doug Thompson" <[email protected]>, "edac-devel" <[email protected]>
Date: Tuesday, September 21, 2010, 12:37 AM
From: Huang Ying <[email protected]>
Date: Mon, Sep 20, 2010 at 08:22:28PM -0400

(Forgot to add edac-devel to Cc)

thanks

Quoted 41 lines
Quoted 2 lines
Quoted 2 lines

What is more, there are a bunch of edac drivers

using the PCI SERR nmi

Quoted 1 line
Quoted 1 line

as a means to check for PCI errors so we

shouldn't be removing it now,

Quoted 3 lines
Quoted 1 line

should we?

After checking the source, I found in mem_parity_error

(will renamed to

Quoted 1 line

pci_serr_error), edac_atomic_assert_error() is called,

which increase

Quoted 2 lines

edac_err_assert, edac_err_assert is used in edac_mc_assert_error_check_and_clear(), which is used

in

Quoted 1 line

edac_mc_workq_function for memory error only, not for

PCI errors.

Yes, I suppose the edac part in the mem_parity_error() was originally
meant for memory parity errors. Now, I understand your incentive of
changing that to handle PCI SERR errors but by axing the edac part,
you're practically disabling the mci->edac_check() call for edac
drivers using NMIs for error reporting (I don't know how many do that,
btw...) and almost every edac driver defines that function pointer to a
driver-specific error checking function.

So if there are no more IBM PC-AT machines running Linux out, I
think we can rip out the whole code around edac_err_assert and thus
remove the edac_mc_assert_error_check_and_clear() part from the
edac_mc_workq_function() which would make all edac drivers solely poll
for mem errors.

Quoted 1 line

What do the others think, Doug?

History lesson:

The addition of PCI bus ERROR scanning was added to EDAC when we (at Linux Networx, 2005 timeframe) discovered a bad PCI riser card. Due to bad manufacturing, it would cause PCI bus errors during transfers. We determined that by scanning the PCI bus, we could isolate bad cards and verify proper PCI bus operation on those cards and other PCI devices.

Once that was in place, we found other systems that also had PCI bus errors, though less frequently than our original system. A handful of errors were discovered and we found that not many systems were handling any of them, nor even reporting them. Hence the PCI bus scanner of EDAC. I profiled it, and it took on average 2500 TSC cycles to perform a complete bus scan which occurred every second. So it was an expensive operation. That is why I added controls to turn off PCI bus scanning if desired.

Memory parity errors via NMI was NOT the prime reason it was added to EDAC. Looking at SERR (or any PCI bus error) was the prime reason it was added. There was a patch to better handle NMI event handling better, but didn't get pushed upstream.'

I have not kept up with the kernel PCI SERR error handling code development else (if any) and don't fully know its features nor current operation.

If it is determined that EDAC no longer needs to scan the PCI bus because some other system is now managing that error checking operation, great!

It was a wart in a way, hanging in the memory driver. I really wanted to revisit that and either set it up as a separate driver or embedded in the EDAC core or something.

doug thompson

Quoted 13 lines
Signature · 10 lines
-- 
Regards/Gruss,
Boris.

Advanced Micro Devices GmbH
Einsteinring 24, 85609 Dornach
General Managers: Alberto Bozzo, Andrew Bowd
Registration: Dornach, Gemeinde Aschheim, Landkreis
Muenchen
Registergericht Muenchen, HRB Nr. 43632
#14 lore raw

Re: [RFC 3/6] x86, NMI, Rename memory parity error to PCI SERR error

In reply to Huang Ying
Quoted 4 lines · Huang Ying

memory parity error is only valid for IBM PC-AT, newer machine use 7 bit (0x80) of 0x61 port for PCI SERR. While memory error is usually reported via MCE. So corresponding function name and kernel log string is changed.

Things perhaps changed over the last few years while I have not been
watching, but for many years the bit #7 of the NMI status port (implemented by the southbridge at 0x61 in the port I/O space) was still used for memory parity or ECC errors even after the original IBM PC/AT. The usual arrangement was in the event of a memory error the memory controller in the northbridge would assert the chip's PCI SERR output line, which in turn would be trapped by the southbridge and converted to an NMI event while setting the said bit in the NMI status port. See e.g. the 82349HX System Controller datasheet (Intel document number 290551).

So the name of the error reported is not that unjustified except, of
course, to be precise the handler would have to scan the state of the SERR output reported by all the PCI devices in the PCI configuration space to find the originator and then interpret the event accordingly. Which obviously means the only piece of code that could exactly know what the reason was is the respective device driver as causes of SERR are device-specific and may require processing of device-specific registers to determine the cause and/or recover (a device reset may be required in some cases).

Of course using the MCE seems natural and better, especially if the
exception can be raised synchronously and stop the failing memory load CPU instruction from completion -- this is important for parity and MBE ECC errors, where in some cases the handler may be able to retry the failing operation having refreshed RAM from the backing store or otherwise the affected process must be killed (unless a kernel memory location is involved that is, where the whole system has to be brought down).

OTOH, for CPU stores and DMA transactions the event will always be
asynchronous and an NMI might be a better option, as in the case of parity and MBE ECC errors the whole system will probably have to be brought down, and with SBE ECC errors scrubbing can be done at any time and otherwise (except from logging and/or marking the physical page bad, as required) no action is needed.

Maciej

#15 lore raw

Re: [RFC 3/6] x86, NMI, Rename memory parity error to PCI SERR error

Hi, Maciej,

Quoted 7 lines · Maciej W. Rozycki
Quoted 4 lines · Huang Ying

memory parity error is only valid for IBM PC-AT, newer machine use 7 bit (0x80) of 0x61 port for PCI SERR. While memory error is usually reported via MCE. So corresponding function name and kernel log string is changed.

Quoted 9 lines

 Things perhaps changed over the last few years while I have not been watching, but for many years the bit #7 of the NMI status port (implemented by the southbridge at 0x61 in the port I/O space) was still used for memory parity or ECC errors even after the original IBM PC/AT. The usual arrangement was in the event of a memory error the memory controller in the northbridge would assert the chip's PCI SERR output line, which in turn would be trapped by the southbridge and converted to an NMI event while setting the said bit in the NMI status port.  See e.g. the 82349HX System Controller datasheet (Intel document number 290551).

Thanks for your information. So EDAC function call in NMI handler should be kept? But as you pointed out, the function name of corresponding handler should be PCI SERR instead of memory parity. It just can be used to report memory error on some system. I think we can rename the function and string to PCI SERR and add some comments for EDAC function call that checks memory errors.

Quoted 9 lines

 So the name of the error reported is not that unjustified except, of course, to be precise the handler would have to scan the state of the SERR output reported by all the PCI devices in the PCI configuration space to find the originator and then interpret the event accordingly.  Which obviously means the only piece of code that could exactly know what the reason was is the respective device driver as causes of SERR are device-specific and may require processing of device-specific registers to determine the cause and/or recover (a device reset may be required in some cases).

In addition to PCI SERR, I think modern system rely more on PCIE AER, which can report more information about error. There are recovery support for PCIE AER in kernel already. Do we need some similar mechanism for PCI SERR? Because PCIE AER becomes more and more common on server platform, I think some minimal check such as scaning devices SERR/PERR bit should be sufficient.

Quoted 8 lines

 Of course using the MCE seems natural and better, especially if the exception can be raised synchronously and stop the failing memory load CPU instruction from completion -- this is important for parity and MBE ECC errors, where in some cases the handler may be able to retry the failing operation having refreshed RAM from the backing store or otherwise the affected process must be killed (unless a kernel memory location is involved that is, where the whole system has to be brought down).

Quoted 6 lines

 OTOH, for CPU stores and DMA transactions the event will always be asynchronous and an NMI might be a better option, as in the case of parity and MBE ECC errors the whole system will probably have to be brought down, and with SBE ECC errors scrubbing can be done at any time and otherwise (except from logging and/or marking the physical page bad, as required) no action is needed.

In fact, MCE is a special exception, it can be used for asynchronous events too. Such as memory error detected by patrol scrubbing, please take a look at latest Intel 64 and IA32 architectures software developer's manual Vol 3A section 15.9.3: Architecturally Defined UCR Errors.

Best Regards,
Huang Ying

#16 lore raw

Re: [RFC 3/6] x86, NMI, Rename memory parity error to PCI SERR error

Hi Huang,

Quoted 10 lines
Quoted 9 lines

 Things perhaps changed over the last few years while I have not been watching, but for many years the bit #7 of the NMI status port (implemented by the southbridge at 0x61 in the port I/O space) was still used for memory parity or ECC errors even after the original IBM PC/AT. The usual arrangement was in the event of a memory error the memory controller in the northbridge would assert the chip's PCI SERR output line, which in turn would be trapped by the southbridge and converted to an NMI event while setting the said bit in the NMI status port.  See e.g. the 82349HX System Controller datasheet (Intel document number 290551).

Quoted 2 lines

Thanks for your information. So EDAC function call in NMI handler should be kept?

I've seen EDAC mentioned before, but I lack further details as to what it
is -- if you give me a link to the relevant piece of documentation, then I'll see if I can find some time to look into it. Otherwise I can't comment on it, sorry.

Quoted 5 lines

But as you pointed out, the function name of
corresponding handler should be PCI SERR instead of memory parity. It just can be used to report memory error on some system. I think we can rename the function and string to PCI SERR and add some comments for EDAC function call that checks memory errors.

Linux can certainly run on pre-PCI x86 machines; while I agree it makes
sense to update the references to match reality, I think you need to be careful about it. "A memory or system error" might be a good compromise -- mentioning parity explicitly may not be a good idea; I'm sure there must have been ECC x86 systems made even before PCI (think EISA -- these often were quite sophisticated; unsure about IBM's MCA).

Otherwise if you think you absolutely *must* mention "PCI SERR" where
relevant, then I suggest that you investigate how to determine whether the kernel is running on a PCI system (something along the lines of (CONFIG_PCI && pci_host_bridge_found)) and adjust the message accordingly based on actual configuration.

Quoted 10 lines
Quoted 9 lines

 So the name of the error reported is not that unjustified except, of course, to be precise the handler would have to scan the state of the SERR output reported by all the PCI devices in the PCI configuration space to find the originator and then interpret the event accordingly.  Which obviously means the only piece of code that could exactly know what the reason was is the respective device driver as causes of SERR are device-specific and may require processing of device-specific registers to determine the cause and/or recover (a device reset may be required in some cases).

Quoted 6 lines

In addition to PCI SERR, I think modern system rely more on PCIE AER, which can report more information about error. There are recovery support for PCIE AER in kernel already. Do we need some similar mechanism for PCI SERR? Because PCIE AER becomes more and more common on server platform, I think some minimal check such as scaning devices SERR/PERR bit should be sufficient.

Even you you don't care about 1990s' vintage computers (which you have
the right not to) I'd expect legacy PCI/PCI-X to stay around for a while as it was with ISA as not all option cards will ever be redesigned as PCIe options. And even if they do, then people may not necessarily want to throw away all their old still-working peripheral hardware when they upgrade the system, especially the more sophisticated or expensive bits.

Quoted 7 lines
Quoted 6 lines

 OTOH, for CPU stores and DMA transactions the event will always be asynchronous and an NMI might be a better option, as in the case of parity and MBE ECC errors the whole system will probably have to be brought down, and with SBE ECC errors scrubbing can be done at any time and otherwise (except from logging and/or marking the physical page bad, as required) no action is needed.

Quoted 5 lines

In fact, MCE is a special exception, it can be used for asynchronous events too. Such as memory error detected by patrol scrubbing, please take a look at latest Intel 64 and IA32 architectures software developer's manual Vol 3A section 15.9.3: Architecturally Defined UCR Errors.

The MCE has always had an asynchronous option -- even the original
implementation in the Pentium CPU had a BUSCHK# input line which was cleverly used by the i430NX/Neptune chipset to signal hard ECC errors (so the handler had the address of the failing bus transaction readily available in the Machine Check Address MSR), but regrettably never after.

The problem with using the MCE for non-CPU related events such as DMA
transfers that failed is the choice of the target CPU in multi-processor systems. Is there a well-defined way for routing such events that an OS like Linux could use? It better not went to an off-line processor for example.

There's no such problem with system NMIs as they can be routed to the CPU
of choice by means of LINT1 input configuration in the local APIC unit of the concerned processors.

Maciej

author
#17 lore raw

RFC 4/6 x86, NMI, Rewrite NMI handler

In reply to Huang Ying

The original NMI handler is quite outdated in many aspects. This patch try to fix it.

In original code, NMI reason io port (0x61) is only processed on BSP. This makes it impossible to hot-remove BSP. To solve the issue, a raw spinlock is used to make the port can be processed on any CPU.

Because the io port 0x61 and other non-CPU-specific NMI sources can be processed on any CPU, the order to process the NMI sources are changed as follow:

notify_die(DIE_NMI_IPI);
notify_die(DIE_NMI);
/* process io port 0x61 */
nmi_watchdog_touch();
notify_die(DIE_NMIUNKNOWN);
unknown_nmi();

DIE_NMI_IPI is used to process CPU specific NMI sources, such as perf event, oprofile, crash IPI, etc. While DIE_NMI is used to process non-CPU-specific NMI sources, such as APEI (ACPI Platform Error Interface) GHES (Generic Hardware Error Source), etc.

DIE_NMI_IPI must be processed before DIE_NMI. For example, perf event trigger a NMI on CPU 1, at the same time, APEI GHES trigger another NMI on CPU 0. If DIE_NMI is processed before DIE_NMI_IPI, it is possible that APEI GHES is processed on CPU 1, while unknown NMI is gotten on CPU 0.

In this new order of processing, performance sensitive NMI sources such as oprofile or perf event will have better performance because the time consuming IO port reading is done after them.

Only one NMI is eaten for each NMI handler call, even for PCI SERR and IOCHK NMIs. Because one NMI should be raised for each of them, eating too many NMI will cause unnecessary unknown NMI.

The die value used in NMI sources are fixed accordingly.

The NMI handler in the patch is designed by Andi Kleen.

---
 arch/x86/kernel/cpu/perf_event.c  |    1 
 arch/x86/kernel/traps.c           |   86 +++++++++++++++++++-------------------
 arch/x86/oprofile/nmi_int.c       |    1 
 arch/x86/oprofile/nmi_timer_int.c |    2 
 drivers/char/ipmi/ipmi_watchdog.c |    2 
 drivers/watchdog/hpwdt.c          |    2 
 6 files changed, 48 insertions(+), 46 deletions(-)
arch/x86/kernel/cpu/perf_event.c +0 -1
@@ -1237,7 +1237,6 @@ perf_event_nmi_handler(struct notifier_b
 		return NOTIFY_DONE;
 
 	switch (cmd) {
-	case DIE_NMI:
 	case DIE_NMI_IPI:
 		break;
 	case DIE_NMIUNKNOWN:
arch/x86/kernel/traps.c +45 -41

115 lines not on this page · raw message

arch/x86/oprofile/nmi_int.c +0 -1
@@ -64,7 +64,6 @@ static int profile_exceptions_notify(str
 	int ret = NOTIFY_DONE;
 
 	switch (val) {
-	case DIE_NMI:
 	case DIE_NMI_IPI:
 		if (ctr_running)
 			model->check_ctrs(args->regs, &__get_cpu_var(cpu_msrs));
arch/x86/oprofile/nmi_timer_int.c +1 -1
@@ -25,7 +25,7 @@ static int profile_timer_exceptions_noti
 	int ret = NOTIFY_DONE;
 
 	switch (val) {
-	case DIE_NMI:
+	case DIE_NMI_IPI:
 		oprofile_add_sample(args->regs, 0);
 		ret = NOTIFY_STOP;
 		break;
drivers/char/ipmi/ipmi_watchdog.c +1 -1
@@ -1080,7 +1080,7 @@ ipmi_nmi(struct notifier_block *self, un
 {
 	struct die_args *args = data;
 
-	if (val != DIE_NMI)
+	if (val != DIE_NMIUNKNOWN)
 		return NOTIFY_OK;
 
 	/* Hack, if it's a memory or I/O error, ignore it. */
drivers/watchdog/hpwdt.c +1 -1
@@ -469,7 +469,7 @@ static int hpwdt_pretimeout(struct notif
 	unsigned long rom_pl;
 	static int die_nmi_called;
 
-	if (ulReason != DIE_NMI && ulReason != DIE_NMI_IPI)
+	if (ulReason != DIE_NMIUNKNOWN)
 		goto out;
 
 	if (!hpwdt_nmi_decoding)
#18 lore raw

Re: [RFC 4/6] x86, NMI, Rewrite NMI handler

Quoted 6 lines · Huang Ying

The original NMI handler is quite outdated in many aspects. This patch try to fix it.

In original code, NMI reason io port (0x61) is only processed on BSP. This makes it impossible to hot-remove BSP. To solve the issue, a raw spinlock is used to make the port can be processed on any CPU.

Do we really want to use a spinlock inside the nmi handler?

I thought the NMIs sent to the io port are only routed to one cpu as determined by the io-apic? Is it spread out to other cpus now?

Cheers,
Don

#19 lore raw

Re: [RFC 4/6] x86, NMI, Rewrite NMI handler

Quoted 9 lines · On Fri, 10 Sep 2010 11:56:05 -0400 Don Zickus <[email protected]> wrote:
Quoted 6 lines · Huang Ying

The original NMI handler is quite outdated in many aspects. This patch try to fix it.

In original code, NMI reason io port (0x61) is only processed on BSP. This makes it impossible to hot-remove BSP. To solve the issue, a raw spinlock is used to make the port can be processed on any CPU.

Do we really want to use a spinlock inside the nmi handler?

As long as it's only between CPUs
(that is only ever used between different NMI handlers) that's fine. It's certainly safer than having races between CPUs.

Quoted 2 lines

I thought the NMIs sent to the io port are only routed to one cpu as determined by the io-apic? Is it spread out to other cpus now?

There can be cases where it can happen I believe.

-Andi

Signature · 1 line
[email protected] -- Speaking for myself only.
#20 lore raw

Re: [RFC 4/6] x86, NMI, Rewrite NMI handler

Quoted 13 lines · Andi Kleen
Quoted 9 lines · On Fri, 10 Sep 2010 11:56:05 -0400 Don Zickus <[email protected]> wrote:
Quoted 6 lines · Huang Ying

The original NMI handler is quite outdated in many aspects. This patch try to fix it.

In original code, NMI reason io port (0x61) is only processed on BSP. This makes it impossible to hot-remove BSP. To solve the issue, a raw spinlock is used to make the port can be processed on any CPU.

Do we really want to use a spinlock inside the nmi handler?

Quoted 8 lines

As long as it's only between CPUs
(that is only ever used between different NMI handlers) that's fine. It's certainly safer than having races between CPUs.

Quoted 2 lines

I thought the NMIs sent to the io port are only routed to one cpu as determined by the io-apic? Is it spread out to other cpus now?

There can be cases where it can happen I believe.

The reason I asked was, I thought it would be easier to have a global variable that tells the nmi handler which cpu has the NMI's routed to its io port. This way if you want to swap out the bsp cpu, you could perhaps just re-route the nmi to a new cpu and the global variable would be updated accordingly?

Cheers,
Don

author
#21 lore raw

Re: [RFC 4/6] x86, NMI, Rewrite NMI handler

Quoted 21 lines · Don Zickus
Quoted 20 lines · Andi Kleen
Quoted 9 lines · On Fri, 10 Sep 2010 11:56:05 -0400 Don Zickus <[email protected]> wrote:
Quoted 6 lines · Huang Ying

The original NMI handler is quite outdated in many aspects. This patch try to fix it.

In original code, NMI reason io port (0x61) is only processed on BSP. This makes it impossible to hot-remove BSP. To solve the issue, a raw spinlock is used to make the port can be processed on any CPU.

Do we really want to use a spinlock inside the nmi handler?

As long as it's only between CPUs
(that is only ever used between different NMI handlers) that's fine. It's certainly safer than having races between CPUs.

Quoted 2 lines

I thought the NMIs sent to the io port are only routed to one cpu as determined by the io-apic? Is it spread out to other cpus now?

Quoted 7 lines
Quoted 1 line

There can be cases where it can happen I believe.

The reason I asked was, I thought it would be easier to have a global variable that tells the nmi handler which cpu has the NMI's routed to its io port. This way if you want to swap out the bsp cpu, you could perhaps just re-route the nmi to a new cpu and the global variable would be updated accordingly?

Then we need some kind of protection or race condition between re-routing NMI and updating the variable. Do you think so?

Best Regards,
Huang Ying

#22 lore raw

Re: [RFC 4/6] x86, NMI, Rewrite NMI handler

Quoted 8 lines · Huang Ying
Quoted 5 lines

The reason I asked was, I thought it would be easier to have a global variable that tells the nmi handler which cpu has the NMI's routed to its io port. This way if you want to swap out the bsp cpu, you could perhaps just re-route the nmi to a new cpu and the global variable would be updated accordingly?

Then we need some kind of protection or race condition between re-routing NMI and updating the variable. Do you think so?

Well, I thought the only reasonable place to update the variable is when the cpu is being taken offline, during the MTRR update. Since no NMIs can be processed when the cpu's are syncing their MTRR, there shouldn't be a race condition, no?

Then again I am probably missing something obvious. Like I don't know how cpu's deal with interrupts/NMIs when they are going offline.

It was just a thought to avoid the spinlock.

Cheers,
Don

author
#23 lore raw

Re: [RFC 4/6] x86, NMI, Rewrite NMI handler

Quoted 15 lines · Don Zickus
Quoted 8 lines · Huang Ying
Quoted 5 lines

The reason I asked was, I thought it would be easier to have a global variable that tells the nmi handler which cpu has the NMI's routed to its io port. This way if you want to swap out the bsp cpu, you could perhaps just re-route the nmi to a new cpu and the global variable would be updated accordingly?

Then we need some kind of protection or race condition between re-routing NMI and updating the variable. Do you think so?

Well, I thought the only reasonable place to update the variable is when the cpu is being taken offline, during the MTRR update. Since no NMIs can be processed when the cpu's are syncing their MTRR, there shouldn't be a race condition, no?

Quoted 4 lines

Then again I am probably missing something obvious. Like I don't know how cpu's deal with interrupts/NMIs when they are going offline.

It was just a thought to avoid the spinlock.

Why do you hate spinlock inside NMI handler? I think it is safe and simple if only used in NMI handler.

Best Regards,
Huang Ying

#24 lore raw

Re: [RFC 4/6] x86, NMI, Rewrite NMI handler

Quoted 16 lines · Huang Ying
Quoted 15 lines · Don Zickus
Quoted 8 lines · Huang Ying
Quoted 5 lines

The reason I asked was, I thought it would be easier to have a global variable that tells the nmi handler which cpu has the NMI's routed to its io port. This way if you want to swap out the bsp cpu, you could perhaps just re-route the nmi to a new cpu and the global variable would be updated accordingly?

Then we need some kind of protection or race condition between re-routing NMI and updating the variable. Do you think so?

Well, I thought the only reasonable place to update the variable is when the cpu is being taken offline, during the MTRR update. Since no NMIs can be processed when the cpu's are syncing their MTRR, there shouldn't be a race condition, no?

Quoted 7 lines
Quoted 4 lines

Then again I am probably missing something obvious. Like I don't know how cpu's deal with interrupts/NMIs when they are going offline.

It was just a thought to avoid the spinlock.

Why do you hate spinlock inside NMI handler? I think it is safe and simple if only used in NMI handler.

I guess I always had the mentality that spinlocks in an NMI context was a big no-no. Never really thought about if there were safe use-cases or not.

Cheers,
Don

#25 lore raw

Re: [RFC 4/6] x86, NMI, Rewrite NMI handler

In reply to Huang Ying
Quoted 10 lines · Huang Ying

DIE_NMI_IPI is used to process CPU specific NMI sources, such as perf event, oprofile, crash IPI, etc. While DIE_NMI is used to process non-CPU-specific NMI sources, such as APEI (ACPI Platform Error Interface) GHES (Generic Hardware Error Source), etc.

DIE_NMI_IPI must be processed before DIE_NMI. For example, perf event trigger a NMI on CPU 1, at the same time, APEI GHES trigger another NMI on CPU 0. If DIE_NMI is processed before DIE_NMI_IPI, it is possible that APEI GHES is processed on CPU 1, while unknown NMI is gotten on CPU 0.

Ingo, Peter,

couldn't this be the reason for the regression we are observing with unhandled nmis?

-Robert

Signature · 2 lines
Advanced Micro Devices, Inc.
Operating System Research Center
author
#26 lore raw

RFC 5/6 x86, NMI, Add support to notify hardware error with unknown NMI

In reply to Huang Ying

On some platforms, fatal hardware error may be notified via unknown NMI.

For example, on some platform with APEI firmware first mode support, firmware generates NMI for fatal error but without error record. The unknown NMI should be treated as notification of fatal hardware error. The unknown_nmi_for_hwerr is added for these platform, if it is not zero, system will treat unknown NMI as notification of fatal hardware error.

These platforms are identified via the presentation of APEI HEST or some PCI ID of the host bridge. The PCI ID of host bridge instead of DMI ID is used, so that the checking can be done based on the platform type instead of motherboard. This should be simpler and sufficient.

The method to identify the platforms is designed by Andi Kleen.

# TODO

- Report unknown NMI as fatal hardware error with new hardware error reporting interface, so that the error record can be gotten after system reboot.

- Because printk is not safe in NMI handler, all printk except that called after bust_spinlocks will be called in a delayed IRQ context to prevent system deadlock.

---
 arch/x86/include/asm/nmi.h |    1 
 arch/x86/kernel/Makefile   |    2 +
 arch/x86/kernel/hwerr.c    |   55 +++++++++++++++++++++++++++++++++++++++++++++
 arch/x86/kernel/traps.c    |   10 ++++++++
 drivers/acpi/apei/hest.c   |    8 ++++++
 5 files changed, 76 insertions(+)
 create mode 100644 arch/x86/kernel/hwerr.c
arch/x86/include/asm/nmi.h +1 -0
@@ -44,6 +44,7 @@ struct ctl_table;
 extern int proc_nmi_enabled(struct ctl_table *, int ,
 			void __user *, size_t *, loff_t *);
 extern int unknown_nmi_panic;
+extern int unknown_nmi_for_hwerr;
 
 void arch_trigger_all_cpu_backtrace(void);
 #define arch_trigger_all_cpu_backtrace arch_trigger_all_cpu_backtrace
arch/x86/kernel/Makefile +2 -0
@@ -116,6 +116,8 @@ obj-$(CONFIG_X86_CHECK_BIOS_CORRUPTION)
 
 obj-$(CONFIG_SWIOTLB)			+= pci-swiotlb.o
 
+obj-y					+= hwerr.o
+
 ###
 # 64 bit specific files
 ifeq ($(CONFIG_X86_64),y)
arch/x86/kernel/hwerr.c new file +55 -0

55 lines not on this page · raw message

arch/x86/kernel/traps.c +10 -0
@@ -83,6 +83,8 @@ EXPORT_SYMBOL_GPL(used_vectors);
 
 static int ignore_nmis;
 
+int unknown_nmi_for_hwerr;
+
 /*
  * Prevent NMI reason port (0x61) being accessed simultaneously, can
  * only be used in NMI handler.
@@ -349,6 +351,14 @@ io_check_error(unsigned char reason, str
 static notrace __kprobes void
 unknown_nmi_error(unsigned char reason, struct pt_regs *regs)
 {
+	/*
+	 * On some platforms, hardware errors may be notified via
+	 * unknown NMI
+	 */
+	if (unknown_nmi_for_hwerr)
+		panic("NMI for hardware error without error record: "
+		      "Not continuing");
+
 #ifdef CONFIG_MCA
 	/*
 	 * Might actually be able to figure out what the guilty party
drivers/acpi/apei/hest.c +8 -0
@@ -35,6 +35,7 @@
 #include <linux/highmem.h>
 #include <linux/io.h>
 #include <linux/platform_device.h>
+#include <linux/nmi.h>
 #include <acpi/apei.h>
 
 #include "apei-internal.h"
@@ -222,6 +223,13 @@ static int __init hest_init(void)
 	if (rc)
 		goto err;
 
+	/*
+	 * System has proper HEST should treat unknown NMI as fatal
+	 * hardware error notification
+	 */
+	pr_info("HEST is valid, will treat unknown NMI as hardware error!\n");
+	unknown_nmi_for_hwerr = 1;
+
 	rc = hest_ghes_dev_register(ghes_count);
 	if (rc)
 		goto err;
#27 lore raw

Re: [RFC 5/6] x86, NMI, Add support to notify hardware error with unknown NMI

arch/x86/kernel/traps.c io_check_error(unsigned char reason, str · line 362

Quoted 13 lines
arch/x86/kernel/traps.c +8 -0
@@ -349,6 +351,14 @@ io_check_error(unsigned char reason, str
 static notrace __kprobes void
 unknown_nmi_error(unsigned char reason, struct pt_regs *regs)
 {
+	/*
+	 * On some platforms, hardware errors may be notified via
+	 * unknown NMI
+	 */
+	if (unknown_nmi_for_hwerr)
+		panic("NMI for hardware error without error record: "
+		      "Not continuing");
+
 #ifdef CONFIG_MCA

I'm not sure I agree with this. I still see PCI SERR's not coming in through port 0x61 and get routed to unknown_nmi_error. Not sure we should just assume that it is an APEI/HEST error and panic the box.

Also all the perf problems we have seen recently have been going through that path as we slowly try to figure out why we are not catching those unknown nmis.

I am grasping for straws here, but is there a register that APEI/HEST can poke to see if it generated the NMI?

Cheers,
Don

#28 lore raw

Re: [RFC 5/6] x86, NMI, Add support to notify hardware error with unknown NMI

Quoted 3 lines

I am grasping for straws here, but is there a register that APEI/HEST can poke to see if it generated the NMI?

HEST knows this yes.

But this is not about HEST errors, but about those without HEST handling.

-Andi

#29 lore raw

Re: [RFC 5/6] x86, NMI, Add support to notify hardware error with unknown NMI

Quoted 8 lines · Andi Kleen
Quoted 3 lines

I am grasping for straws here, but is there a register that APEI/HEST can poke to see if it generated the NMI?

HEST knows this yes.

But this is not about HEST errors, but about those without HEST handling.

Don't most unknown NMIs fall into the same boat, that they were not being handled properly?

On the other hand could you use the die_notifier_chain(DIE_UNKNOWNNMI) for the same purpose and keep the unknown_nmi_error() handler a little cleaner?

Cheers,
Don

author
#30 lore raw

Re: [RFC 5/6] x86, NMI, Add support to notify hardware error with unknown NMI

Quoted 5 lines · Don Zickus
Quoted 4 lines · Andi Kleen
Quoted 3 lines

I am grasping for straws here, but is there a register that APEI/HEST can poke to see if it generated the NMI?

Quoted 7 lines
Quoted 4 lines

HEST knows this yes.

But this is not about HEST errors, but about those without HEST handling.

Don't most unknown NMIs fall into the same boat, that they were not being handled properly?

As far as I know, at least on some platforms, unknown NMIs are used for hardware error reporting. They will cause "Blue Screen" in Windows.

Quoted 3 lines

On the other hand could you use the die_notifier_chain(DIE_UNKNOWNNMI) for the same purpose and keep the unknown_nmi_error() handler a little cleaner?

I think explicit function call has better readability than notifier chain.

Best Regards,
Huang Ying

#31 lore raw

Re: [RFC 5/6] x86, NMI, Add support to notify hardware error with unknown NMI

Quoted 8 lines · Huang Ying
Quoted 7 lines · Don Zickus
Quoted 6 lines · Andi Kleen
Quoted 3 lines

I am grasping for straws here, but is there a register that APEI/HEST can poke to see if it generated the NMI?

HEST knows this yes.

Quoted 8 lines
Quoted 5 lines
Quoted 2 lines

But this is not about HEST errors, but about those without HEST handling.

Don't most unknown NMIs fall into the same boat, that they were not being handled properly?

As far as I know, at least on some platforms, unknown NMIs are used for hardware error reporting. They will cause "Blue Screen" in Windows.

Unfortunately, most of the bugzillas I deal with, unkown NMIs are the result of SERRs. While you can consider that hardware error reporting, the easiest way for me to debug those problems currently is to have reporters run 'lspci -vvv' after the NMI is displayed to figure out who caused the NMI.

My fear is that panic'ing the box on unknown NMIs on those platforms will hinder my ability to easily debug those NMIs.

Quoted 7 lines
Quoted 4 lines

On the other hand could you use the die_notifier_chain(DIE_UNKNOWNNMI) for the same purpose and keep the unknown_nmi_error() handler a little cleaner?

I think explicit function call has better readability than notifier chain.

Ok. What criteria should we establish to determine which functions go on the notifier chain and which ones can explicitly called?

Cheers,
Don

#32 lore raw

Re: [RFC 5/6] x86, NMI, Add support to notify hardware error with unknown NMI

Don,

Quoted 8 lines

Unfortunately, most of the bugzillas I deal with, unkown NMIs are the result of SERRs. While you can consider that hardware error reporting, the easiest way for me to debug those problems currently is to have reporters run 'lspci -vvv' after the NMI is displayed to figure out who caused the NMI.

My fear is that panic'ing the box on unknown NMIs on those platforms will hinder my ability to easily debug those NMIs.

The reason the NMI is sent is that there is a "lost" data corruption somewhere in the system and if you don't stop it the system the corruption may make it to disk, become permanent etc. The hardware was designed under the assumption that the system is stopped by software when this happens (same reason as why many machine checks cause panics)

Then there isn't necessarily something to "debug": data corruption can happen without any bugs being around (and in fact that's the common case, assuming production systems)

So I'm not sure what you're debugging here. Are you being the support technician for the system through bugzilla? That sounds inefficient.

Anyways for hardware support we could probably dump some more information at panic or better through error serialization, but the word "debug" is IMHO an very wrong way to think about that.

If this is about driver debugging it's entirely reasonable to have a special setting (e.g. disable the panic), but the defaults should be set in a way to avoid spreading data corruption,.

-Andi

Signature · 1 line
[email protected] -- Speaking for myself only.
#33 lore raw

Re: [RFC 5/6] x86, NMI, Add support to notify hardware error with unknown NMI

Quoted 13 lines · Andi Kleen

Don,

Quoted 8 lines

Unfortunately, most of the bugzillas I deal with, unkown NMIs are the result of SERRs. While you can consider that hardware error reporting, the easiest way for me to debug those problems currently is to have reporters run 'lspci -vvv' after the NMI is displayed to figure out who caused the NMI.

My fear is that panic'ing the box on unknown NMIs on those platforms will hinder my ability to easily debug those NMIs.

Quoted 7 lines

The reason the NMI is sent is that there is a "lost" data corruption somewhere in the system and if you don't stop it the system the corruption may make it to disk, become permanent etc. The hardware was designed under the assumption that the system is stopped by software when this happens (same reason as why many machine checks cause panics)

Yeah, I know. I was being too ignorant perhaps.

Quoted 8 lines

Then there isn't necessarily something to "debug": data corruption can happen without any bugs being around (and in fact that's the common case, assuming production systems)

So I'm not sure what you're debugging here. Are you being the support technician for the system through bugzilla? That sounds inefficient.

The problem I repeatedly deal with for RHEL systems is a customer sees an unknown NMI printed on their screen and sometimes the machine falls apart shortly after, sometimes it doesn't. Obviously they are going to file a bug asking why. A chunk of the problems are bad hardware/firmware. But the problem is which one.

Replacing a slot card is easy, replacing a motherboard is not. So I usually try to determine which device is failing by walking the pci bus and looking for the serr bits or some of the pci-e status bits.

It is inefficient, but I haven't had time to figure out a way to clean it up. And just for the record, I usually see an unknown NMI report every other week.

Quoted 5 lines

Anyways for hardware support we could probably dump some more information at panic or better through error serialization, but the word "debug" is IMHO an very wrong way to think about that.

Well, I can use 'diagnos' or 'determine' if you want. But at the end of the day, we have customers that see scary software messages and expect us to give them reasonable direction to fix their problems.

Quoted 5 lines

If this is about driver debugging it's entirely reasonable to have a special setting (e.g. disable the panic), but the defaults should be set in a way to avoid spreading data corruption,.

Ok. I can accept that.

Cheers,
Don

#34 lore raw

Re: [RFC 5/6] x86, NMI, Add support to notify hardware error with unknown NMI

Quoted 10 lines · On Mon, 13 Sep 2010 11:47:50 -0400 Don Zickus <[email protected]> wrote:
Quoted 9 lines

Then there isn't necessarily something to "debug": data corruption can happen without any bugs being around (and in fact that's the common case, assuming production systems)

So I'm not sure what you're debugging here. Are you being the support technician for the system through bugzilla? That sounds inefficient.

Quoted 5 lines

The problem I repeatedly deal with for RHEL systems is a customer sees an unknown NMI printed on their screen and sometimes the machine falls apart shortly after, sometimes it doesn't. Obviously they are going to file a bug asking why. A chunk of the problems are bad hardware/firmware. But the problem is which one.

NMIs are usually hardware.

BTW one big issue here is that we don't display anything on the screen so the system seems hung. KMS solves this, but unfortunately not for the video chipsets often used in servers.

Part of it is solved by serializing the error
and defaulting to reboot after panic (currently NMI doesn't do that, MCE already does, NMI should too imho)

Quoted 4 lines

Replacing a slot card is easy, replacing a motherboard is not. So I usually try to determine which device is failing by walking the pci bus and looking for the serr bits or some of the pci-e status bits.

You don't necessarily need to replace anything, it could be just unlucky data corruption (e.g. a big enough cosmic ray that flipped enough bits that the normal error correction couldn't fix it anymore)

Quoted 4 lines

It is inefficient, but I haven't had time to figure out a way to clean it up. And just for the record, I usually see an unknown NMI report every other week.

At least ignoring the data corruption is not the way to handle it. I don't think you'll do your customers a favor this way.

Quoted 8 lines
Quoted 4 lines

Anyways for hardware support we could probably dump some more information at panic or better through error serialization, but the word "debug" is IMHO an very wrong way to think about that.

Well, I can use 'diagnos' or 'determine' if you want. But at the end of the day, we have customers that see scary software messages and expect us to give them reasonable direction to fix their problems.

Usually these problems shouldn't be handled by kernel hackers, it's something for a hardware technician. If kernel hackers have to handle it something is very wrong.

IMHO the software should give the customer enough information to fix (or rather let their hardware technician) work it out.

If it's not good enough for this we have to improve it. But ignoring the errors is not the way to do that.

BTW one issue is that the screen is not big enough for all the information that is really useful for this. So I suspect to have it really useful you need to accept that some information will only be available through serialization (e.g. if you wanted to dump parts of the PCI config space)

-Andi

Signature · 1 line
[email protected] -- Speaking for myself only.
#35 lore raw

Re: [RFC 5/6] x86, NMI, Add support to notify hardware error with unknown NMI

Quoted 19 lines · Andi Kleen
Quoted 10 lines · On Mon, 13 Sep 2010 11:47:50 -0400 Don Zickus <[email protected]> wrote:
Quoted 9 lines

Then there isn't necessarily something to "debug": data corruption can happen without any bugs being around (and in fact that's the common case, assuming production systems)

So I'm not sure what you're debugging here. Are you being the support technician for the system through bugzilla? That sounds inefficient.

Quoted 5 lines

The problem I repeatedly deal with for RHEL systems is a customer sees an unknown NMI printed on their screen and sometimes the machine falls apart shortly after, sometimes it doesn't. Obviously they are going to file a bug asking why. A chunk of the problems are bad hardware/firmware. But the problem is which one.

Quoted 6 lines

NMIs are usually hardware.

BTW one big issue here is that we don't display anything on the screen so the system seems hung. KMS solves this, but unfortunately not for the video chipsets often used in servers.

No most of our customer see messages being sent to the console or serial part. I haven't seen KMS hiding the info yet.

Quoted 6 lines

Part of it is solved by serializing the error
and defaulting to reboot after panic (currently NMI doesn't do that, MCE already does, NMI should too imho)

Quoted 1 line
Quoted 8 lines
Quoted 3 lines

Replacing a slot card is easy, replacing a motherboard is not. So I usually try to determine which device is failing by walking the pci bus and looking for the serr bits or some of the pci-e status bits.

You don't necessarily need to replace anything, it could be just unlucky data corruption (e.g. a big enough cosmic ray that flipped enough bits that the normal error correction couldn't fix it anymore)

No, these are easily reproducible NMIs. So far it they have been the result of bad firmware (either features that are marked supported but not, or register settings that changed between updates), nic cards that had issues, or bad motherboards.

None of these issues went away because of a reboot.

Quoted 8 lines
Quoted 5 lines

It is inefficient, but I haven't had time to figure out a way to clean it up. And just for the record, I usually see an unknown NMI report every other week.

At least ignoring the data corruption is not the way to handle it. I don't think you'll do your customers a favor this way.

I never said I ignore them. We try to resolve them quickly.

Quoted 10 lines
Quoted 9 lines
Quoted 5 lines

Anyways for hardware support we could probably dump some more information at panic or better through error serialization, but the word "debug" is IMHO an very wrong way to think about that.

Well, I can use 'diagnos' or 'determine' if you want. But at the end of the day, we have customers that see scary software messages and expect us to give them reasonable direction to fix their problems.

Quoted 6 lines

Usually these problems shouldn't be handled by kernel hackers, it's something for a hardware technician. If kernel hackers have to handle it something is very wrong.

IMHO the software should give the customer enough information to fix (or rather let their hardware technician) work it out.

Yes, I agree, but the hardware folks usually like it when we give them a better clue than 'hardware is broken'. Something like the network stopped working or your storage controller's firmware went bad, is usually more helpful.

And the thing is, the hardware usually leaves a bread cumb trail of where things went wrong. It is just a matter of poking different chips to find out who generated the error and report that.

Quoted 6 lines

BTW one issue is that the screen is not big enough for all the information that is really useful for this. So I suspect to have it really useful you need to accept that some information will only be available through serialization (e.g. if you wanted to dump parts of the PCI config space)

Honestly, I don't think you need much screen real estate. It would be nice when an unknown NMI comes in, if the kernel just pokes around the hardware registers and display a summary of what it found. For example,

The following devices had error bits set in the status registers:
PCI device x:y.z - STATUS_BIT1 | STATUS_BIT2
HW device xyz - STATUS_BIT3
...

This at least gives the users some hardware they can remove/replace to see if the problem goes away.

Right now I feel like it is one giant guessing game.

But I guess if we accept the fact that an unknown NMI will panic the box, then we can probably be a little more liberal in breaking spinlocks and poking around the hardware to display some userful info.

Just some thoughts.

Cheers,
Don

#36 lore raw

Re: [RFC 5/6] x86, NMI, Add support to notify hardware error with unknown NMI

Quoted 11 lines

Honestly, I don't think you need much screen real estate. It would be nice when an unknown NMI comes in, if the kernel just pokes around the hardware registers and display a summary of what it found. For example,

The following devices had error bits set in the status registers:
PCI device x:y.z - STATUS_BIT1 | STATUS_BIT2
HW device xyz - STATUS_BIT3
...

You mean data from the generic PCI config space?

I don't think i would feel comfortable with arbitrary driver callbacks (the risk of the driver breaking the panic would be high)

But if it's generic if not on the screen it should
be at least in the error serialization data and logged after boot.

At least on PCI-E it may be enough to simply dump all recent AER data.

Quoted 4 lines

But I guess if we accept the fact that an unknown NMI will panic the box, then we can probably be a little more liberal in breaking spinlocks and poking around the hardware to display some userful info.

You have to be a bit careful with that, you may caused nested errors (e.g. machine checks or more NMIs). I suppose this could be checked for though.

-Andi

Signature · 1 line
[email protected] -- Speaking for myself only.
#37 lore raw

Re: [RFC 5/6] x86, NMI, Add support to notify hardware error with unknown NMI

Quoted 7 lines · Andi Kleen
Quoted 7 lines

Honestly, I don't think you need much screen real estate. It would be nice when an unknown NMI comes in, if the kernel just pokes around the hardware registers and display a summary of what it found. For example,

Quoted 6 lines
Quoted 4 lines

The following devices had error bits set in the status registers:
PCI device x:y.z - STATUS_BIT1 | STATUS_BIT2
HW device xyz - STATUS_BIT3
...

You mean data from the generic PCI config space?

Yes. I normally just look at the Status register. With PCI-e I'll look at the other status registers in the capabilities field too.

Quoted 3 lines

I don't think i would feel comfortable with arbitrary driver callbacks (the risk of the driver breaking the panic would be high)

Neither would I.

Quoted 3 lines

But if it's generic if not on the screen it should
be at least in the error serialization data and logged after boot.

I guess I don't know what that is, 'error serialization data'. Is there somewhere I can read more about it?

Quoted 3 lines

At least on PCI-E it may be enough to simply dump all recent AER data.

This assumes AER is supported on the bridge? Which for newer chips is probably true, but I wasn't sure about older ones.

How would I dump AER data from within the kernel?

Quoted 9 lines
Quoted 5 lines

But I guess if we accept the fact that an unknown NMI will panic the box, then we can probably be a little more liberal in breaking spinlocks and poking around the hardware to display some userful info.

You have to be a bit careful with that, you may caused nested errors (e.g. machine checks or more NMIs). I suppose this could be checked for though.

Of course.

Cheers,
Don

#38 lore raw

Re: [RFC 5/6] x86, NMI, Add support to notify hardware error with unknown NMI

Quoted 7 lines
Quoted 4 lines

But if it's generic if not on the screen it should
be at least in the error serialization data and logged after boot.

I guess I don't know what that is, 'error serialization data'. Is there somewhere I can read more about it?

That's already supported in MCE -- saving the error record to NVRAM and logging it after reboot. NMI should probably do the same. It's much nicer than getting it from a console.

Quoted 6 lines
Quoted 3 lines

At least on PCI-E it may be enough to simply dump all recent AER data.

This assumes AER is supported on the bridge? Which for newer chips is probably true, but I wasn't sure about older ones.

Today's servers should usually have AER at least.

For old systems you only can get the few bits in PCI space.

Quoted 1 line

How would I dump AER data from within the kernel?

Would need a buffer that is dumped for past events and reading the registers for not yet reported. Right now some infrastructure is needed.

-Andi

Signature · 1 line
[email protected] -- Speaking for myself only.
#39 lore raw

Re: [RFC 5/6] x86, NMI, Add support to notify hardware error with unknown NMI

Quoted 11 lines · Andi Kleen
Quoted 7 lines
Quoted 4 lines

But if it's generic if not on the screen it should
be at least in the error serialization data and logged after boot.

I guess I don't know what that is, 'error serialization data'. Is there somewhere I can read more about it?

That's already supported in MCE -- saving the error record to NVRAM and logging it after reboot. NMI should probably do the same. It's much nicer than getting it from a console.

Hmm, that assumes these boxes have NVRAM. I am not sure if many of the boxes I see with problems have NVRAM on them.

Quoted 10 lines
Quoted 7 lines
Quoted 4 lines

At least on PCI-E it may be enough to simply dump all recent AER data.

This assumes AER is supported on the bridge? Which for newer chips is probably true, but I wasn't sure about older ones.

Today's servers should usually have AER at least.

Quoted 7 lines

For old systems you only can get the few bits in PCI space.

Quoted 1 line

How would I dump AER data from within the kernel?

Would need a buffer that is dumped for past events and reading the registers for not yet reported. Right now some infrastructure is needed.

Oh ok.

Cheers,
Don

#40 lore raw

Re: [RFC 5/6] x86, NMI, Add support to notify hardware error with unknown NMI

Quoted 10 lines · On Mon, 13 Sep 2010 15:36:55 -0400 Don Zickus <[email protected]> wrote:
Quoted 9 lines · Andi Kleen
Quoted 8 lines
Quoted 5 lines

But if it's generic if not on the screen it should be at least in the error serialization data and logged after boot.

I guess I don't know what that is, 'error serialization data'. Is there somewhere I can read more about it?

Quoted 6 lines
Quoted 3 lines

That's already supported in MCE -- saving the error record to NVRAM and logging it after reboot. NMI should probably do the same. It's much nicer than getting it from a console.

Hmm, that assumes these boxes have NVRAM. I am not sure if many of the boxes I see with problems have NVRAM on them.

Practically every PC has a small amount of NVRAM.

-Andi

#41 lore raw

Re: [RFC 5/6] x86, NMI, Add support to notify hardware error with unknown NMI

Quoted 8 lines · Andi Kleen
Quoted 6 lines
Quoted 3 lines

That's already supported in MCE -- saving the error record to NVRAM and logging it after reboot. NMI should probably do the same. It's much nicer than getting it from a console.

Hmm, that assumes these boxes have NVRAM. I am not sure if many of the boxes I see with problems have NVRAM on them.

Practically every PC has a small amount of NVRAM.

The big question is how much NVRAM the PC has that is safe for our NMI code to hijack/borrow across the reboot without scrogging something that the BIOS has squirreled away in there. I recall one patch that saved progress indicators during early boot or something - but at the expense of stomping on the saved clock settings or whatever so you rebooted and then you knew where your previos boot wedged, but your system thought it was 1970 again.

#42 lore raw

Re: [RFC 5/6] x86, NMI, Add support to notify hardware error with unknown NMI

Quoted 9 lines · On Mon, 13 Sep 2010 17:25:21 -0400 [email protected] wrote:
Quoted 1 line

Practically every PC has a small amount of NVRAM.

The big question is how much NVRAM the PC has that is safe for our NMI code to hijack/borrow across the reboot without scrogging something that the BIOS has squirreled away in there. I recall one patch that saved progress indicators during early boot or something - but at the expense of stomping on the saved clock settings or whatever so you rebooted and then you knew where your previos boot wedged, but your system thought it was 1970 again.

It's already implemented for MCE and it works on servers.

-Andi

Signature · 1 line
[email protected] -- Speaking for myself only.
#43 lore raw

Re: [RFC 5/6] x86, NMI, Add support to notify hardware error with unknown NMI

Quoted 1 line · Andi Kleen

It's already implemented for MCE and it works on servers.

I'm OK with snarfing up some NVRAM, as long as we have some way to check "yes, there are NN bytes of NVRAM architected for our use".

Did I miss the memo where it says "This code will only run on chipsets that have architected nvram space for this sort of use"? Wouldn't be the first time I've misunderstood what code is really doing. ;)

#44 lore raw

Re: [RFC 5/6] x86, NMI, Add support to notify hardware error with unknown NMI

In reply to Don Zickus
Quoted 10 lines · Don Zickus
Quoted 10 lines
Quoted 5 lines
Quoted 2 lines

At least on PCI-E it may be enough to simply dump all recent AER data.

This assumes AER is supported on the bridge? Which for newer chips is probably true, but I wasn't sure about older ones.

Today's servers should usually have AER at least.

For old systems you only can get the few bits in PCI space.

Quoted 7 lines
Quoted 5 lines
Quoted 1 line

How would I dump AER data from within the kernel?

Would need a buffer that is dumped for past events and reading the registers for not yet reported. Right now some infrastructure is needed.

Oh ok.

The proper approach would be not to add hacks to the NMI code but to implement southbridge drivers - which would also have NMI callbacks. These are unchartered waters, but variance in that space is reducing systematically so it would be worth a shot.

Thanks,

Ingo

#45 lore raw

Re: [RFC 5/6] x86, NMI, Add support to notify hardware error with unknown NMI

Quoted 19 lines · Ingo Molnar
Quoted 16 lines · Don Zickus
Quoted 15 lines
Quoted 5 lines
Quoted 2 lines

At least on PCI-E it may be enough to simply dump all recent AER data.

This assumes AER is supported on the bridge? Which for newer chips is probably true, but I wasn't sure about older ones.

Today's servers should usually have AER at least.

For old systems you only can get the few bits in PCI space.

Quoted 1 line

How would I dump AER data from within the kernel?

Would need a buffer that is dumped for past events and reading the registers for not yet reported. Right now some infrastructure is needed.

Quoted 6 lines
Quoted 1 line

Oh ok.

The proper approach would be not to add hacks to the NMI code but to implement southbridge drivers - which would also have NMI callbacks. These are unchartered waters, but variance in that space is reducing systematically so it would be worth a shot.

Interesting. I think the only southbridge I see regularly are Intel, AMD and Nvidia (with Nvidia being more problematic than others). Unfortunately, getting specs for Nvidia is very difficult.

But that might help narrow down where the NMI problem is.

Cheers,
Don

Quoted 4 lines

Thanks,

Ingo

#46 lore raw

Re: [RFC 5/6] x86, NMI, Add support to notify hardware error with unknown NMI

In reply to Ingo Molnar
Quoted 1 line · Ingo Molnar

...

Quoted 8 lines

The proper approach would be not to add hacks to the NMI code but to implement southbridge drivers - which would also have NMI callbacks. These are unchartered waters, but variance in that space is reducing systematically so it would be worth a shot.

Thanks,

Ingo

Hi Ingo,

while there is a conversation about makeing NMI handler robust/modern or whatever, I think the naming Huang has implemented for NMI Stat&Ctrl registers/ports look quite good and convenient (I thought about this times ago when being merging nmi-32/64 code but didn't implemented it properly). So I presume perhaps we could merge this snippets first? Or I miss something on discussion in general?

-- Cyrill

#47 lore raw

Re: [RFC 5/6] x86, NMI, Add support to notify hardware error with unknown NMI

Quoted 12 lines · Cyrill Gorcunov
Quoted 1 line · Ingo Molnar

...

Quoted 8 lines

The proper approach would be not to add hacks to the NMI code but to implement southbridge drivers - which would also have NMI callbacks. These are unchartered waters, but variance in that space is reducing systematically so it would be worth a shot.

Thanks,

Ingo

Quoted 8 lines

Hi Ingo,

while there is a conversation about makeing NMI handler robust/modern or whatever, I think the naming Huang has implemented for NMI Stat&Ctrl registers/ports look quite good and convenient (I thought about this times ago when being merging nmi-32/64 code but didn't implemented it properly). So I presume perhaps we could merge this snippets first? Or I miss something on discussion in general?

Yeah, i'm waiting for Don to pick out the good ones and send them to me after a bit of testing (he's been driving this topic lately). We can obviously make some progress.

Thanks,

Ingo

author
#48 lore raw

RFC 6/6 x86, NMI, Remove do_nmi_callback logic

In reply to Huang Ying

do_nmi_callback related logic is removed, because it is useless, just adds code complexity.

unknown_nmi_panic sysctl is reserved to keep kernel ABI unchanged.

---
 arch/x86/include/asm/nmi.h    |   10 +++++++++-
 arch/x86/kernel/apic/hw_nmi.c |    1 -
 arch/x86/kernel/apic/nmi.c    |   29 +----------------------------
 arch/x86/kernel/traps.c       |   16 ++++++++++------
 4 files changed, 20 insertions(+), 36 deletions(-)
arch/x86/include/asm/nmi.h +9 -1
@@ -30,9 +30,17 @@ extern void setup_apic_nmi_watchdog(void
 extern void stop_apic_nmi_watchdog(void *);
 extern void disable_timer_nmi_watchdog(void);
 extern void enable_timer_nmi_watchdog(void);
-extern int nmi_watchdog_tick(struct pt_regs *regs, unsigned reason);
 extern void cpu_nmi_set_wd_enabled(void);
 
+#if defined(CONFIG_X86_LOCAL_APIC) && !defined(CONFIG_LOCKUP_DETECTOR)
+extern int nmi_watchdog_tick(struct pt_regs *regs);
+#else
+static inline int nmi_watchdog_tick(struct pt_regs *regs)
+{
+	return 0;
+}
+#endif
+
 extern atomic_t nmi_active;
 extern unsigned int nmi_watchdog;
 #define NMI_NONE	0
arch/x86/kernel/apic/hw_nmi.c +0 -1
@@ -100,7 +100,6 @@ void acpi_nmi_disable(void) { return; }
 #endif
 atomic_t nmi_active = ATOMIC_INIT(0);           /* oprofile uses this */
 EXPORT_SYMBOL(nmi_active);
-int unknown_nmi_panic;
 void cpu_nmi_set_wd_enabled(void) { return; }
 void stop_apic_nmi_watchdog(void *unused) { return; }
 void setup_apic_nmi_watchdog(void *unused) { return; }
arch/x86/kernel/apic/nmi.c +1 -28
@@ -37,7 +37,6 @@
 
 #include <asm/mach_traps.h>
 
-int unknown_nmi_panic;
 int nmi_watchdog_enabled;
 
 /* For reliability, we're prepared to waste bits here. */
@@ -389,7 +388,7 @@ void touch_nmi_watchdog(void)
 EXPORT_SYMBOL(touch_nmi_watchdog);
 
 notrace __kprobes int
-nmi_watchdog_tick(struct pt_regs *regs, unsigned reason)
+nmi_watchdog_tick(struct pt_regs *regs)
 {
 	/*
 	 * Since current_thread_info()-> is always on the stack, and we
@@ -483,23 +482,6 @@ static void disable_ioapic_nmi_watchdog(
 	on_each_cpu(stop_apic_nmi_watchdog, NULL, 1);
 }
 
-static int __init setup_unknown_nmi_panic(char *str)
-{
-	unknown_nmi_panic = 1;
-	return 1;
-}
-__setup("unknown_nmi_panic", setup_unknown_nmi_panic);
-
-static int unknown_nmi_panic_callback(struct pt_regs *regs, int cpu)
-{
-	unsigned char reason = get_nmi_reason();
-	char buf[64];
-
-	sprintf(buf, "NMI received for unknown reason %02x\n", reason);
-	die_nmi(buf, regs, 1); /* Always panic here */
-	return 0;
-}
-
 /*
  * proc handler for /proc/sys/kernel/nmi
  */
@@ -540,15 +522,6 @@ int proc_nmi_enabled(struct ctl_table *t
 
 #endif /* CONFIG_SYSCTL */
 
-int do_nmi_callback(struct pt_regs *regs, int cpu)
-{
-#ifdef CONFIG_SYSCTL
-	if (unknown_nmi_panic)
-		return unknown_nmi_panic_callback(regs, cpu);
-#endif
-	return 0;
-}
-
 void arch_trigger_all_cpu_backtrace(void)
 {
 	int i;
arch/x86/kernel/traps.c +10 -6
@@ -84,6 +84,7 @@ EXPORT_SYMBOL_GPL(used_vectors);
 static int ignore_nmis;
 
 int unknown_nmi_for_hwerr;
+int unknown_nmi_panic;
 
 /*
  * Prevent NMI reason port (0x61) being accessed simultaneously, can
@@ -308,6 +309,13 @@ gp_in_kernel:
 	die("general protection fault", regs, error_code);
 }
 
+static int __init setup_unknown_nmi_panic(char *str)
+{
+	unknown_nmi_panic = 1;
+	return 1;
+}
+__setup("unknown_nmi_panic", setup_unknown_nmi_panic);
+
 static notrace __kprobes void
 pci_serr_error(unsigned char reason, struct pt_regs *regs)
 {
@@ -374,7 +382,7 @@ unknown_nmi_error(unsigned char reason,
 			reason, smp_processor_id());
 
 	printk(KERN_EMERG "Do you have a strange power saving mode enabled?\n");
-	if (panic_on_unrecovered_nmi)
+	if (unknown_nmi_panic || panic_on_unrecovered_nmi)
 		panic("NMI: Not continuing");
 
 	printk(KERN_EMERG "Dazed and confused, but trying to continue\n");
@@ -421,12 +429,8 @@ static notrace __kprobes void default_do
 	}
 	raw_spin_unlock(&nmi_reason_lock);
 
-#if defined(CONFIG_X86_LOCAL_APIC) && !defined(CONFIG_LOCKUP_DETECTOR)
-	if (nmi_watchdog_tick(regs, reason))
-		return;
-	if (do_nmi_callback(regs, smp_processor_id()))
+	if (nmi_watchdog_tick(regs))
 		return;
-#endif
 
 	if (notify_die(DIE_NMIUNKNOWN, "nmi_unknown", regs, reason, 2, SIGINT)
 	    == NOTIFY_STOP)
#49 lore raw

Re: [RFC 6/6] x86, NMI, Remove do_nmi_callback logic

Quoted 13 lines · Huang Ying
arch/x86/include/asm/nmi.h +9 -0
+#if defined(CONFIG_X86_LOCAL_APIC) && !defined(CONFIG_LOCKUP_DETECTOR)
+extern int nmi_watchdog_tick(struct pt_regs *regs);
+#else
+static inline int nmi_watchdog_tick(struct pt_regs *regs)
+{
+	return 0;
+}
+#endif
+
 extern atomic_t nmi_active;
 extern unsigned int nmi_watchdog;
 #define NMI_NONE	0

<snip>

Quoted 4 lines
arch/x86/kernel/traps.c +0 -0
@@ -421,12 +429,8 @@ static notrace __kprobes void default_do
 	}
 	raw_spin_unlock(&nmi_reason_lock);
 

arch/x86/kernel/traps.c static notrace __kprobes void default_do · line 436

Quoted 10 lines
arch/x86/kernel/traps.c +1 -5
-#if defined(CONFIG_X86_LOCAL_APIC) && !defined(CONFIG_LOCKUP_DETECTOR)
-	if (nmi_watchdog_tick(regs, reason))
-		return;
-	if (do_nmi_callback(regs, smp_processor_id()))
+	if (nmi_watchdog_tick(regs))
 		return;
-#endif
 
 	if (notify_die(DIE_NMIUNKNOWN, "nmi_unknown", regs, reason, 2, SIGINT)
 	    == NOTIFY_STOP)

I wonder if these two chunks are going to confuse people when they read the code. The old nmi watchdog exists in the arch/x86 area but the new nmi watchdog code is now in kernel/watchdog.c.

If someone sees nmi_watchdog_tick() here will they assume the nmi watchdog code is still inside arch/x86?

I would suggest keep it wrapped with CONFIG_LOCKUP_DETECTOR to make it obvious. Thoughts?

Cheers,
Don

author
#50 lore raw

Re: [RFC 6/6] x86, NMI, Remove do_nmi_callback logic

Quoted 36 lines · Don Zickus
Quoted 13 lines · Huang Ying
arch/x86/include/asm/nmi.h +9 -0
+#if defined(CONFIG_X86_LOCAL_APIC) && !defined(CONFIG_LOCKUP_DETECTOR)
+extern int nmi_watchdog_tick(struct pt_regs *regs);
+#else
+static inline int nmi_watchdog_tick(struct pt_regs *regs)
+{
+	return 0;
+}
+#endif
+
 extern atomic_t nmi_active;
 extern unsigned int nmi_watchdog;
 #define NMI_NONE	0

<snip>

Quoted 4 lines
arch/x86/kernel/traps.c +0 -0
@@ -421,12 +429,8 @@ static notrace __kprobes void default_do
 	}
 	raw_spin_unlock(&nmi_reason_lock);
 

arch/x86/kernel/traps.c static notrace __kprobes void default_do · line 436

Quoted 10 lines
arch/x86/kernel/traps.c +1 -5
-#if defined(CONFIG_X86_LOCAL_APIC) && !defined(CONFIG_LOCKUP_DETECTOR)
-	if (nmi_watchdog_tick(regs, reason))
-		return;
-	if (do_nmi_callback(regs, smp_processor_id()))
+	if (nmi_watchdog_tick(regs))
 		return;
-#endif
 
 	if (notify_die(DIE_NMIUNKNOWN, "nmi_unknown", regs, reason, 2, SIGINT)
 	    == NOTIFY_STOP)

I wonder if these two chunks are going to confuse people when they read the code. The old nmi watchdog exists in the arch/x86 area but the new nmi watchdog code is now in kernel/watchdog.c.

Quoted 5 lines

If someone sees nmi_watchdog_tick() here will they assume the nmi watchdog code is still inside arch/x86?

I would suggest keep it wrapped with CONFIG_LOCKUP_DETECTOR to make it obvious. Thoughts?

Is it planned to remove old NMI watchdog implementation in near future?

If it is not, I think it is better to document/comment the two implementation more explicitly, maybe in nmi_watchdog_tick() comments. I suspect nmi_watchdog_tick() wrapped with "! defined(CONFIG_LOCKUP_DETECTOR)" will make people more confused.

Best Regards,
Huang Ying

#51 lore raw

Re: [RFC 6/6] x86, NMI, Remove do_nmi_callback logic

Quoted 37 lines · Huang Ying
Quoted 36 lines · Don Zickus
Quoted 13 lines · Huang Ying
arch/x86/include/asm/nmi.h +9 -0
+#if defined(CONFIG_X86_LOCAL_APIC) && !defined(CONFIG_LOCKUP_DETECTOR)
+extern int nmi_watchdog_tick(struct pt_regs *regs);
+#else
+static inline int nmi_watchdog_tick(struct pt_regs *regs)
+{
+	return 0;
+}
+#endif
+
 extern atomic_t nmi_active;
 extern unsigned int nmi_watchdog;
 #define NMI_NONE	0

<snip>

Quoted 4 lines
arch/x86/kernel/traps.c +0 -0
@@ -421,12 +429,8 @@ static notrace __kprobes void default_do
 	}
 	raw_spin_unlock(&nmi_reason_lock);
 

arch/x86/kernel/traps.c static notrace __kprobes void default_do · line 436

Quoted 10 lines
arch/x86/kernel/traps.c +1 -5
-#if defined(CONFIG_X86_LOCAL_APIC) && !defined(CONFIG_LOCKUP_DETECTOR)
-	if (nmi_watchdog_tick(regs, reason))
-		return;
-	if (do_nmi_callback(regs, smp_processor_id()))
+	if (nmi_watchdog_tick(regs))
 		return;
-#endif
 
 	if (notify_die(DIE_NMIUNKNOWN, "nmi_unknown", regs, reason, 2, SIGINT)
 	    == NOTIFY_STOP)

I wonder if these two chunks are going to confuse people when they read the code. The old nmi watchdog exists in the arch/x86 area but the new nmi watchdog code is now in kernel/watchdog.c.

Quoted 8 lines
Quoted 5 lines

If someone sees nmi_watchdog_tick() here will they assume the nmi watchdog code is still inside arch/x86?

I would suggest keep it wrapped with CONFIG_LOCKUP_DETECTOR to make it obvious. Thoughts?

Is it planned to remove old NMI watchdog implementation in near future?

Yes. With the new Pentium4 PMU driver we've covered the last old-NMI-watchdog hardware support corner too, so we can and should remove the old code. (keeping the boot option to stay compatible)

Thanks,

Ingo

#53 lore raw
Quoted 5 lines · Peter Zijlstra
Quoted 1 line · Huang Ying

+ return inb(NMI_REASON_PORT);

I've always wondered, where is this magic stuff documented?

A lot of the pre-2000 stuff is documented in Ralf Brown's Interrupt List, and the associated book "The Undocumented PC". A lot was originally documented in various now-out-of-print IBM technical manuals.

RBIL unfortunately hasn't been updated since 2000. The good news is that post 2000 there are a lot more formal standards out there, but not collected in one convenient place like that.

-hpa

#54 lore raw
In reply to Peter Zijlstra
Quoted 4 lines · On Fri, 10 Sep 2010 22:37:07 +0200 Peter Zijlstra <[email protected]> wrote:
Quoted 1 line · Huang Ying

+ return inb(NMI_REASON_PORT);

I've always wondered, where is this magic stuff documented?

In the datasheet of the chipset. But sometimes you need to dig deeper to get a global picture including all chipsets.

-Andi

Signature · 1 line
[email protected] -- Speaking for myself only.
#55 lore raw
Quoted 10 lines · Andi Kleen
Quoted 4 lines · On Fri, 10 Sep 2010 22:37:07 +0200 Peter Zijlstra <[email protected]> wrote:
Quoted 1 line · Huang Ying

+ return inb(NMI_REASON_PORT);

I've always wondered, where is this magic stuff documented?

In the datasheet of the chipset. But sometimes you need to dig deeper to get a global picture including all chipsets.

It is also here:

http://support.amd.com/us/Embedded_TechDocs/43009_sb7xx_rrg_pub_1.00.pdf
2.3.3.1.1 IO-Mapped Control Registers

-Robert

Signature · 2 lines
Advanced Micro Devices, Inc.
Operating System Research Center
#56 lore raw
In reply to Huang Ying
Quoted 1 line · Huang Ying

Replace the NMI related magic numbers with symbol constants.

Hi Huang,

Sorry for disappearing for a week..

Ingo asked me to shepherd these patches. I finally got around to do some testing on them. I'll do some more tomorrow.

Anyway, I don't have a problem with patches 1-3 and 6 (I guess the rename and rename again doesn't really bother me and it kinda makes some logical sense).

I am ok with most of patch 4 but I was wondering if you could split out the part of using other cpus to access the reason register. To me it seem like the nmi handler rewrite and allowing !bsp cpus to access the reason registers were two different ideas. For bisecting reasons it would be easier to seperate them in case we have problems with lost NMIs later. It would be easier to determine if the lost NMIs were from the rewrite or the migration of the reason register to other cpus.

I still have a stupid hangup about the raw_spin_lock but if no one else has any issues, then I'll just shutup about it. :-)

As for patch 5, I am worried about breaking existing user systems. I went through the fedora buglist and noticed a couple dozen bugzillas complaining about unknown nmis. The people complaining still seemed to have functioning systems (at least they seemed to think so). Adding in the panic gets me worried that we might break a user's setup and cause them regressions.

Though I understand what Andi is saying an unknown NMI is bad and the system should panic, but on the other hand, unless we have a way of analyzing it and give a user an option to either fix it or override it, just panicing may not be the best way right now IMO.

I guess adding either another knob to override the hardware error option or tying it in with the panic_on_unknown_error option might make me more comfortable. That way enterprise customers can always just enable it by default and desktop users (for now) could have it off.

Thoughts?

Cheers,
Don

Quoted 89 lines
---
 arch/x86/include/asm/mach_traps.h |   12 +++++++++++-
 arch/x86/kernel/traps.c           |   18 +++++++++---------
 2 files changed, 20 insertions(+), 10 deletions(-)
arch/x86/include/asm/mach_traps.h +11 -1
@@ -7,9 +7,19 @@
 
 #include <asm/mc146818rtc.h>
 
+#define NMI_REASON_PORT		0x61
+
+#define NMI_REASON_MEMPAR	0x80
+#define NMI_REASON_IOCHK	0x40
+#define NMI_REASON_MASK		(NMI_REASON_MEMPAR | NMI_REASON_IOCHK)
+
+#define NMI_REASON_CLEAR_MEMPAR	0x04
+#define NMI_REASON_CLEAR_IOCHK	0x08
+#define NMI_REASON_CLEAR_MASK	0x0f
+
 static inline unsigned char get_nmi_reason(void)
 {
-	return inb(0x61);
+	return inb(NMI_REASON_PORT);
 }
 
 static inline void reassert_nmi(void)
arch/x86/kernel/traps.c +9 -10
@@ -323,8 +323,8 @@ mem_parity_error(unsigned char reason, s
 	printk(KERN_EMERG "Dazed and confused, but trying to continue\n");
 
 	/* Clear and disable the memory parity error line. */
-	reason = (reason & 0xf) | 4;
-	outb(reason, 0x61);
+	reason = (reason & NMI_REASON_CLEAR_MASK) | NMI_REASON_CLEAR_MEMPAR;
+	outb(reason, NMI_REASON_PORT);
 }
 
 static notrace __kprobes void
@@ -339,15 +339,15 @@ io_check_error(unsigned char reason, str
 		panic("NMI IOCK error: Not continuing");
 
 	/* Re-enable the IOCK line, wait for a few seconds */
-	reason = (reason & 0xf) | 8;
-	outb(reason, 0x61);
+	reason = (reason & NMI_REASON_CLEAR_MASK) | NMI_REASON_CLEAR_IOCHK;
+	outb(reason, NMI_REASON_PORT);
 
 	i = 2000;
 	while (--i)
 		udelay(1000);
 
-	reason &= ~8;
-	outb(reason, 0x61);
+	reason &= ~NMI_REASON_CLEAR_IOCHK;
+	outb(reason, NMI_REASON_PORT);
 }
 
 static notrace __kprobes void
@@ -388,7 +388,7 @@ static notrace __kprobes void default_do
 	if (!cpu)
 		reason = get_nmi_reason();
 
-	if (!(reason & 0xc0)) {
+	if (!(reason & NMI_REASON_MASK)) {
 		if (notify_die(DIE_NMI_IPI, "nmi_ipi", regs, reason, 2, SIGINT)
 								== NOTIFY_STOP)
 			return;
@@ -418,9 +418,9 @@ static notrace __kprobes void default_do
 		return;
 
 	/* AK: following checks seem to be broken on modern chipsets. FIXME */
-	if (reason & 0x80)
+	if (reason & NMI_REASON_MEMPAR)
 		mem_parity_error(reason, regs);
-	if (reason & 0x40)
+	if (reason & NMI_REASON_IOCHK)
 		io_check_error(reason, regs);
 #ifdef CONFIG_X86_32
 	/*
--

To unsubscribe from this list: send the line "unsubscribe linux-kernel" in the body of a message to [email protected]
More majordomo info at http://vger.kernel.org/majordomo-info.html
Please read the FAQ at http://www.tux.org/lkml/

#57 lore raw
Quoted 6 lines

I guess adding either another knob to override the hardware error option or tying it in with the panic_on_unknown_error option might make me more comfortable. That way enterprise customers can always just enable it by default and desktop users (for now) could have it off.

Anything that needs explicit enabling is a bad idea, that would lead to a lot of users running in "corrupt my data" mode.

The code currently uses the presence of a HEST error table to detect a server. HEST should be only available on servers.

On servers at least panic should be default.

-Andi

#58 lore raw
Quoted 9 lines · Andi Kleen
Quoted 6 lines

I guess adding either another knob to override the hardware error option or tying it in with the panic_on_unknown_error option might make me more comfortable. That way enterprise customers can always just enable it by default and desktop users (for now) could have it off.

Anything that needs explicit enabling is a bad idea, that would lead to a lot of users running in "corrupt my data" mode.

I know. But as I said earlier in my emails, I am trying to figure out how to deal with the fallout from unknown nmis turning into panics. Today people see unknown nmis. They may or may not be corrupting data. They usually file a bug. Currently it is hard for me to diagnosis the problem. Usually the old 'upgrade your bios/firmware' does the trick. Sometimes it doesn't and people feel like their machines still run fine. So they ignore it (for good or for bad).

Turning unknown nmis into panics would break their current setup without much gain. So I was trying to propose something temporarily until we could get a better infrastructure to isolate the source and provide better info on what to do.

I agree with you that long term unknown nmis should be panics. I just get nervous about doing that now from a support perspective.

Quoted 5 lines

The code currently uses the presence of a HEST error table to detect a server. HEST should be only available on servers.

On servers at least panic should be default.

Ok. That's fine. But then what. What does a developer do with that panic? There's no useful info. That is sorta my problem. Then again I do not know much about HEST.

Cheers,
Don

#59 lore raw

Hi, Don,

Quoted 19 lines · Don Zickus
Quoted 9 lines · Andi Kleen
Quoted 6 lines

I guess adding either another knob to override the hardware error option or tying it in with the panic_on_unknown_error option might make me more comfortable.  That way enterprise customers can always just enable it by default and desktop users (for now) could have it off.

Anything that needs explicit enabling is a bad idea, that would lead to a lot of users running in "corrupt my data" mode.

I know.  But as I said earlier in my emails, I am trying to figure out how to deal with the fallout from unknown nmis turning into panics.  Today people see unknown nmis.  They may or may not be corrupting data.  They usually file a bug.  Currently it is hard for me to diagnosis the problem. Usually the old 'upgrade your bios/firmware' does the trick.  Sometimes it doesn't and people feel like their machines still run fine.  So they ignore it (for good or for bad).

Quoted 7 lines

Turning unknown nmis into panics would break their current setup without much gain.  So I was trying to propose something temporarily until we could get a better infrastructure to isolate the source and provide better info on what to do.

I agree with you that long term unknown nmis should be panics.  I just get nervous about doing that now from a support perspective.

In fact, we use white list policy here. Only systems with HEST or identified by chipset host bridge PCI ID will panic for unknown NMI. So I think systems you worried about will not have this enabled.

Quoted 8 lines
Quoted 4 lines

The code currently uses the presence of a HEST error table to detect a server. HEST should be only available on servers.

On servers at least panic should be default.

Ok.  That's fine. But then what.  What does a developer do with that panic?  There's no useful info.  That is sorta my problem.  Then again I do not know much about HEST.

On some system, there is some hardware error log in BMC/BIOS. The hardware error log can be gotten via IPMI or BIOS menu. Otherwise, can we get some useful info for unknown NMI? If we can, can we collect the info, then print it on console and save it into flash via ERST (part of APEI too) before panic?

HEST is defined in ACPI spec 4.0 and later version in section named APEI (ACPI Platform Error Interface). It is used to describe the error sources of system. It should be available only on server platform.

Best Regards,
Huang Ying

#60 lore raw
Quoted 43 lines · huang ying

Hi, Don,

Quoted 19 lines · Don Zickus
Quoted 9 lines · Andi Kleen
Quoted 6 lines

I guess adding either another knob to override the hardware error option or tying it in with the panic_on_unknown_error option might make me more comfortable.  That way enterprise customers can always just enable it by default and desktop users (for now) could have it off.

Anything that needs explicit enabling is a bad idea, that would lead to a lot of users running in "corrupt my data" mode.

I know.  But as I said earlier in my emails, I am trying to figure out how to deal with the fallout from unknown nmis turning into panics.  Today people see unknown nmis.  They may or may not be corrupting data.  They usually file a bug.  Currently it is hard for me to diagnosis the problem. Usually the old 'upgrade your bios/firmware' does the trick.  Sometimes it doesn't and people feel like their machines still run fine.  So they ignore it (for good or for bad).

Quoted 7 lines

Turning unknown nmis into panics would break their current setup without much gain.  So I was trying to propose something temporarily until we could get a better infrastructure to isolate the source and provide better info on what to do.

I agree with you that long term unknown nmis should be panics.  I just get nervous about doing that now from a support perspective.

In fact, we use white list policy here. Only systems with HEST or identified by chipset host bridge PCI ID will panic for unknown NMI. So I think systems you worried about will not have this enabled.

Quoted 8 lines
Quoted 4 lines

The code currently uses the presence of a HEST error table to detect a server. HEST should be only available on servers.

On servers at least panic should be default.

Ok.  That's fine. But then what.  What does a developer do with that panic?  There's no useful info.  That is sorta my problem.  Then again I do not know much about HEST.

Quoted 5 lines

On some system, there is some hardware error log in BMC/BIOS. The hardware error log can be gotten via IPMI or BIOS menu. Otherwise, can we get some useful info for unknown NMI? If we can, can we collect the info, then print it on console and save it into flash via ERST (part of APEI too) before panic?

Ok. Does the BIOS/BMC automatically do this? Can we just print a message on panic saying checking your BIOS/BMC logs for more info?

I would love to add code to gather more useful info for unknown NMIs, but is it expected that HEST does some of this? I guess what I am trying to figure out, if we are going to put intelligence to detect a HEST enabled machine and panic when unknown NMI comes along (presumably from HEST??), then can we leverage HEST at all to understand why the NMI happened or point the user to the BIOS/BMC to get more info. In other words, what value do we get HEST other than we detect its there, lets panic.

Quoted 4 lines

HEST is defined in ACPI spec 4.0 and later version in section named APEI (ACPI Platform Error Interface). It is used to describe the error sources of system. It should be available only on server platform.

Ok. Does the kernel have intelligence to use it or the BIOS yet?

Cheers,
Don

#61 lore raw
Quoted 44 lines · Don Zickus
Quoted 43 lines · huang ying

Hi, Don,

Quoted 19 lines · Don Zickus
Quoted 9 lines · Andi Kleen
Quoted 6 lines

I guess adding either another knob to override the hardware error option or tying it in with the panic_on_unknown_error option might make me more comfortable.  That way enterprise customers can always just enable it by default and desktop users (for now) could have it off.

Anything that needs explicit enabling is a bad idea, that would lead to a lot of users running in "corrupt my data" mode.

I know.  But as I said earlier in my emails, I am trying to figure out how to deal with the fallout from unknown nmis turning into panics.  Today people see unknown nmis.  They may or may not be corrupting data.  They usually file a bug.  Currently it is hard for me to diagnosis the problem. Usually the old 'upgrade your bios/firmware' does the trick.  Sometimes it doesn't and people feel like their machines still run fine.  So they ignore it (for good or for bad).

Quoted 7 lines

Turning unknown nmis into panics would break their current setup without much gain.  So I was trying to propose something temporarily until we could get a better infrastructure to isolate the source and provide better info on what to do.

I agree with you that long term unknown nmis should be panics.  I just get nervous about doing that now from a support perspective.

In fact, we use white list policy here. Only systems with HEST or identified by chipset host bridge PCI ID will panic for unknown NMI. So I think systems you worried about will not have this enabled.

Quoted 8 lines
Quoted 4 lines

The code currently uses the presence of a HEST error table to detect a server. HEST should be only available on servers.

On servers at least panic should be default.

Ok.  That's fine. But then what.  What does a developer do with that panic?  There's no useful info.  That is sorta my problem.  Then again I do not know much about HEST.

Quoted 8 lines
Quoted 5 lines

On some system, there is some hardware error log in BMC/BIOS. The hardware error log can be gotten via IPMI or BIOS menu. Otherwise, can we get some useful info for unknown NMI? If we can, can we collect the info, then print it on console and save it into flash via ERST (part of APEI too) before panic?

Ok.  Does the BIOS/BMC automatically do this?  Can we just print a message on panic saying checking your BIOS/BMC logs for more info?

Yes. BIOS/BMC automatically do that. And I will add it to panic message.

Quoted 7 lines

I would love to add code to gather more useful info for unknown NMIs, but is it expected that HEST does some of this?  I guess what I am trying to figure out, if we are going to put intelligence to detect a HEST enabled machine and panic when unknown NMI comes along (presumably from HEST??), then can we leverage HEST at all to understand why the NMI happened or point the user to the BIOS/BMC to get more info.  In other words, what value do we get HEST other than we detect its there, lets panic.

Yes. HEST can be used to report some hardware error information. I am working on that now.

Quoted 5 lines
Quoted 3 lines

HEST is defined in ACPI spec 4.0 and later version in section named APEI (ACPI Platform Error Interface). It is used to describe the error sources of system. It should be available only on server platform.

Ok.  Does the kernel have intelligence to use it or the BIOS yet?

HEST works in kernel BIOS cooperative way. I am working on a HEST driver which will get notified for NMI and collect the error information reported by BIOS. But It is possible that some systems have only BMC/BIOS log and do not report that to OS except unknown NMI. The unknown NMI panic logic is for these systems.

Best Regards,
Huang Ying

#62 lore raw
Quoted 27 lines · huang ying
Quoted 8 lines · Don Zickus
Quoted 5 lines

On some system, there is some hardware error log in BMC/BIOS. The hardware error log can be gotten via IPMI or BIOS menu. Otherwise, can we get some useful info for unknown NMI? If we can, can we collect the info, then print it on console and save it into flash via ERST (part of APEI too) before panic?

Ok.  Does the BIOS/BMC automatically do this?  Can we just print a message on panic saying checking your BIOS/BMC logs for more info?

Yes. BIOS/BMC automatically do that. And I will add it to panic message.

Quoted 7 lines

I would love to add code to gather more useful info for unknown NMIs, but is it expected that HEST does some of this?  I guess what I am trying to figure out, if we are going to put intelligence to detect a HEST enabled machine and panic when unknown NMI comes along (presumably from HEST??), then can we leverage HEST at all to understand why the NMI happened or point the user to the BIOS/BMC to get more info.  In other words, what value do we get HEST other than we detect its there, lets panic.

Yes. HEST can be used to report some hardware error information. I am working on that now.

Quoted 4 lines
Quoted 3 lines

HEST is defined in ACPI spec 4.0 and later version in section named APEI (ACPI Platform Error Interface). It is used to describe the error sources of system. It should be available only on server platform.

Quoted 7 lines
Quoted 1 line

Ok.  Does the kernel have intelligence to use it or the BIOS yet?

HEST works in kernel BIOS cooperative way. I am working on a HEST driver which will get notified for NMI and collect the error information reported by BIOS. But It is possible that some systems have only BMC/BIOS log and do not report that to OS except unknown NMI. The unknown NMI panic logic is for these systems.

Ah ok, thanks for the info. I think adding the info to the panic message would be valuable. I have no more objections to your patch now. :-)

I appreciate your patience for clue-ing me in on how HEST works!

Cheers,
Don

#63 lore raw
In reply to Don Zickus

Hi, Don,

Quoted 9 lines · Don Zickus
Quoted 1 line · Huang Ying

Replace the NMI related magic numbers with symbol constants.

Hi Huang,

Sorry for disappearing for a week..

Ingo asked me to shepherd these patches.  I finally got around to do some testing on them.  I'll do some more tomorrow.

Thanks. I will post a new version in next week according to comments collected so far.

Quoted 11 lines

Anyway, I don't have a problem with patches 1-3 and 6 (I guess the rename and rename again doesn't really bother me and it kinda makes some logical sense).

I am ok with most of patch 4 but I was wondering if you could split out the part of using other cpus to access the reason register.  To me it seem like the nmi handler rewrite and allowing !bsp cpus to access the reason registers were two different ideas.  For bisecting reasons it would be easier to seperate them in case we have problems with lost NMIs later.  It would be easier to determine if the lost NMIs were from the rewrite or the migration of the reason register to other cpus.

Yes. It's reasonable, I will do it.

Best Regards,
Huang Ying