RFC: x86/mce: Deprecate mcelog and other funsies
Subsystems (2)
Borislav Petkov
maintainercover RFC: x86/mce: Deprecate mcelog and other funsies
From: Borislav Petkov <[email protected]>
So this set cannot have a bigger "RFC" tag:
It attempts to revive the correctable errors collector. In this version, the thing gets counters in debugfs:
(debugfs)/ras/cec/
|-- array
`-- pfn
which give some stats on the CEC like how many CEs it caught and the number of pages it soft-offlined due to excessive amount of CEs.
$ cat ras/cec/array
{ n: 2
000: [0000000000000012|001]
001: [0000003735928559|e01]
}
Stats:
CEs: 514
offlined pages: 0
Flags: 0x0Then, we stuck all the /dev/mcelog functionality behind a CONFIG_X86_MCELOG which people can disable if they don't need mcelog and are going to use the rasdaemon. This is probably going to happen sooner or later with the distros anyway.
And then, as a result of that move, mce-inject is being made independent from mcelog and gets the error injection debugfs nodes of mce_amd_inj which you can echo into and don't need special tools:
$ tree --charset=ascii /sys/kernel/debug/mce-inject/ /sys/kernel/debug/mce-inject/ |-- addr |-- bank |-- cpu |-- flags |-- misc |-- README `-- status
0 directories, 7 files
Comments are more than welcome, Tony and will be polishing on those but thought it would be a nice point in time to send them out to get other people's opinions.
Thanks.
Borislav Petkov (3):
RAS: Add a Corrected Errors Collector
x86/mce: Merge mce_amd_inj into mce-inject
x86/mce-inject: Use debugfs_remove_recursive()
Tony Luck (1):
x86/mce: Deprecate /dev/mcelog
arch/x86/Kconfig | 8 + arch/x86/kernel/cpu/mcheck/Makefile | 2 + arch/x86/kernel/cpu/mcheck/dev-mcelog.c | 397 +++++++++++++++++++++++ arch/x86/kernel/cpu/mcheck/mce-inject.c | 519 +++++++++++++++++++++++++++++- arch/x86/kernel/cpu/mcheck/mce-internal.h | 8 + arch/x86/kernel/cpu/mcheck/mce.c | 513 +++++------------------------ arch/x86/ras/Kconfig | 11 + arch/x86/ras/Makefile | 2 - arch/x86/ras/mce_amd_inj.c | 484 ---------------------------- drivers/ras/Makefile | 3 +- drivers/ras/cec.c | 425 ++++++++++++++++++++++++ drivers/ras/debugfs.c | 2 +- drivers/ras/debugfs.h | 8 + include/linux/ras.h | 12 +- 14 files changed, 1467 insertions(+), 927 deletions(-) create mode 100644 arch/x86/kernel/cpu/mcheck/dev-mcelog.c delete mode 100644 arch/x86/ras/mce_amd_inj.c create mode 100644 drivers/ras/cec.c create mode 100644 drivers/ras/debugfs.h
Signature · 1 line
2.7.3
Borislav Petkov
maintainerRFC PATCH 1/4 RAS: Add a Corrected Errors Collector
From: Borislav Petkov <[email protected]>
A simple data structure for collecting correctable errors along with accessors. More detailed description in the code itself.
--- arch/x86/kernel/cpu/mcheck/mce.c | 120 +++++------ arch/x86/ras/Kconfig | 11 + drivers/ras/Makefile | 3 +- drivers/ras/cec.c | 425 +++++++++++++++++++++++++++++++++++++++ drivers/ras/debugfs.c | 2 +- drivers/ras/debugfs.h | 8 + include/linux/ras.h | 12 +- 7 files changed, 521 insertions(+), 60 deletions(-) create mode 100644 drivers/ras/cec.c create mode 100644 drivers/ras/debugfs.h
arch/x86/kernel/cpu/mcheck/mce.c +63 -57
@@ -35,6 +35,7 @@#include <linux/poll.h> #include <linux/nmi.h> #include <linux/cpu.h> +#include <linux/ras.h> #include <linux/smp.h> #include <linux/fs.h> #include <linux/mm.h>
@@ -151,14 +152,71 @@ static struct mce_log mcelog = { .recordlen = sizeof(struct mce), }; -void mce_log (struct mce *mce) +static bool memory_error (struct mce *m) +{ +struct cpuinfo_x86 *c = &boot_cpu_data; + +if (c->x86_vendor ==X86_VENDOR_AMD ) { +/* ErrCodeExt[20:16] */ +u8 xec = (m->status >>16 ) &0x1f ; + +return (xec ==0x0 || xec ==0x8 ); + }else if (c->x86_vendor ==X86_VENDOR_INTEL ) { +/* +* Intel SDM Volume 3B - 15.9.2 Compound Error Codes +* +* Bit 7 of the MCACOD field of IA32_MCi_STATUS is used for +* indicating a memory error. Bit 8 is used for indicating a +* cache hierarchy error. The combination of bit 2 and bit 3 +* is used for indicating a `generic' cache hierarchy error +* But we can't just blindly check the above bits, because if +* bit 11 is set, then it is a bus/interconnect error - and +* either way the above bits just gives more detail on what +* bus/interconnect error happened. Note that bit 12 can be +* ignored, as it's the "filter" bit. +*/ +return (m->status &0xef80 ) ==BIT (7 ) || + (m->status &0xef00 ) ==BIT (8 ) || + (m->status &0xeffc ) ==0xc ; + } + +return false; +} + +/* +* Check if the address reported by the CPU is in a format we can parse. +* It would be possible to add code for most other cases, but all would +* be somewhat complicated (e.g. segment offset would require an instruction +* parser). So only support physical addresses up to page granuality for now. +*/ +static int mce_usable_address (struct mce *m) +{ +if (!(m->status &MCI_STATUS_MISCV ) || !(m->status &MCI_STATUS_ADDRV )) +return 0 ; + +/* Checks after this one are Intel-specific: */ +if (boot_cpu_data.x86_vendor !=X86_VENDOR_INTEL ) +return 1 ; + +if (MCI_MISC_ADDR_LSB (m->misc) >PAGE_SHIFT ) +return 0 ; +if (MCI_MISC_ADDR_MODE (m->misc) !=MCI_MISC_ADDR_PHYS ) +return 0 ; +return 1 ; +} + +void mce_log (struct mce *m) {unsigned next, entry; +if (!in_atomic () &&memory_error (m) &&mce_usable_address (m)) +if (!ce_add_elem (m->addr >>PAGE_SHIFT )) +return ; +/* Emit the trace record: */ -trace_mce_record (mce); +trace_mce_record (m); -if (!mce_gen_pool_add (mce)) +if (!mce_gen_pool_add (m))irq_work_queue (&mce_irq_work);wmb ();
@@ -188,7 +246,7 @@ void mce_log(struct mce *mce)if (cmpxchg (&mcelog.next, entry, next) == entry)break ; } -memcpy (mcelog.entry + entry, mce,sizeof (struct mce )); +memcpy (mcelog.entry + entry, m,sizeof (struct mce ));wmb (); mcelog.entry[entry].finished =1 ;wmb ();
@@ -518,28 +576,6 @@ static void mce_report_event(struct pt_regs *regs)irq_work_queue (&mce_irq_work); } -/* -* Check if the address reported by the CPU is in a format we can parse. -* It would be possible to add code for most other cases, but all would -* be somewhat complicated (e.g. segment offset would require an instruction -* parser). So only support physical addresses up to page granuality for now. -*/ -static int mce_usable_address (struct mce *m) -{ -if (!(m->status &MCI_STATUS_MISCV ) || !(m->status &MCI_STATUS_ADDRV )) -return 0 ; - -/* Checks after this one are Intel-specific: */ -if (boot_cpu_data.x86_vendor !=X86_VENDOR_INTEL ) -return 1 ; - -if (MCI_MISC_ADDR_LSB (m->misc) >PAGE_SHIFT ) -return 0 ; -if (MCI_MISC_ADDR_MODE (m->misc) !=MCI_MISC_ADDR_PHYS ) -return 0 ; -return 1 ; -} -static int srao_decode_notifier (struct notifier_block *nb,unsigned long val,void *data) {
@@ -582,37 +618,6 @@ static void mce_read_aux(struct mce *m, int i) } } -static bool memory_error (struct mce *m) -{ -struct cpuinfo_x86 *c = &boot_cpu_data; - -if (c->x86_vendor ==X86_VENDOR_AMD ) { -/* ErrCodeExt[20:16] */ -u8 xec = (m->status >>16 ) &0x1f ; - -return (xec ==0x0 || xec ==0x8 ); - }else if (c->x86_vendor ==X86_VENDOR_INTEL ) { -/* -* Intel SDM Volume 3B - 15.9.2 Compound Error Codes -* -* Bit 7 of the MCACOD field of IA32_MCi_STATUS is used for -* indicating a memory error. Bit 8 is used for indicating a -* cache hierarchy error. The combination of bit 2 and bit 3 -* is used for indicating a `generic' cache hierarchy error -* But we can't just blindly check the above bits, because if -* bit 11 is set, then it is a bus/interconnect error - and -* either way the above bits just gives more detail on what -* bus/interconnect error happened. Note that bit 12 can be -* ignored, as it's the "filter" bit. -*/ -return (m->status &0xef80 ) ==BIT (7 ) || - (m->status &0xef00 ) ==BIT (8 ) || - (m->status &0xeffc ) ==0xc ; - } - -return false; -} -DEFINE_PER_CPU (unsigned, mce_poll_count);/*
@@ -2679,6 +2684,7 @@ static int __init mcheck_debugfs_init(void) { return -EINVAL; }static int __init mcheck_late_init (void ) {mcheck_debugfs_init (); +ce_init ();/* * Flush out everything that has been logged during early boot, now that
arch/x86/ras/Kconfig +11 -0
@@ -7,3 +7,14 @@ config MCE_AMD_INJ aspects of the MCE handling code. WARNING: Do not even assume this interface is staying stable! + +config RAS_CEC + bool "Correctable Errors Collector" + depends on X86_MCE && MEMORY_FAILURE && DEBUG_FS + ---help--- + This is a small cache which collects correctable memory errors per 4K + page PFN and counts their repeated occurrence. Once the counter for a + PFN overflows, we try to soft-offline that page as we take it to mean + that it has reached a relatively high error count and would probably + be best if we don't use it anymore. +
drivers/ras/Makefile +2 -1
@@ -1 +1,2 @@ -obj-$(CONFIG_RAS) += ras.o debugfs.o +obj-$(CONFIG_RAS) += ras.o debugfs.o +obj-$(CONFIG_RAS_CEC) += cec.o
drivers/ras/cec.c new file +425 -0
425 lines not on this page · raw message
drivers/ras/debugfs.c +1 -1
@@ -1,6 +1,6 @@#include <linux/debugfs.h> -static struct dentry *ras_debugfs_dir; +struct dentry *ras_debugfs_dir;static atomic_t trace_count =ATOMIC_INIT (0 );
drivers/ras/debugfs.h new file +8 -0
@@ -0,0 +1,8 @@ +#ifndef __RAS_DEBUGFS_H__ +#define __RAS_DEBUGFS_H__ + +#include <linux/debugfs.h> + +extern struct dentry *ras_debugfs_dir; + +#endif /* __RAS_DEBUGFS_H__ */
include/linux/ras.h +11 -1
@@ -1,14 +1,24 @@#ifndef __RAS_H__#define __RAS_H__ +#include <asm/errno.h> +#ifdef CONFIG_DEBUG_FS int ras_userspace_consumers (void );void ras_debugfs_init (void );int ras_add_daemon_trace (void );#else static inline int ras_userspace_consumers (void ) {return 0 ; } -static inline void ras_debugfs_init (void ) {return ; } +static inline void ras_debugfs_init (void ) { }static inline int ras_add_daemon_trace (void ) {return 0 ; }#endif +#ifdef CONFIG_RAS_CEC +void __init ce_init (void ); +int ce_add_elem (u64 pfn); +#else +static inline void __init ce_init (void ) { } +static inline int ce_add_elem (u64 pfn) {return -ENODEV ; }#endif + +#endif /* __RAS_H__ */
Signature · 1 line
2.7.3
Luck, Tony
maintainerRe: [RFC PATCH 1/4] RAS: Add a Corrected Errors Collector
Quoted 5 lines · Borislav Petkov
arch/x86/kernel/cpu/mcheck/mce.c +2
-0+void mce_log (struct mce *m) {unsigned next, entry; +if (!in_atomic () &&memory_error (m) &&mce_usable_address (m))
arch/x86/kernel/cpu/mcheck/mce.c static struct mce_log mcelog = { · line 220
Quoted 10 lines
arch/x86/kernel/cpu/mcheck/mce.c +5
-2+if (!ce_add_elem (m->addr >>PAGE_SHIFT )) +return ; +/* Emit the trace record: */ -trace_mce_record (mce); +trace_mce_record (m); -if (!mce_gen_pool_add (mce)) +if (!mce_gen_pool_add (m))irq_work_queue (&mce_irq_work);
Is there a reason that we need to call the ce_add_elem() inline here instead of having it just register on the mce_notifier chain? This series just cleaned out all the /dev/mcelog special code from here, and you are adding something back before the ink is dry on that change.
I'm also strongly divided about whether this corrected error handler should be allowed to preempt anything else even seeing the error.
Argument for:
Lonely corrected errors are "No Big Deal"(TM). Just counting them and moving on is a good thing.
Arguments against:
1) We may miss out on a one-time opportunity to get extra information (from acpi_extlog.c).
2) I think this subverts our CMCI storm detection and mitigation code?
We could make the chain more caller friendly by adding a filter argument so users could say "just tell me about memory errors" (currently each of the EDAC drivers has inline code to do the same as "memory_error(m) && mce_usable_address(m)")
-Tony
Borislav Petkov
maintainerRe: [RFC PATCH 1/4] RAS: Add a Corrected Errors Collector
Quoted 9 lines · Tony
Is there a reason that we need to call the
ce_add_elem()inline here instead of having it just register on themce_notifierchain? This series just cleaned out all the/dev/mcelogspecial code from here, and you are adding something back before the ink is dry on that change.I'm also strongly divided about whether this corrected error handler should be allowed to preempt anything else even seeing the error.
Well, so this is the main reason for adding the CEC: not to disturb users with random CECC errors which might happen a couple of times due to alpha particles and then never again. I.e., address all those sporadic bursts of correctable errors which don't mean that the hw is going faulty.
If the CEC consumes the error and does the leaky bucket of "forgetting" about it after a while and after no more of that same PFN triggers errors, then we do that silently and do not scare users. Yeah, they think their hw is broken and whether they should start swapping things.
And then there's the aspect of soft-offlining PFNs when the error threshold has been reached. I don't think we have had any automatic recovery actions wrt errors so far without external agents.
Quoted 3 lines
Argument for:
Lonely corrected errors are "No Big Deal"(TM). Just counting them and moving on is a good thing.
Yap, exactly.
Quoted 4 lines
Arguments against:
1) We may miss out on a one-time opportunity to get extra information (fromacpi_extlog.c).
2) I think this subverts our CMCI storm detection and mitigation code?
...and we can address that by adding "ras=cec_doesnt_consume_errors" or somesuch so that the rest of the chain sees them too.
I think we can be pretty flexible about it. And again, my main angle is the "do not disturb users unnecessarily".
Quoted 4 lines
We could make the chain more caller friendly by adding a filter argument so users could say "just tell me about memory errors" (currently each of the EDAC drivers has inline code to do the same as "
memory_error(m)&&mce_usable_address(m)")
Sure, that too.
And it can work on any system without the need for an EDAC driver.
Thanks.
Signature · 4 lines
Regards/Gruss,
Boris.
ECO tip #101: Trim your mails when you reply.Borislav Petkov
maintainerRFC PATCH 2/4 x86/mce: Deprecate /dev/mcelog
In reply to Borislav PetkovFrom: Tony Luck <[email protected]>
Move all code relating to /dev/mcelog to own source file. /dev/mcelog driver can now operate from the machine check notifier with lowest prio.
Boris:
* Move the mce_helper and trigger functionality behind CONFIG_X86_MCELOG.
--- arch/x86/Kconfig | 10 +- arch/x86/kernel/cpu/mcheck/Makefile | 2 + arch/x86/kernel/cpu/mcheck/dev-mcelog.c | 397 +++++++++++++++++++++++++++++ arch/x86/kernel/cpu/mcheck/mce-internal.h | 8 + arch/x86/kernel/cpu/mcheck/mce.c | 405 +++--------------------------- 5 files changed, 445 insertions(+), 377 deletions(-) create mode 100644 arch/x86/kernel/cpu/mcheck/dev-mcelog.c
arch/x86/Kconfig +9 -1
@@ -1004,6 +1004,14 @@ config X86_MCE The action the kernel takes depends on the severity of the problem, ranging from warning messages to halting the machine. +config X86_MCELOG + bool "Support for deprecated /dev/mcelog character device" + depends on X86_MCE + ---help--- + Enable support for /dev/mcelog which is needed by the old mcelog + userspace logging daemon. Consider switching to the new generation + rasdaemon solution. + config X86_MCE_INTEL def_bool y prompt "Intel MCE features"
@@ -1033,7 +1041,7 @@ config X86_MCE_THRESHOLD def_bool y config X86_MCE_INJECT - depends on X86_MCE + depends on X86_MCE && X86_MCELOG tristate "Machine check injector support" ---help--- Provide support for injecting machine checks for testing purposes.
arch/x86/kernel/cpu/mcheck/Makefile +2 -0
@@ -9,3 +9,5 @@ obj-$(CONFIG_X86_MCE_INJECT) += mce-inject.o obj-$(CONFIG_X86_THERMAL_VECTOR) += therm_throt.o obj-$(CONFIG_ACPI_APEI) += mce-apei.o + +obj-$(CONFIG_X86_MCELOG) += dev-mcelog.o
arch/x86/kernel/cpu/mcheck/dev-mcelog.c new file +397 -0
397 lines not on this page · raw message
arch/x86/kernel/cpu/mcheck/mce-internal.h +8 -0
@@ -96,3 +96,11 @@ static inline bool mce_cmp(struct mce *m1, struct mce *m2) m1->addr != m2->addr || m1->misc != m2->misc; } + +extern struct device_attribute dev_attr_trigger; + +#ifdef CONFIG_X86_MCELOG +extern void mce_work_trigger (void ); +#else +static inline void mce_work_trigger (void ) { } +#endif
arch/x86/kernel/cpu/mcheck/mce.c +29 -376
490 lines not on this page · raw message
Signature · 1 line
2.7.3
Re: [RFC PATCH 2/4] x86/mce: Deprecate /dev/mcelog
Quoted 8 lines · Borislav Petkov
From: Tony Luck <[email protected]>
Move all code relating to
/dev/mcelogto own source file./dev/mcelogdriver can now operate from the machine check notifier with lowest prio.Boris:
* Move themce_helperand trigger functionality behindCONFIG_X86_MCELOG.
Borislav Petkov
maintainerRFC PATCH 3/4 x86/mce: Merge mce_amd_inj into mce-inject
In reply to Borislav PetkovFrom: Borislav Petkov <[email protected]>
We're reusing mce_amd_inj's debugfs interface so that mce-inject can benefit from it too. The old functionality is still preserved under CONFIG_X86_MCELOG.
--- arch/x86/Kconfig | 2 +- arch/x86/kernel/cpu/mcheck/mce-inject.c | 525 +++++++++++++++++++++++++++++++- arch/x86/ras/Makefile | 2 - arch/x86/ras/mce_amd_inj.c | 484 ----------------------------- 4 files changed, 515 insertions(+), 498 deletions(-) delete mode 100644 arch/x86/ras/mce_amd_inj.c
arch/x86/Kconfig +1 -1
@@ -1041,7 +1041,7 @@ config X86_MCE_THRESHOLD def_bool y config X86_MCE_INJECT - depends on X86_MCE && X86_MCELOG + depends on X86_MCE tristate "Machine check injector support" ---help--- Provide support for injecting machine checks for testing purposes.
arch/x86/kernel/cpu/mcheck/mce-inject.c +514 -11
564 lines not on this page · raw message
arch/x86/ras/Makefile +0 -2
@@ -1,2 +0,0 @@ -obj-$(CONFIG_MCE_AMD_INJ) += mce_amd_inj.o -
arch/x86/ras/mce_amd_inj.c deleted +0 -484
484 lines not on this page · raw message
Signature · 1 line
2.7.3
Borislav Petkov
maintainerRFC PATCH 4/4 x86/mce-inject: Use debugfs_remove_recursive()
In reply to Borislav PetkovFrom: Borislav Petkov <[email protected]>
... and simplify a bit.
No functionality change.
--- arch/x86/kernel/cpu/mcheck/mce-inject.c | 30 ++++++++++++------------------ 1 file changed, 12 insertions(+), 18 deletions(-)
arch/x86/kernel/cpu/mcheck/mce-inject.c +12 -18
@@ -676,8 +676,8 @@ static struct dfs_node {static int __init debugfs_init (void ) { -int i;u64 cap; +int i;rdmsrl (MSR_IA32_MCG_CAP , cap); n_banks = cap &MCG_BANKCNT_MASK ;
@@ -700,28 +700,12 @@ static int __init debugfs_init(void)return 0 ; err_dfs_add: -while (--i >=0 ) -debugfs_remove (dfs_fls[i].d); - -debugfs_remove (dfs_inj); +debugfs_remove_recursive (dfs_inj); dfs_inj =NULL ;return -ENOMEM ; } -static void __exit debugfs_exit(void) -{ -int i; - -for (i =0 ; i <ARRAY_SIZE (dfs_fls); i++) -debugfs_remove (dfs_fls[i].d); - -memset (&dfs_fls,0 ,sizeof (dfs_fls)); - -debugfs_remove (dfs_inj); - dfs_inj =NULL ; -} -static int inject_init (void ) {int err;
@@ -752,6 +736,16 @@ module_init(inject_init);* guarantee all openers of mce_chrdev will get a reference to us. */ #ifndef CONFIG_X86_MCELOG +static void __exit debugfs_exit(void) +{ +debugfs_remove_recursive (dfs_inj); + dfs_inj =NULL ; + +unregister_nmi_handler (NMI_LOCAL ,"mce_notify" ); + +free_cpumask_var (mce_inject_cpumask); +} +module_exit (debugfs_exit);#endif
Signature · 1 line
2.7.3