x86:mce: Some cleanups and bios-set CMCI thresholds
Subsystems (2)
cover x86:mce: Some cleanups and bios-set CMCI thresholds
Hello,
I'm putting together my previous set of patches for MCE.
The first two patches are cleanups, as per previous discussions. The first patch makes mce sysfs tunables available under global machinecheck sysfs directory. We no longer remove the existing entries in the per-cpu location. The second patch consolidates some of the mce boot options into a single bitfield. The structure name has been changed and is now in mce-internal.h
The third patch adds a new boot option to honour bios-set CMCI thresholds. This has been rebased on the above patches.
This patch series applies on top of -tip.
Thanks,
Naveen
---
Naveen N. Rao (3):
x86/mce: Make sysfs tunables available globally across all cpus x86/mce: Pack boolean MCE flags into a structure x86/mce: Honour bios-set CMCI threshold
Documentation/x86/x86_64/boot-options.txt | 5 + Documentation/x86/x86_64/machinecheck | 4 + arch/x86/include/asm/mce.h | 2 - arch/x86/kernel/cpu/mcheck/mce-internal.h | 10 +++ arch/x86/kernel/cpu/mcheck/mce.c | 112 ++++++++++++++++++++++------- arch/x86/kernel/cpu/mcheck/mce_intel.c | 41 ++++++++++- 6 files changed, 140 insertions(+), 34 deletions(-)
PATCH 1/3 x86/mce: Make sysfs tunables available globally across all cpus
All the MCE attributes currently exported via sysfs appear under /sys/devices/system/machinecheck/machinecheck<n>/. Pretty much all of these are global in nature and not specific to a processor. So, make these available under /sys/devices/system/machinecheck/ where they rightly belong. Update documentation to also point to the new location so that user-space tools can pick up on the new location. We would eventually want to remove these from the per-cpu location.
--- Documentation/x86/x86_64/machinecheck | 4 ++-- arch/x86/kernel/cpu/mcheck/mce.c | 24 +++++++++++++++++++++++- 2 files changed, 25 insertions(+), 3 deletions(-)
Documentation/x86/x86_64/machinecheck +2 -2
@@ -31,8 +31,8 @@ bankNctl Note that BIOS maintain another mask to disable specific events per bank. This is not visible here -The following entries appear for each CPU, but they are truly shared -between all CPUs. +The following entries are shared between all CPUs and appear under +/sys/devices/system/machinecheck: check_interval How often to poll for corrected machine check errors, in seconds
arch/x86/kernel/cpu/mcheck/mce.c +23 -1
@@ -2205,6 +2205,7 @@ static struct dev_ext_attribute dev_attr_cmci_disabled = { &mce_cmci_disabled }; +/* Use this _only_ for per-cpu attributes */ static struct device_attribute *mce_device_attrs[] = { &dev_attr_tolerant.attr, &dev_attr_check_interval.attr,
@@ -2216,6 +2217,27 @@ static struct device_attribute *mce_device_attrs[] = { NULL }; +/* All new global attributes go here */ +static struct attribute *mce_device_global_attrs[] = { + &dev_attr_tolerant.attr.attr, + &dev_attr_check_interval.attr.attr, + &dev_attr_trigger.attr, + &dev_attr_monarch_timeout.attr.attr, + &dev_attr_dont_log_ce.attr.attr, + &dev_attr_ignore_ce.attr.attr, + &dev_attr_cmci_disabled.attr.attr, +NULL +}; + +static struct attribute_group mce_device_attr_group = { + .attrs = mce_device_global_attrs, +}; + +static const struct attribute_group *mce_device_attr_groups[] = { + &mce_device_attr_group, +NULL , +}; +static cpumask_var_t mce_device_initialized;static void mce_device_release (struct device *dev)
@@ -2397,7 +2419,7 @@ static __init int mcheck_init_device(void)mce_init_banks (); - err =subsys_system_register (&mce_subsys,NULL ); + err =subsys_system_register (&mce_subsys, mce_device_attr_groups);if (err)return err;
PATCH 1/1 Start using the new sysfs tunables location
All the current mce tunables are now available under /sys/devices/system/machinecheck. Start using this new location, but fall back to the older per-cpu location so that we continue working with older kernels.
--- README | 2 +- mcelog.init | 5 ++++- tests/test | 7 ++++++- 3 files changed, 11 insertions(+), 3 deletions(-)
README +1 -1
@@ -18,7 +18,7 @@ significantly (upto 10 minutes) and does not allow mcelog to keep extended state trigger is a newer method where the kernel runs mcelog on a error. This is configured with -echo /usr/sbin/mcelog > /sys/devices/system/machinecheck/machinecheck0/trigger +echo /usr/sbin/mcelog > /sys/devices/system/machinecheck/trigger This is faster, but still doesn't allow mcelog to keep state, and has relatively high overhead for each error because a program has to be initialized from scratch.
mcelog.init +4 -1
@@ -31,7 +31,10 @@ MCELOG_OPTIONS="" # private settings MCELOG=${MCELOG:-/usr/sbin/mcelog} -TRIGGER=/sys/devices/system/machinecheck/machinecheck0/trigger +TRIGGER=/sys/devices/system/machinecheck/trigger +if [ ! -e $TRIGGER ] ; then + TRIGGER=/sys/devices/system/machinecheck/machinecheck0/trigger +fi [ ! -x $MCELOG ] && ( echo "mcelog not found" ; exit 1 ) [ ! -r /dev/mcelog ] && ( echo "/dev/mcelog not active" ; exit 0 )
tests/test +6 -1
@@ -17,10 +17,15 @@ if [ "$(whoami)" != "root" ] ; then exit 1 fi +TRIGGER=/sys/devices/system/machinecheck/trigger +if [ ! -e $TRIGGER ] ; then + TRIGGER=/sys/devices/system/machinecheck/machinecheck0/trigger +fi + echo "++++++++++++ running $1 test +++++++++++++++++++" # disable trigger -echo -n "" > /sys/devices/system/machinecheck/machinecheck0/trigger +echo -n "" > $TRIGGER killall mcelog || true #killwatchdog() {
Re: [PATCH] [mcelog] Start using the new sysfs tunables location
Quoted 3 lines · Naveen N. Rao
All the current mce tunables are now available under
/sys/devices/system/machinecheck. Start using this new location, but fall back to the older per-cpu location so that we continue working with older kernels.
Who did that change in the kernel?
That breaks Linus rule that the kernel should not break userland. Kernel needs to fix that.
-Andi
Re: [PATCH] [mcelog] Start using the new sysfs tunables location
Quoted 9 lines · Andi Kleen
Quoted 3 lines · Naveen N. Rao
All the current mce tunables are now available under
/sys/devices/system/machinecheck. Start using this new location, but fall back to the older per-cpu location so that we continue working with older kernels.Who did that change in the kernel?
That breaks Linus rule that the kernel should not break userland. Kernel needs to fix that.
The change is still under discussion. Stage one is to add the new global pathnames in addition to keeping the old per-cpu ones. Also fix all utilities (just mcelog(8) as far as we know) to prefer the new paths.
After some time[1] ... delete the old paths. This is allowable under Linus' modified edict that you can change ABI "if nobody complains". If we wait long enough that the new mcelog is widely deployed, then nobody should complain.
-Tony
[1] several years - not just a kernel release or two.
Re: [PATCH] [mcelog] Start using the new sysfs tunables location
Quoted 15 lines · Tony Luck
Quoted 9 lines · Andi Kleen
Quoted 3 lines · Naveen N. Rao
All the current mce tunables are now available under
/sys/devices/system/machinecheck. Start using this new location, but fall back to the older per-cpu location so that we continue working with older kernels.Who did that change in the kernel?
That breaks Linus rule that the kernel should not break userland. Kernel needs to fix that.
The change is still under discussion. Stage one is to add the new global pathnames in addition to keeping the old per-cpu ones. Also fix all utilities (just mcelog(8) as far as we know) to prefer the new paths.
Quoted 9 lines
After some time[1] ... delete the old paths. This is allowable under Linus' modified edict that you can change ABI "if nobody complains". If we wait long enough that the new mcelog is widely deployed, then nobody should complain.
-Tony
[1] several years - not just a kernel release or two.
Tony,
Thanks for clarifying. I should have mentioned in the patch description that this is indeed subject to the original patch making it into the kernel.
On a related topic. I recently noticed that we don't have an entry for machinecheck in Documentation/ABI/. Should we add an entry in there? We could perhaps add the existing entries under obsolete/ and the new location under testing/?
- Naveen
Re: [PATCH] [mcelog] Start using the new sysfs tunables location
In reply to Tony LuckQuoted 3 lines
The change is still under discussion. Stage one is to add the new global pathnames in addition to keeping the old per-cpu ones. Also fix all utilities (just mcelog(8) as far as we know) to prefer the new paths.
But why do you even want to change it? Does it fix anything? AFAIK the old setup -- while not being pretty -- works just fine.
-Andi
Re: [PATCH] [mcelog] Start using the new sysfs tunables location
Quoted 6 lines · Andi Kleen
Quoted 3 lines
The change is still under discussion. Stage one is to add the new global pathnames in addition to keeping the old per-cpu ones. Also fix all utilities (just mcelog(8) as far as we know) to prefer the new paths.
But why do you even want to change it? Does it fix anything? AFAIK the old setup -- while not being pretty -- works just fine.
The reason for this was explained in this thread:
http://www.mail-archive.com/[email protected]/msg298302.html
Even if we decide not to remove these tunables from under their current per-cpu location, I still think it is much cleaner to have them available under /sys/devices/system/machinecheck.
- Naveen
Re: [PATCH] [mcelog] Start using the new sysfs tunables location
Quoted 8 lines · On Thu, 06 Sep 2012 18:04:27 +0530 "Naveen N. Rao"…
Quoted 6 lines · Andi Kleen
Quoted 3 lines
The change is still under discussion. Stage one is to add the new global pathnames in addition to keeping the old per-cpu ones. Also fix all utilities (just mcelog(8) as far as we know) to prefer the new paths.
But why do you even want to change it? Does it fix anything? AFAIK the old setup -- while not being pretty -- works just fine.
Quoted 6 lines
The reason for this was explained in this thread:
http://www.mail-archive.com/[email protected]/msg298302.htmlEven if we decide not to remove these tunables from under their current per-cpu location, I still think it is much cleaner to have them available under
/sys/devices/system/machinecheck.
That to me seems a ridiculous proposal. What are you going to do if in future they ceased to be system wide ? Move them back ?
The threshold for playing musical chairs with sysfs nodes is a lot higher than "I think it's much cleaner"
The current approach is a lot more futureproof even if a spot more ugly.
Alan
Re: [PATCH] [mcelog] Start using the new sysfs tunables location
In reply to Naveen N. RaoQuoted 3 lines
Even if we decide not to remove these tunables from under their current per-cpu location, I still think it is much cleaner to have them available under
/sys/devices/system/machinecheck.
"much cleaner" is not sufficient justification to break an ABI.
-Andi
Signature · 1 line
[email protected] -- Speaking for myself only.
PATCH 2/3 x86/mce: Pack boolean MCE flags into a structure
In reply to Naveen N. RaoMany MCE flags are boolean in nature, but are declared as integers currently. We can pack these into a bitfield to save some space.
--- arch/x86/include/asm/mce.h | 2 - arch/x86/kernel/cpu/mcheck/mce-internal.h | 9 +++ arch/x86/kernel/cpu/mcheck/mce.c | 88 +++++++++++++++++++---------- arch/x86/kernel/cpu/mcheck/mce_intel.c | 2 - 4 files changed, 67 insertions(+), 34 deletions(-)
arch/x86/include/asm/mce.h +0 -2
@@ -169,8 +169,6 @@ DECLARE_PER_CPU(struct device *, mce_device);#define MAX_NR_BANKS 32#ifdef CONFIG_X86_MCE_INTEL -extern int mce_cmci_disabled; -extern int mce_ignore_ce;void mce_intel_feature_init (struct cpuinfo_x86 *c);void cmci_clear (void );void cmci_reenable (void );
arch/x86/kernel/cpu/mcheck/mce-internal.h +9 -0
@@ -11,6 +11,15 @@ enum severity_level { MCE_PANIC_SEVERITY, }; +struct mce_config { +__u32 cmci_disabled :1 , + ignore_ce :1 , + dont_log_ce :1 , + __pad :29 ; +}; + +extern struct mce_config mce_cfg; +#define ATTR_LEN 16/* One object for each MCE bank, shared by all CPUs */
arch/x86/kernel/cpu/mcheck/mce.c +57 -31
175 lines not on this page · raw message
arch/x86/kernel/cpu/mcheck/mce_intel.c +1 -1
@@ -53,7 +53,7 @@ static int cmci_supported(int *banks) {u64 cap; -if (mce_cmci_disabled || mce_ignore_ce) +if (mce_cfg.cmci_disabled || mce_cfg.ignore_ce)return 0 ;/*
Re: [PATCH 2/3] x86/mce: Pack boolean MCE flags into a structure
Quoted 2 lines · Naveen N. Rao
Many MCE flags are boolean in nature, but are declared as integers currently. We can pack these into a bitfield to save some space.
[]
arch/x86/kernel/cpu/mcheck/mce-internal.h
Quoted 1 line
arch/x86/kernel/cpu/mcheck/mce-internal.h +0
-0
[]
arch/x86/kernel/cpu/mcheck/mce-internal.h enum severity_level { · line 18
Quoted 9 lines
arch/x86/kernel/cpu/mcheck/mce-internal.h +5
-0@@ -11,6 +11,15 @@ enum severity_level { MCE_PANIC_SEVERITY, }; +struct mce_config { +__u32 cmci_disabled :1 , + ignore_ce :1 , + dont_log_ce :1 , + __pad :29 ;
You don't need to declare __pad
Perhaps bool:1 instead.
Re: [PATCH 2/3] x86/mce: Pack boolean MCE flags into a structure
In reply to Naveen N. RaoQuoted 2 lines · Naveen N. Rao
Many MCE flags are boolean in nature, but are declared as integers currently. We can pack these into a bitfield to save some space.
Before this patch:
size arch/x86/kernel/cpu/mcheck/mce.o
text data bss dec hex filename 18946 4930 776 24652 604c arch/x86/kernel/cpu/mcheck/mce.o
After:
size arch/x86/kernel/cpu/mcheck/mce.o
text data bss dec hex filename 19335 4890 776 25001 61a9 arch/x86/kernel/cpu/mcheck/mce.o
So we do indeed see "data" reduced by 40 bytes. But "text" is up by 389. This seems to be because you have another change, not described in the commit log, buried in part 2 to add get_dont_log_ce(), set_dont_log_ce() etc.
Compiler version: gcc version 4.4.6 20120305 (Red Hat 4.4.6-4) (GCC)
I know I'm contradicting the feedback you got from Borislav here, but is this code churn really worth it to save 40 bytes? I don't think so.
-Tony
Re: [PATCH 2/3] x86/mce: Pack boolean MCE flags into a structure
Quoted 20 lines · Tony Luck
Quoted 2 lines · Naveen N. Rao
Many MCE flags are boolean in nature, but are declared as integers currently. We can pack these into a bitfield to save some space.
Before this patch:
sizearch/x86/kernel/cpu/mcheck/mce.otext data bss dec hex filename 18946 4930 776 24652 604c arch/x86/kernel/cpu/mcheck/mce.oAfter:
sizearch/x86/kernel/cpu/mcheck/mce.otext data bss dec hex filename 19335 4890 776 25001 61a9 arch/x86/kernel/cpu/mcheck/mce.oSo we do indeed see "data" reduced by 40 bytes. But "text" is up by 389. This seems to be because you have another change, not described in the commit log, buried in part 2 to add
get_dont_log_ce(),set_dont_log_ce()etc.
Quoted 4 lines
Compiler version: gcc version 4.4.6 20120305 (Red Hat 4.4.6-4) (GCC)
I know I'm contradicting the feedback you got from Borislav here, but is this code churn really worth it to save 40 bytes? I don't think so.
Hmm.. I think I agree. I don't see a good way to get rid of the individual getters and setters without adding some more code churn. I guess using boolean would be better. Boris?
Thanks,
Naveen
Re: [PATCH 2/3] x86/mce: Pack boolean MCE flags into a structure
Quoted 3 lines · Naveen N. Rao
Quoted 3 lines
I know I'm contradicting the feedback you got from Borislav here, but is this code churn really worth it to save 40 bytes? I don't think so.
Well, to answer Tony's question, I wanted to have all those config booleans at the beginning of mce.c packed tightly together as a single-bit bool values in a config-like struct so that they don't scatter all over the place and grow out of proportion with more features being added. So actually I'm willing to swallow the slight increase in code size for better/more clear code.
Quoted 3 lines
Hmm.. I think I agree. I don't see a good way to get rid of the individual getters and setters without adding some more code churn. I guess using boolean would be better. Boris?
Yes, it would be worth to try to add a DEVICE_BOOL_ATTR which introduce a setter/getter flavour for bools and then use that for all.
Thanks.
Signature · 8 lines
Regards/Gruss, Boris. Advanced Micro Devices GmbH Einsteinring 24, 85609 Dornach GM: Alberto Bozzo Reg: Dornach, Landkreis Muenchen HRB Nr. 43632 WEEE Registernr: 129 19551
PATCH 3/3 x86/mce: Honour bios-set CMCI threshold
In reply to Naveen N. RaoThe ACPI spec doesn't provide for a way for the bios to pass down recommended thresholds to the OS on a _per-bank_ basis. This patch adds a new boot option, which if passed, allows bios to initialize the CMCI threshold. In such a case, we simply skip programming any threshold value.
As fail-safe, we initialize threshold to 1 if some banks have not been initialized by the bios and warn the user.
--- Documentation/x86/x86_64/boot-options.txt | 5 ++++ arch/x86/kernel/cpu/mcheck/mce-internal.h | 3 +- arch/x86/kernel/cpu/mcheck/mce.c | 12 +++++++++ arch/x86/kernel/cpu/mcheck/mce_intel.c | 39 +++++++++++++++++++++++++++-- 4 files changed, 55 insertions(+), 4 deletions(-)
Documentation/x86/x86_64/boot-options.txt +5 -0
@@ -50,6 +50,11 @@ Machine check monarchtimeout: Sets the time in us to wait for other CPUs on machine checks. 0 to disable. + mce=bios_cmci_threshold + Don't overwrite the bios-set CMCI threshold. This boot option + prevents Linux from overwriting the CMCI threshold set by the + bios. Without this option, Linux always sets the CMCI + threshold to 1. nomce (for compatibility with i386): same as mce=off
arch/x86/kernel/cpu/mcheck/mce-internal.h +2 -1
@@ -15,7 +15,8 @@ struct mce_config { __u32 cmci_disabled : 1, ignore_ce : 1, dont_log_ce : 1, - __pad : 29; + bios_cmci_threshold : 1, + __pad : 28; };extern struct mce_config mce_cfg;
arch/x86/kernel/cpu/mcheck/mce.c +12 -0
@@ -1945,6 +1945,7 @@ static struct miscdevice mce_chrdev_device = {* check, or 0 to not wait * mce=bootlog Log MCEs from before booting. Disabled by default on AMD. * mce=nobootlog Don't log MCEs from before booting. +* mce=bios_cmci_threshold Don't program the CMCI threshold */ static int __init mcheck_enable (char *str) {
@@ -1964,6 +1965,8 @@ static int __init mcheck_enable(char *str) mce_cfg.ignore_ce =1 ; else if (!strcmp(str, "bootlog") || !strcmp(str, "nobootlog")) mce_bootlog = (str[0 ] =='b' ); + else if (!strcmp(str, "bios_cmci_threshold")) + mce_cfg.bios_cmci_threshold =1 ; else if (isdigit(str[0])) {get_option (&str, &tolerant);if (*str ==',' ) {
@@ -2210,6 +2213,13 @@ static ssize_t set_cmci_disabled(struct device *s,return size; } +static ssize_t get_bios_cmci_threshold (struct device *dev, +struct device_attribute *attr, +char *buf) +{ +return snprintf (buf,PAGE_SIZE ,"%d\n" , mce_cfg.bios_cmci_threshold); +} +static ssize_t store_int_with_restart (struct device *s,struct device_attribute *attr,const char *buf,size_t size)
@@ -2225,6 +2235,7 @@ static DEVICE_INT_ATTR(monarch_timeout, 0644, monarch_timeout); static DEVICE_ATTR(dont_log_ce, 0644, get_dont_log_ce, set_dont_log_ce); static DEVICE_ATTR(ignore_ce, 0644, get_ignore_ce, set_ignore_ce); static DEVICE_ATTR(cmci_disabled, 0644, get_cmci_disabled, set_cmci_disabled); +static DEVICE_ATTR(bios_cmci_threshold, 0444, get_bios_cmci_threshold, NULL);static struct dev_ext_attribute dev_attr_check_interval = {__ATTR (check_interval,0644 , device_show_int, store_int_with_restart),
@@ -2252,6 +2263,7 @@ static struct attribute *mce_device_global_attrs[] = { &dev_attr_dont_log_ce.attr, &dev_attr_ignore_ce.attr, &dev_attr_cmci_disabled.attr, + &dev_attr_bios_cmci_threshold.attr, NULL };
arch/x86/kernel/cpu/mcheck/mce_intel.c +36 -3
@@ -181,10 +181,16 @@ static void cmci_discover(int banks)unsigned long *owned = (void *)&__get_cpu_var (mce_banks_owned);unsigned long flags;int i; +int bios_wrong_thresh =0 ; + +if (mce_cfg.bios_cmci_threshold) +printk_once (KERN_INFO +"bios_cmci_threshold: Using bios-set threshold values for CMCI" );raw_spin_lock_irqsave (&cmci_discover_lock, flags);for (i =0 ; i < banks; i++) {u64 val; +int bios_zero_thresh =0 ;if (test_bit (i, owned))continue ;
@@ -198,8 +204,20 @@ static void cmci_discover(int banks)continue ; } - val &= ~MCI_CTL2_CMCI_THRESHOLD_MASK ; - val |=MCI_CTL2_CMCI_EN |CMCI_THRESHOLD ; +if (!mce_cfg.bios_cmci_threshold) { + val &= ~MCI_CTL2_CMCI_THRESHOLD_MASK ; + val |=CMCI_THRESHOLD ; + }else if (!(val &MCI_CTL2_CMCI_THRESHOLD_MASK )) { +/* +* If bios_cmci_threshold boot option was specified +* but the threshold is zero, we'll try to initialize +* it to 1. +*/ + bios_zero_thresh =1 ; + val |=CMCI_THRESHOLD ; + } + + val |=MCI_CTL2_CMCI_EN ;wrmsrl (MSR_IA32_MCx_CTL2 (i), val);rdmsrl (MSR_IA32_MCx_CTL2 (i), val);
@@ -207,11 +225,26 @@ static void cmci_discover(int banks)if (val &MCI_CTL2_CMCI_EN ) {set_bit (i, owned);__clear_bit (i,__get_cpu_var (mce_poll_banks)); +/* +* We are able to set thresholds for some banks that +* had a threshold of 0. This means the BIOS has not +* set the thresholds properly or does not work with +* this boot option. Note down now and report later. +*/ +if (mce_cfg.bios_cmci_threshold && bios_zero_thresh && + (val &MCI_CTL2_CMCI_THRESHOLD_MASK )) + bios_wrong_thresh =1 ; }else {WARN_ON (!test_bit (i,__get_cpu_var (mce_poll_banks))); } }raw_spin_unlock_irqrestore (&cmci_discover_lock, flags); +if (mce_cfg.bios_cmci_threshold && bios_wrong_thresh) { +printk_once (KERN_INFO +"bios_cmci_threshold: Some banks do not have valid thresholds set" ); +printk_once (KERN_INFO +"bios_cmci_threshold: Make sure your BIOS supports this boot option" ); + } }/*
@@ -249,7 +282,7 @@ void cmci_clear(void)continue ;/* Disable CMCI */ rdmsrl (MSR_IA32_MCx_CTL2 (i), val); - val &= ~(MCI_CTL2_CMCI_EN |MCI_CTL2_CMCI_THRESHOLD_MASK ); + val &= ~MCI_CTL2_CMCI_EN ;wrmsrl (MSR_IA32_MCx_CTL2 (i), val);__clear_bit (i,__get_cpu_var (mce_banks_owned)); }