3/3: MCA/MCE correctable error handling
PATCH 1/1 3/3: MCA/MCE correctable error handling
This is patch 3/3.
Signature · 12 lines
AMD Saxony, Dresden, Germany Operating System Research Center Legal Information: AMD Saxony Limited Liability Company & Co. KG Sitz (Geschäftsanschrift): Wilschdorfer Landstr. 101, 01109 Dresden, Deutschland Registergericht Dresden: HRA 4896 vertretungsberechtigter Komplementär: AMD Saxony LLC (Sitz Wilmington, Delaware, USA) Geschäftsführer der AMD Saxony LLC: Dr. Hans-R. Deppe, Thomas McCoy
xen/arch/x86/cpu/mcheck/mce.c +92 -0
@@ -13,6 +13,7 @@#include <asm/system.h> #include "mce.h" +#include "x86_mca.h" int mce_disabled =0 ;unsigned int nr_mce_banks;
@@ -89,3 +90,94 @@ static int __init mcheck_enable(char *stcustom_param ("nomce" , mcheck_disable);custom_param ("mce" , mcheck_enable); + + + + +void x86_mcinfo_clear (struct shared_info *si) +{ +memset (&si->arch.mc_info,0 ,sizeof (struct arch_mc_info )); +x86_mcinfo_nentries (si) =0 ; +} + + +int x86_mcinfo_add (struct shared_info *si,void *mcinfo) +{ +int i; +unsigned long end1, end2; +struct mcinfo_common *mic, *mic_base, *mic_index; + + mic = (struct mcinfo_common *)mcinfo; + mic_index = mic_base =x86_mcinfo_first (si); + +/* go to first free entry */ +for (i =0 ; i <x86_mcinfo_nentries (si); i++) { + mic_index =x86_mcinfo_next (mic_index); + } + +/* check if there is enough size */ + end1 = (unsigned long )((uint8_t *)mic_base +sizeof (struct arch_mc_info )); + end2 = (unsigned long )((uint8_t *)mic_index + mic->size); + +if (end1 < end2) +return -1 ;/* No space. Can't add entry. */ + +/* there's enough space. add entry. */ +memcpy (mic_index, mic, mic->size); +x86_mcinfo_nentries (si)++; + +return 0 ; +} + +/* Dump machine check information in a format, +* mcelog can parse. This is used only when +* Dom0 does not take the notification. */ +void x86_mcinfo_dump (struct shared_info *si) +{ +struct mcinfo_common *mic; +struct mcinfo_global *mc_global; +struct mcinfo_bank *mc_bank; + +/* first print the global info */ +x86_mcinfo_lookup (mic, si,MC_TYPE_GLOBAL ); +if (mic ==NULL ) +return ; + mc_global = (struct mcinfo_global *)mic; +if (mc_global->mc_flags &MC_FLAG_UNCORRECTABLE ) { +printk (KERN_EMERG +"CPU%d: Machine Check Exception: %16" PRIx64"\n" , + mc_global->mc_coreid, mc_global->mc_gstatus); + }else { +printk (KERN_INFO "MCE: The hardware reports a non " +"fatal, correctable incident occured on " +"CPU %d.\n" , + mc_global->mc_coreid); + } + +/* then the bank information */ +x86_mcinfo_lookup (mic, si,MC_TYPE_BANK );/* finds the first entry */ +do { +if (mic ==NULL ) +return ; +if (mic->type !=MC_TYPE_BANK ) +continue ; + + mc_bank = (struct mcinfo_bank *)mic; + +if (mc_global->mc_flags &MC_FLAG_UNCORRECTABLE ) +printk (KERN_EMERG ); +else +printk (KERN_INFO ); + +printk ("Bank %d: %16" PRIx64, + mc_bank->mc_bank, + mc_bank->mc_status); +if (mc_bank->mc_status & MCi_STATUS_MISCV) +printk ("[%16" PRIx64"]" , mc_bank->mc_misc); +if (mc_bank->mc_status & MCi_STATUS_ADDRV) +printk (" at %16" PRIx64, mc_bank->mc_addr); + +printk ("\n" ); + mic =x86_mcinfo_next (mic);/* next entry */ + }while (mic !=NULL ); +}
xen/arch/x86/cpu/mcheck/mce.h +4 -0
@@ -6,6 +6,10 @@ void intel_p6_mcheck_init(struct cpuinfovoid intel_p6_mcheck_init (struct cpuinfo_x86 *c);void winchip_mcheck_init (struct cpuinfo_x86 *c); +void x86_mcinfo_clear (struct shared_info *si); +int x86_mcinfo_add (struct shared_info *si,void *mcinfo); +void x86_mcinfo_dump (struct shared_info *si); +/* Call the installed machine check handler for this CPU setup. */ extern fastcall void (*machine_check_vector)(struct cpu_user_regs *, long error_code);
xen/arch/x86/cpu/mcheck/non-fatal.c +223 -16
271 lines not on this page · raw message
xen/common/event_channel.c +15 -0
@@ -539,6 +539,21 @@ void evtchn_set_pending(struct vcpu *v, } +int guest_enabled_event (struct vcpu *v,int virq) +{ +int port; + +if (unlikely (v ==NULL ) ) +return 0 ; + + port = v->virq_to_evtchn[virq]; +if ( port ==0 ) +return 0 ; + +/* virq is in use */ +return 1 ; +} +void send_guest_vcpu_virq (struct vcpu *v,int virq) {int port;
xen/include/asm-x86/event.h +6 -1
@@ -61,7 +61,12 @@ static inline void local_event_delivery_/* No arch specific virq definition now. Default to global. */ static inline int arch_virq_is_global (int virq) { -return 1 ; +switch (virq) { +case VIRQ_MCA : +return 1 ; +default : +return 1 ; + } }#endif
xen/include/public/arch-x86/xen.h +2 -0
@@ -82,6 +82,8 @@ DEFINE_XEN_GUEST_HANDLE(xen_pfn_t);#ifndef __ASSEMBLY__ +#define VIRQ_MCA VIRQ_ARCH_0/* G. (DOM0) Machine Check Architecture */ +/* * Machine Check Architecure: * structs are read-only and used to report all kinds of
xen/include/xen/event.h +3 -0
@@ -51,6 +51,9 @@ void free_xen_event_channel(void free_xen_event_channel (struct vcpu *local_vcpu,int port); +/* Query if event channel is in use by the guest */ +int guest_enabled_event (struct vcpu *v,int virq); +/* Notify remote end of a Xen-attached event channel.*/ void notify_via_xen_event_channel (int lport);
xen/arch/x86/cpu/mcheck/x86_mca.h new file +72 -0
72 lines not on this page · raw message
_______________________________________________
Xen-devel mailing list
[email protected]
http://lists.xensource.com/xen-devel
xen/arch/x86/cpu/mcheck/mce.c static int __init mcheck_enable(char *st · line 170
Quoted 4 lines
xen/arch/x86/cpu/mcheck/mce.c +4
-0+if (mc_global->mc_flags &MC_FLAG_UNCORRECTABLE ) +printk (KERN_EMERG ); +else +printk (KERN_INFO );
KERN_INFO seems gross understatement here - generally, correctable MCs are considered indicators that within not too distant future uncorrectable MCs might result, so this generally is a call for action (and hence shouldn't be hidden with default log level settings).
Also, I'm not sure adjusting the polling frequency makes much sense - 30s seems an awful lot of time to me.
Jan
Quoted 9 lines · On Tuesday 21 August 2007 18:02:54 Jan Beulich wrote:
xen/arch/x86/cpu/mcheck/mce.c static int __init mcheck_enable(char *st · line 170
Quoted 4 lines
xen/arch/x86/cpu/mcheck/mce.c +4
-0+if (mc_global->mc_flags &MC_FLAG_UNCORRECTABLE ) +printk (KERN_EMERG ); +else +printk (KERN_INFO );
KERN_INFOseems gross understatement here - generally, correctable MCs are considered indicators that within not too distant future uncorrectable MCs might result, so this generally is a call for action (and hence shouldn't be hidden with default log level settings).
Well, that is what the "old" code did. It used KERN_EMERG for fatal errors and KERN_INFO in the polling service routine. What do you want me to suggest?
Quoted 2 lines
Also, I'm not sure adjusting the polling frequency makes much sense - 30s seems an awful lot of time to me.
It's not clear to me what you are trying to tell me. Please explain/elaborate.
Christoph
Signature · 12 lines
AMD Saxony, Dresden, Germany Operating System Research Center Legal Information: AMD Saxony Limited Liability Company & Co. KG Sitz (Geschäftsanschrift): Wilschdorfer Landstr. 101, 01109 Dresden, Deutschland Registergericht Dresden: HRA 4896 vertretungsberechtigter Komplementär: AMD Saxony LLC (Sitz Wilmington, Delaware, USA) Geschäftsführer der AMD Saxony LLC: Dr. Hans-R. Deppe, Thomas McCoy
Quoted 12 lines
Quoted 1 line
Quoted 1 line
"Christoph Egger" <[email protected]> 22.08.07 11:00 >>>
Quoted 9 lines · On Tuesday 21 August 2007 18:02:54 Jan Beulich wrote:
xen/arch/x86/cpu/mcheck/mce.c static int __init mcheck_enable(char *st · line 170
Quoted 4 lines
xen/arch/x86/cpu/mcheck/mce.c +4
-0+if (mc_global->mc_flags &MC_FLAG_UNCORRECTABLE ) +printk (KERN_EMERG ); +else +printk (KERN_INFO );
KERN_INFOseems gross understatement here - generally, correctable MCs are considered indicators that within not too distant future uncorrectable MCs might result, so this generally is a call for action (and hence shouldn't be hidden with default log level settings).
Quoted 2 lines
Well, that is what the "old" code did. It used
KERN_EMERGfor fatal errors andKERN_INFOin the polling service routine. What do you want me to suggest?
This should be at least KERN_WARNING, probably even KERN_ERR (note though that KERN_ERR and KERN_EMERG both resolve to XENLOG_ERR).
Quoted 4 lines
Quoted 2 lines
Also, I'm not sure adjusting the polling frequency makes much sense - 30s seems an awful lot of time to me.
It's not clear to me what you are trying to tell me. Please explain/elaborate.
What I'm trying to say is that I'd think this should be polled at a much higher frequency (I'd suggest 1Hz), without adjustments. Typically, a healthy system will not encounter problems soon after boot, but after running for perhaps a very long time (and a system in bad condition is likely to encounter problems right away, so wouldn't be affected by changing the polling rate). Thus, in the general case, you'd have a comparably long latency, during which some kind of (automated) action could already be taken to preserve data consistency.
Jan
Quoted 13 lines · On Wednesday 22 August 2007 12:09:41 Jan Beulich wrote:
Quoted 13 lines
Quoted 1 line
Quoted 1 line
"Christoph Egger" <[email protected]> 22.08.07 11:00 >>>
Quoted 9 lines · On Tuesday 21 August 2007 18:02:54 Jan Beulich wrote:
xen/arch/x86/cpu/mcheck/mce.c static int __init mcheck_enable(char *st · line 170
Quoted 4 lines
xen/arch/x86/cpu/mcheck/mce.c +4
-0+if (mc_global->mc_flags &MC_FLAG_UNCORRECTABLE ) +printk (KERN_EMERG ); +else +printk (KERN_INFO );
KERN_INFOseems gross understatement here - generally, correctable MCs are considered indicators that within not too distant future uncorrectable MCs might result, so this generally is a call for action (and hence shouldn't be hidden with default log level settings).
Quoted 6 lines
Quoted 3 lines
Well, that is what the "old" code did. It used
KERN_EMERGfor fatal errors andKERN_INFOin the polling service routine. What do you want me to suggest?This should be at least
KERN_WARNING, probably evenKERN_ERR(note though thatKERN_ERRandKERN_EMERGboth resolve toXENLOG_ERR).
I changed to KERN_WARNING. This made the above if block superflous. Tnx.
I will re-submit this patch as well.
Quoted 6 lines
Quoted 5 lines
Quoted 2 lines
Also, I'm not sure adjusting the polling frequency makes much sense - 30s seems an awful lot of time to me.
It's not clear to me what you are trying to tell me. Please explain/elaborate.
Quoted 8 lines
What I'm trying to say is that I'd think this should be polled at a much higher frequency (I'd suggest 1Hz), without adjustments. Typically, a healthy system will not encounter problems soon after boot, but after running for perhaps a very long time (and a system in bad condition is likely to encounter problems right away, so wouldn't be affected by changing the polling rate). Thus, in the general case, you'd have a comparably long latency, during which some kind of (automated) action could already be taken to preserve data consistency.
The polling routine that is in the -unstable tree (the version taken from Linux) runs every 15 seconds without adjustments.
1Hz causes too much system load for a healthy system IMO. That's why I introduced the adjustments with use of hw threshold registers to come to a compromise solution.
Signature · 12 lines
AMD Saxony, Dresden, Germany Operating System Research Center Legal Information: AMD Saxony Limited Liability Company & Co. KG Sitz (Geschäftsanschrift): Wilschdorfer Landstr. 101, 01109 Dresden, Deutschland Registergericht Dresden: HRA 4896 vertretungsberechtigter Komplementär: AMD Saxony LLC (Sitz Wilmington, Delaware, USA) Geschäftsführer der AMD Saxony LLC: Dr. Hans-R. Deppe, Thomas McCoy
Quoted 9 lines · Christoph Egger
Quoted 8 lines
What I'm trying to say is that I'd think this should be polled at a much higher frequency (I'd suggest 1Hz), without adjustments. Typically, a healthy system will not encounter problems soon after boot, but after running for perhaps a very long time (and a system in bad condition is likely to encounter problems right away, so wouldn't be affected by changing the polling rate). Thus, in the general case, you'd have a comparably long latency, during which some kind of (automated) action could already be taken to preserve data consistency.
Quoted 5 lines
The polling routine that is in the -unstable tree (the version taken from Linux) runs every 15 seconds without adjustments.
1Hz causes too much system load for a healthy system IMO. That's why I introduced the adjustments with use of hw threshold registers to come to a compromise solution.
What's the deal here? Do correctable errors not cause an MCE, yet are still detected via the machine-check architecture (albeit by a polling method)?
Are there going to be patches on the Linux side to pick up this MCA info? What is Linux going to do with it, apart from log it (which Xen can already do itself)? Or is this all Solaris-specific?
-- Keir
Quoted 6 lines · Keir Fraser
Quoted 5 lines
The polling routine that is in the -unstable tree (the version taken from Linux) runs every 15 seconds without adjustments.
1Hz causes too much system load for a healthy system IMO. That's why I introduced the adjustments with use of hw threshold registers to come to a compromise solution.
Quoted 6 lines
What's the deal here? Do correctable errors not cause an MCE, yet are still detected via the machine-check architecture (albeit by a polling method)?
Are there going to be patches on the Linux side to pick up this MCA info? What is Linux going to do with it, apart from log it (which Xen can already do itself)? Or is this all Solaris-specific?
Oh, and is AMD-specific code really needed in non-fatal.c? I though the MCA stuff was architectural now rather than vendor specific? If there are vendor-specific extensions then they belong in the vendor's .c file.
-- Keir
Quoted 10 lines · On Wednesday 22 August 2007 18:10:24 Keir Fraser wrote:
Quoted 9 lines · Keir Fraser
Quoted 5 lines
The polling routine that is in the -unstable tree (the version taken from Linux) runs every 15 seconds without adjustments. 1Hz causes too much system load for a healthy system IMO. That's why I introduced the adjustments with use of hw threshold registers to come to a compromise solution.
What's the deal here? Do correctable errors not cause an MCE, yet are still detected via the machine-check architecture (albeit by a polling method)?
The deal here is, detect correctable errors via polling und uncorrectable errors via MCE.
This patchset is about correctable errors.
Quoted 3 lines
Quoted 3 lines
Are there going to be patches on the Linux side to pick up this MCA info? What is Linux going to do with it, apart from log it (which Xen can already do itself)? Or is this all Solaris-specific?
The general idea is the Dom0 picks up this MCA info and a) uses the error-handling infrastructure provided for the non-virtualized form and b) will use hypercalls to tell xen to also report MCA to a DomU and/or kill a DomU.
Some hw features for self-healing can only use Dom0 (because registers sit in the PCI extended config space, Xen doesn't have access to) and some can use Xen itself.
I wrote a demo driver that mainly tests that the Dom0 actually receives the MCA info for NetBSD/Xen (Sun prefers to look into BSD licensed code). It should be easy to port it to Linux.
Quoted 3 lines
Oh, and is AMD-specific code really needed in
non-fatal.c? I though the MCA stuff was architectural now rather than vendor specific? If there are vendor-specific extensions then they belong in the vendor's .c file.
AMD-specific is the use of the hw register code. Intel has some additional machine check MSR's containing the register set. Intel may add a structure to patch 2/3 that make use of them. Should I move the amd polling handler to amd.c ?
Signature · 12 lines
AMD Saxony, Dresden, Germany Operating System Research Center Legal Information: AMD Saxony Limited Liability Company & Co. KG Sitz (Geschäftsanschrift): Wilschdorfer Landstr. 101, 01109 Dresden, Deutschland Registergericht Dresden: HRA 4896 vertretungsberechtigter Komplementär: AMD Saxony LLC (Sitz Wilmington, Delaware, USA) Geschäftsführer der AMD Saxony LLC: Dr. Hans-R. Deppe, Thomas McCoy
PATCH 1/1 resend 3/3: MCA/MCE correctable error handling
Yesterday I said, I will re-send this patch. Here is it. It incorporates feedback from Jan Beulich.
Signature · 12 lines
AMD Saxony, Dresden, Germany Operating System Research Center Legal Information: AMD Saxony Limited Liability Company & Co. KG Sitz (Geschäftsanschrift): Wilschdorfer Landstr. 101, 01109 Dresden, Deutschland Registergericht Dresden: HRA 4896 vertretungsberechtigter Komplementär: AMD Saxony LLC (Sitz Wilmington, Delaware, USA) Geschäftsführer der AMD Saxony LLC: Dr. Hans-R. Deppe, Thomas McCoy
xen/arch/x86/cpu/mcheck/mce.c +88 -0
@@ -8,11 +8,13 @@#include <xen/kernel.h> #include <xen/config.h> #include <xen/smp.h> +#include <xen/errno.h> #include <asm/processor.h> #include <asm/system.h> #include "mce.h" +#include "x86_mca.h" int mce_disabled =0 ;unsigned int nr_mce_banks;
@@ -89,3 +91,89 @@ static int __init mcheck_enable(char *stcustom_param ("nomce" , mcheck_disable);custom_param ("mce" , mcheck_enable); + + + + +void x86_mcinfo_clear (struct shared_info *si) +{ +memset (&si->arch.mc_info,0 ,sizeof (struct arch_mc_info )); +x86_mcinfo_nentries (si) =0 ; +} + + +int x86_mcinfo_add (struct shared_info *si,void *mcinfo) +{ +int i; +unsigned long end1, end2; +struct mcinfo_common *mic, *mic_base, *mic_index; + + mic = (struct mcinfo_common *)mcinfo; + mic_index = mic_base =x86_mcinfo_first (si); + +/* go to first free entry */ +for (i =0 ; i <x86_mcinfo_nentries (si); i++) { + mic_index =x86_mcinfo_next (mic_index); + } + +/* check if there is enough size */ + end1 = (unsigned long )((uint8_t *)mic_base +sizeof (struct arch_mc_info )); + end2 = (unsigned long )((uint8_t *)mic_index + mic->size); + +if (end1 < end2) +return -ENOSPC ;/* No space. Can't add entry. */ + +/* there's enough space. add entry. */ +memcpy (mic_index, mic, mic->size); +x86_mcinfo_nentries (si)++; + +return 0 ; +} + +/* Dump machine check information in a format, +* mcelog can parse. This is used only when +* Dom0 does not take the notification. */ +void x86_mcinfo_dump (struct shared_info *si) +{ +struct mcinfo_common *mic; +struct mcinfo_global *mc_global; +struct mcinfo_bank *mc_bank; + +/* first print the global info */ +x86_mcinfo_lookup (mic, si,MC_TYPE_GLOBAL ); +if (mic ==NULL ) +return ; + mc_global = (struct mcinfo_global *)mic; +if (mc_global->mc_flags &MC_FLAG_UNCORRECTABLE ) { +printk (KERN_WARNING +"CPU%d: Machine Check Exception: %16" PRIx64"\n" , + mc_global->mc_coreid, mc_global->mc_gstatus); + }else { +printk (KERN_WARNING "Machine Check: The hardware reports a non " +"fatal, correctable incident occured on " +"CPU %d.\n" , + mc_global->mc_coreid); + } + +/* then the bank information */ +x86_mcinfo_lookup (mic, si,MC_TYPE_BANK );/* finds the first entry */ +do { +if (mic ==NULL ) +return ; +if (mic->type !=MC_TYPE_BANK ) +continue ; + + mc_bank = (struct mcinfo_bank *)mic; + +printk (KERN_WARNING "Bank %d: %16" PRIx64, + mc_bank->mc_bank, + mc_bank->mc_status); +if (mc_bank->mc_status & MCi_STATUS_MISCV) +printk ("[%16" PRIx64"]" , mc_bank->mc_misc); +if (mc_bank->mc_status & MCi_STATUS_ADDRV) +printk (" at %16" PRIx64, mc_bank->mc_addr); + +printk ("\n" ); + mic =x86_mcinfo_next (mic);/* next entry */ + }while (mic !=NULL ); +}
xen/arch/x86/cpu/mcheck/mce.h +4 -0
@@ -6,6 +6,10 @@ void intel_p6_mcheck_init(struct cpuinfovoid intel_p6_mcheck_init (struct cpuinfo_x86 *c);void winchip_mcheck_init (struct cpuinfo_x86 *c); +void x86_mcinfo_clear (struct shared_info *si); +int x86_mcinfo_add (struct shared_info *si,void *mcinfo); +void x86_mcinfo_dump (struct shared_info *si); +/* Call the installed machine check handler for this CPU setup. */ extern fastcall void (*machine_check_vector)(struct cpu_user_regs *, long error_code);
xen/arch/x86/cpu/mcheck/non-fatal.c +224 -16
272 lines not on this page · raw message
xen/common/event_channel.c +15 -0
@@ -539,6 +539,21 @@ void evtchn_set_pending(struct vcpu *v, } +int guest_enabled_event (struct vcpu *v,int virq) +{ +int port; + +if (unlikely (v ==NULL ) ) +return 0 ; + + port = v->virq_to_evtchn[virq]; +if ( port ==0 ) +return 0 ; + +/* virq is in use */ +return 1 ; +} +void send_guest_vcpu_virq (struct vcpu *v,int virq) {int port;
xen/include/asm-x86/event.h +6 -1
@@ -61,7 +61,12 @@ static inline void local_event_delivery_/* No arch specific virq definition now. Default to global. */ static inline int arch_virq_is_global (int virq) { -return 1 ; +switch (virq) { +case VIRQ_MCA : +return 1 ; +default : +return 1 ; + } }#endif
xen/include/public/arch-x86/xen.h +2 -0
@@ -82,6 +82,8 @@ DEFINE_XEN_GUEST_HANDLE(xen_pfn_t);#ifndef __ASSEMBLY__ +#define VIRQ_MCA VIRQ_ARCH_0/* G. (DOM0) Machine Check Architecture */ +/* * Machine Check Architecure: * structs are read-only and used to report all kinds of
xen/include/xen/event.h +3 -0
@@ -51,6 +51,9 @@ void free_xen_event_channel(void free_xen_event_channel (struct vcpu *local_vcpu,int port); +/* Query if event channel is in use by the guest */ +int guest_enabled_event (struct vcpu *v,int virq); +/* Notify remote end of a Xen-attached event channel.*/ void notify_via_xen_event_channel (int lport);
_______________________________________________
Xen-devel mailing list
[email protected]
http://lists.xensource.com/xen-devel
Quoted 8 lines · Christoph Egger
Quoted 3 lines
Oh, and is AMD-specific code really needed in
non-fatal.c? I though the MCA stuff was architectural now rather than vendor specific? If there are vendor-specific extensions then they belong in the vendor's .c file.AMD-specific is the use of the hw register code. Intel has some additional machine check MSR's containing the register set. Intel may add a structure to patch 2/3 that make use of them. Should I move the amd polling handler to
amd.c?
I think so.
-- Keir