MCA Recovery for Enterprise Server
Hi.
Now I am considering the way to apply Linux to Mission-Critical Enterprise system on IPF (Itanium Processor Family) Server. Generally, Enterprise Server requires high-reliability and high-availability, so I recognize following features as fundamentals:
- Recovery from device error
- Recovery from intermittent corrected error (ex. Single-bit ECC error)
- Structured Error logging
Aims of these are:
- Keep stable.
- Quick maintenance by early error detecting/declaring.
These features we working on are realized by functions that recover system from hardware error, block suffered device by judging from CPU/Memory/Chipset error severity. An outline is here:
a) Fault Location and Error Classification
Detect suffered unit and determine error severity on interrupted timing.
b) Recovery from device error
If error is local, disable suffered devices and block operations target to them. Else, reboot system immediately.
c) Error Logging
Structured error log helps maintenance engineer, remote maintenance system, and policed error observer.
d) Error Prediction (from intermittent corrected error)
To prevent expected error on sick component, check every corrected error and alert user to confirmed. This feature will be realized by daemon in user-land.
I am planning to offer a) to c) by the mid of March 2004, and d) by the end of 2005.
However, some of these features seem to depend on the platform implementation. So I am designing a Platform-MCA (Machine Check Abort) handler for our IPF machine.
Is there any guideline(s) to implement Platform-MCA handler?
I have found a symbol named PLATFORM_MCA_HANDLERS in /arch/ia64/kernel/mca.c, but it seems not to work.
Also, if you know any technique for debugging MCA codes, please show me the smart way.
Thanks.
------
H.Seto <[email protected]>
Quoted 3 lines
Is there any guideline(s) to implement Platform-MCA handler? I have found a symbol named
PLATFORM_MCA_HANDLERSin /arch/ia64/kernel/mca.c, > but it seems not to work.
I posted a set of three patches against 2.6.0-test5 on October 3rd. Only the first of those parts was accepted, but since then I have broken out some of the bug-fix components from the 3rd part and they were accepted by David and Linus, and are part of 2.6.0-test8.
That still leaves 666 lines of patch required to get this working for the case of MCA due to TLB fault. I've attached the remaining part to this e-mail for reference, but without any real hope that David will take such a large patch at this stage of 2.6.0 stablilization.
Quoted 2 lines
Also, if you know any technique for debugging MCA codes, please show me the smart way.
The "smart" way is (as always) to avoid putting bugs into the code, especially as this is fault handler code, which has extra
challenges to debug :-) Since this approach is very hard, you'll
need either a simulator, or an ITP to allow you to set breakpoints, examine registers and single-step. There may be pieces of the code that could be tested by writing some surrounding support code to debug them in a more user friendly environment (e.g. user mode).
David: does "ski" have any hooks for fault injection?
-Tony Luck
arch/ia64/kernel/asm-offsets.c +4 -0
@@ -12,6 +12,7 @@#include <asm-ia64/ptrace.h> #include <asm-ia64/siginfo.h> #include <asm-ia64/sigcontext.h> +#include <asm-ia64/mca.h> #include "../kernel/sigframe.h"
@@ -204,4 +205,7 @@# error "CLONE_SETTLS_BIT incorrect, please fix"#endif +BLANK (); +DEFINE (IA64_MCA_TLB_INFO_SIZE ,sizeof (struct ia64_mca_tlb_info )); + }
arch/ia64/kernel/efi.c +12 -0
@@ -30,6 +30,7 @@#include <asm/kregs.h> #include <asm/pgtable.h> #include <asm/processor.h> +#include <asm/mca.h> #define EFI_DEBUG 0
@@ -402,6 +403,9 @@int pal_code_count =0 ;u64 mask, psr;u64 vaddr; +#ifdef CONFIG_IA64_MCA +int cpu; +#endif efi_map_start =__va (ia64_boot_param->efi_memmap); efi_map_end = efi_map_start + ia64_boot_param->efi_memmap_size;
@@ -462,6 +466,14 @@ IA64_GRANULE_SHIFT);ia64_set_psr (psr);/* restore psr */ ia64_srlz_i (); + +#ifdef CONFIG_IA64_MCA + cpu =smp_processor_id (); + +/* insert this TR into our list for MCA recovery purposes */ + ia64_mca_tlb_list[cpu].pal_base=vaddr & mask; + ia64_mca_tlb_list[cpu].pal_paddr=pte_val (mk_pte_phys (md->phys_addr,PAGE_KERNEL )); +#endif } }
arch/ia64/kernel/mca.c +18 -6
@@ -78,9 +78,8 @@u64 ia64_mca_stackframe[32 ];u64 ia64_mca_bspstore[1024 ]; u64 ia64_init_stack[KERNEL_STACK_SIZE/8] __attribute__((aligned(16))); -u64 ia64_mca_sal_data_area[1356 ]; -u64 ia64_tlb_functional;u64 ia64_os_mca_recovery_successful; +u64 ia64_mca_serialize;static void ia64_mca_wakeup_ipi_wait (void );static void ia64_mca_wakeup (int cpu);static void ia64_mca_wakeup_all (void );
@@ -89,6 +88,8 @@extern void ia64_slave_init_handler (void );extern struct hw_interrupt_type irq_type_iosapic_level; +struct ia64_mca_tlb_info ia64_mca_tlb_list[NR_CPUS ]; +static struct irqaction cmci_irqaction = { .handler = ia64_mca_cmc_int_handler, .flags =SA_INTERRUPT ,
@@ -924,6 +925,9 @@void ia64_return_to_sal_check (void ) { +pal_processor_state_info_t *psp = (pal_processor_state_info_t *) + &ia64_sal_to_os_handoff_state.proc_state_param; +/* Copy over some relevant stuff from the sal_to_os_mca_handoff * so that it can be used at the time of os_mca_to_sal_handoff */
@@ -933,14 +937,22 @@ ia64_os_to_sal_handoff_state.imots_sal_check_ra = ia64_sal_to_os_handoff_state.imsto_sal_check_ra; -/* Cold Boot for uncorrectable MCA */ - ia64_os_to_sal_handoff_state.imots_os_status =IA64_MCA_COLD_BOOT ; +/* +* Did we correct the error? At the moment the only error that +* we fix is a TLB error, if any other kind of error occurred +* we must reboot. +*/ +if (psp->cc ==1 && psp->bc ==1 && psp->rc ==1 && psp->uc ==1 ) + ia64_os_to_sal_handoff_state.imots_os_status =IA64_MCA_COLD_BOOT ; +else + ia64_os_to_sal_handoff_state.imots_os_status =IA64_MCA_CORRECTED ;/* Default = tell SAL to return to same context */ ia64_os_to_sal_handoff_state.imots_context =IA64_MCA_SAME_CONTEXT ; ia64_os_to_sal_handoff_state.imots_new_min_state = (u64 *)ia64_sal_to_os_handoff_state.pal_min_state; + }/*
@@ -1318,8 +1330,8 @@void ia64_log_prt_guid (efi_guid_t *p_guid,prfunc_t prfunc) { -char out[40 ]; -printk (KERN_DEBUG "GUID = %s\n" ,efi_guid_unparse (p_guid, out)); +//char out[40]; +//printk(KERN_DEBUG "GUID = %s\n", efi_guid_unparse(p_guid, out)); } static void
arch/ia64/kernel/mca_asm.S +237 -120
440 lines not on this page · raw message
arch/ia64/mm/init.c +22 -0
@@ -33,6 +33,7 @@#include <asm/tlb.h> #include <asm/uaccess.h> #include <asm/unistd.h> +#include <asm/mca.h> DEFINE_PER_CPU(struct mmu_gather, mmu_gathers);
@@ -274,6 +275,10 @@ {unsigned long psr, pta, impl_va_bits;extern void __init tlb_init (void ); +#ifdef CONFIG_IA64_MCA +int cpu; +#endif +#ifdef CONFIG_DISABLE_VHPT # define VHPT_ENABLE_BIT 0#else
@@ -332,6 +337,23 @@ia64_set_pta (pta | (0 <<8 ) | (vmlpt_bits <<2 ) |VHPT_ENABLE_BIT );ia64_tlb_init (); + +#ifdef CONFIG_IA64_MCA + cpu =smp_processor_id (); + +/* mca handler uses cr.lid as key to pick the right entry */ + ia64_mca_tlb_list[cpu].cr_lid =ia64_getreg (_IA64_REG_CR_LID); + +/* insert this percpu data information into our list for MCA recovery purposes */ + ia64_mca_tlb_list[cpu].percpu_paddr=pte_val (mk_pte_phys (__pa (my_cpu_data),PAGE_KERNEL )); +/* Also save per-cpu tlb flush recipe for use in physical mode mca handler */ + ia64_mca_tlb_list[cpu].ptce_base=local_cpu_data->ptce_base; + ia64_mca_tlb_list[cpu].ptce_count[0 ]=local_cpu_data->ptce_count[0 ]; + ia64_mca_tlb_list[cpu].ptce_count[1 ]=local_cpu_data->ptce_count[1 ]; + ia64_mca_tlb_list[cpu].ptce_stride[0 ]=local_cpu_data->ptce_stride[0 ]; + ia64_mca_tlb_list[cpu].ptce_stride[1 ]=local_cpu_data->ptce_stride[1 ]; +#endif + }#ifdef CONFIG_VIRTUAL_MEM_MAP
include/asm-ia64/mca.h +12 -0
@@ -18,6 +18,7 @@#include <asm/param.h> #include <asm/sal.h> #include <asm/processor.h> +#include <asm/mca_asm.h> /* These are the return codes from all the IA64_MCA specific interfaces */ typedef int ia64_mca_return_code_t ;
@@ -61,6 +62,17 @@IA64_MCA_RENDEZ_CHECKIN_DONE =0x1 }; +/* the following data structure is used for TLB error recovery purposes */ +extern struct ia64_mca_tlb_info { +u64 cr_lid; +u64 percpu_paddr; +u64 ptce_base; +u32 ptce_count[2 ]; +u32 ptce_stride[2 ]; +u64 pal_paddr; +u64 pal_base; +} ia64_mca_tlb_list[NR_CPUS ]; +/* Information maintained by the MC infrastructure */ typedef struct ia64_mc_info_s { u64 imi_mca_handler;
include/asm-ia64/pgtable.h +4 -0
@@ -229,6 +229,10 @@#define mk_pte (page, pgprot) pfn_pte(page_to_pfn(page), (pgprot)) +/* This takes a physical page address that is used by the remapping functions */ +#define mk_pte_phys (physpage, pgprot) \ +({ pte_t __pte; pte_val(__pte) = physpage + pgprot_val(pgprot); __pte; }) +#define pte_modify (_pte, newprot) \ (__pte((pte_val(_pte) & _PAGE_CHG_MASK) | pgprot_val(newprot)))
Quoted 1 line
Quoted 1 line
Quoted 1 line
Quoted 1 line
Quoted 1 line · Tony
Tony> David: does "ski" have any hooks for fault injection?
Not that I know of. However, wouldn't it be possible to extend fw-emu.c to do such things? You could hit Ctrl-C at any point in time, then force a branch to a special fault-injection routine. That ought to be workable, though setting up all the info etc. is probably non-trivial.
--david
Hi Tony,
I think my words were too short or vague to bring you a real implementation
image. What I intend to do is to show my basic idea of general MCA handling on Linux.
I will explain more details to you with my next post.
Thanks.
------
H.Seto <[email protected]>