sparc64: sun4v TLB error power off events
PATCH v2 1/1 sparc64: sun4v TLB error power off events
From: bob picco <[email protected]>
We've witnessed a few TLB events causing the machine to power off because of prom_halt. In one case it was some nfs related area during rmmod. Another was an mmapper of /dev/mem. A more recent one is an ITLB issue with a bad pagesize which could be a hardware bug. Bugs happen but we should attempt to not power off the machine and/or hang it when possible.
This is a DTLB error from an mmapper of /dev/mem:
[root@sparcie ~]# SUN4V-DTLB: Error at TPC[fffff80100903e6c], tl 1
SUN4V-DTLB: TPC<0xfffff80100903e6c>
SUN4V-DTLB: O7[fffff801081979d0]
SUN4V-DTLB: O7<0xfffff801081979d0>
SUN4V-DTLB: vaddr[fffff80100000000] ctx[1250] pte[98000000000f0610] error[2] .
This is recent mainline for ITLB:
[ 3708.179864] SUN4V-ITLB: TPC<0xfffffc010071cefc> [ 3708.188866] SUN4V-ITLB: O7[fffffc010071cee8] [ 3708.197377] SUN4V-ITLB: O7<0xfffffc010071cee8> [ 3708.206539] SUN4V-ITLB: vaddr[e0003] ctx[1a3c] pte[2900000dcc800eeb] error[4]
.
Normally sun4v_itlb_error_report() and sun4v_dtlb_error_report() would call prom_halt() and drop us to OF command prompt "ok". This isn't the case for LDOMs and the machine powers off.
For the HV reported error of HV_ENORADDR for HV HV_MMU_MAP_ADDR_TRAP we cause a SIGBUS error by qualifying it within do_sparc64_fault() for fault code mask of FAULT_CODE_BAD_RA. This is done when trap level (%tl) is less or equal one("1"). Otherwise, for %tl > 1, we proceed eventually to die_if_kernel().
The logic of this patch was partially inspired by David Miller's feedback.
Power off of large sparc64 machines is painful. Plus die_if_kernel provides more context. A reset sequence isn't a brief period on large sparc64 but better than power-off/power-on sequence.
---
V2: bus error for cases at %tl <= 1.
for %tl > 1 always call die_if_kernel().
arch/sparc/include/asm/thread_info_64.h | 1 + arch/sparc/kernel/sun4v_tlb_miss.S | 35 ++++++++++++++++++------------ arch/sparc/kernel/traps_64.c | 15 ++++++++----- arch/sparc/mm/fault_64.c | 3 ++ 4 files changed, 34 insertions(+), 20 deletions(-)
arch/sparc/include/asm/thread_info_64.h +1 -0
@@ -102,6 +102,7 @@ struct thread_info {#define FAULT_CODE_ITLB 0x04/* Miss happened in I-TLB */ #define FAULT_CODE_WINFIXUP 0x08/* Miss happened during spill/fill */ #define FAULT_CODE_BLKCOMMIT 0x10/* Use blk-commit ASI in copy_page */ +#define FAULT_CODE_BAD_RA 0x20/* Bad RA for sun4v */ #if PAGE_SHIFT = 13#define THREAD_SIZE (2*PAGE_SIZE)
arch/sparc/kernel/sun4v_tlb_miss.S +21 -14
@@ -195,6 +195,11 @@ ldx [%g2 + TRAP_PER_CPU_PGD_PADDR], %g7 sun4v_itlb_error: + rdpr %tl, %g1 + cmp %g1, 1 + ble,pt %icc, sun4v_bad_ra + or %g0, FAULT_CODE_BAD_RA | FAULT_CODE_ITLB, %g1 + sethi %hi(sun4v_err_itlb_vaddr), %g1 stx %g4, [%g1 + %lo(sun4v_err_itlb_vaddr)] sethi %hi(sun4v_err_itlb_ctx), %g1
@@ -206,15 +211,10 @@ sethi %hi(sun4v_err_itlb_error), %g1 stx %o0, [%g1 + %lo(sun4v_err_itlb_error)] + sethi %hi(1f), %g7 rdpr %tl, %g4 - cmp %g4, 1 - ble,pt %icc, 1f - sethi %hi(2f), %g7 ba,pt %xcc, etraptl1 - or %g7, %lo(2f), %g7 - -1: ba,pt %xcc, etrap -2: or %g7, %lo(2b), %g7 +1: or %g7, %lo(1f), %g7 mov %l4, %o1 call sun4v_itlb_error_report add %sp, PTREGS_OFF, %o0
@@ -222,6 +222,11 @@ /* NOTREACHED */ sun4v_dtlb_error: + rdpr %tl, %g1 + cmp %g1, 1 + ble,pt %icc, sun4v_bad_ra + or %g0, FAULT_CODE_BAD_RA | FAULT_CODE_DTLB, %g1 + sethi %hi(sun4v_err_dtlb_vaddr), %g1 stx %g4, [%g1 + %lo(sun4v_err_dtlb_vaddr)] sethi %hi(sun4v_err_dtlb_ctx), %g1
@@ -233,21 +238,23 @@ sethi %hi(sun4v_err_dtlb_error), %g1 stx %o0, [%g1 + %lo(sun4v_err_dtlb_error)] + sethi %hi(1f), %g7 rdpr %tl, %g4 - cmp %g4, 1 - ble,pt %icc, 1f - sethi %hi(2f), %g7 ba,pt %xcc, etraptl1 - or %g7, %lo(2f), %g7 - -1: ba,pt %xcc, etrap -2: or %g7, %lo(2b), %g7 +1: or %g7, %lo(1f), %g7 mov %l4, %o1 call sun4v_dtlb_error_report add %sp, PTREGS_OFF, %o0 /* NOTREACHED */ +sun4v_bad_ra: + or %g0, %g4, %g5 + ba,pt %xcc, sparc64_realfault_common + or %g1, %g0, %g4 + + /* NOTREACHED */ + /* Instruction Access Exception, tl0. */ sun4v_iacc: ldxa [%g0] ASI_SCRATCHPAD, %g2
arch/sparc/kernel/traps_64.c +9 -6
@@ -2104,6 +2104,11 @@ void sun4v_nonresum_overflow(struct pt_regs *regs)atomic_inc (&sun4v_nonresum_oflow_cnt); } +static void sun4v_tlb_error (struct pt_regs *regs) +{ +die_if_kernel ("TLB/TSB error" , regs); +} +unsigned long sun4v_err_itlb_vaddr;unsigned long sun4v_err_itlb_ctx;unsigned long sun4v_err_itlb_pte;
@@ -2111,8 +2116,7 @@ unsigned long sun4v_err_itlb_error;void sun4v_itlb_error_report (struct pt_regs *regs,int tl) { -if (tl >1 ) -dump_tl1_traplog ((struct tl1_traplog *)(regs +1 )); +dump_tl1_traplog ((struct tl1_traplog *)(regs +1 ));printk (KERN_EMERG "SUN4V-ITLB: Error at TPC[%lx], tl %d\n" , regs->tpc, tl);
@@ -2125,7 +2129,7 @@ void sun4v_itlb_error_report(struct pt_regs *regs, int tl) sun4v_err_itlb_vaddr, sun4v_err_itlb_ctx, sun4v_err_itlb_pte, sun4v_err_itlb_error); -prom_halt (); +sun4v_tlb_error (regs); }unsigned long sun4v_err_dtlb_vaddr;
@@ -2135,8 +2139,7 @@ unsigned long sun4v_err_dtlb_error;void sun4v_dtlb_error_report (struct pt_regs *regs,int tl) { -if (tl >1 ) -dump_tl1_traplog ((struct tl1_traplog *)(regs +1 )); +dump_tl1_traplog ((struct tl1_traplog *)(regs +1 ));printk (KERN_EMERG "SUN4V-DTLB: Error at TPC[%lx], tl %d\n" , regs->tpc, tl);
@@ -2149,7 +2152,7 @@ void sun4v_dtlb_error_report(struct pt_regs *regs, int tl) sun4v_err_dtlb_vaddr, sun4v_err_dtlb_ctx, sun4v_err_dtlb_pte, sun4v_err_dtlb_error); -prom_halt (); +sun4v_tlb_error (regs); }void hypervisor_tlbop_error (unsigned long err,unsigned long op)
arch/sparc/mm/fault_64.c +3 -0
@@ -346,6 +346,9 @@ asmlinkage void __kprobes do_sparc64_fault(struct pt_regs *regs)down_read (&mm->mmap_sem); } +if (fault_code &FAULT_CODE_BAD_RA ) + goto do_sigbus; + vma =find_vma (mm, address);if (!vma) goto bad_area;
Signature · 1 line
1.7.1
David Miller
maintainerFrom: Bob Picco <[email protected]>
Date: Tue, 16 Sep 2014 09:26:47 -0400
I just looked into this a bit while reviewing this patch.
Quoted 2 lines
A more recent one is an ITLB issue with a bad pagesize which could be a hardware bug.
The error is signalled solely by the hypervisor. Let's look at the PTE it doesn't like.
Quoted 5 lines
This is recent mainline for ITLB:
[ 3708.179864] SUN4V-ITLB: TPC<0xfffffc010071cefc> [ 3708.188866] SUN4V-ITLB: O7[fffffc010071cee8] [ 3708.197377] SUN4V-ITLB: O7<0xfffffc010071cee8> [ 3708.206539] SUN4V-ITLB: vaddr[e0003] ctx[1a3c] pte[2900000dcc800eeb] error[4]
Indeed bad pagesize is signalled. The page size bits are in the low 3 bits of the PTE, which here is 0x3 which should encode a 4MB page.
However I notice that 0x8 is set, which is a reserved bit. That shouldn't happen, and could be what the hypervisor really doesn't like.
David Miller
maintainerFrom: David Miller <[email protected]>
Date: Tue, 16 Sep 2014 17:47:25 -0400 (EDT)
Quoted 3 lines
However I notice that
0x8is set, which is a reserved bit. That shouldn't happen, and could be what the hypervisor really doesn't like.
I just confirmed that this is why the hypervisor throws an error. It actually inspects all of 0xf in the PTE as a size field (rather than just 0x7) via the TTE_SIZE() macro (in include/sun4v/mmu.h), which uses TTE_SZ_MASK (also in include/sun4v/mmu.h), which is defined as 0xf.
It then makes sure that only CPU supported sizes are set in the PTE by using the TTE_VALIDSIZEARRAY (f.e. in greatlakes/ontario/include/mmu.h or greatlakes/huron/include/mmu.h) bitmask, which is different for each platform hypervisor build.
Anyways, we have to figure out why 0x8 is set in the PTE that we try to load into ITLB. I haven't found any smoking guns yet.
David Miller
maintainerFrom: David Miller <[email protected]>
Date: Tue, 16 Sep 2014 20:16:51 -0400 (EDT)
Quoted 2 lines
Anyways, we have to figure out why
0x8is set in the PTE that we try to load into ITLB. I haven't found any smoking guns yet.
Looking more closely, the PTE 0x2900000dcc800eeb doesn't even have the valid bit set.
Non-valid PTEs should not even get into the TSB, and non-valid PTEs found via page table lookup should vector us to the full fault path.
There used to be a bug where we'd put non-valid TTEs into the TSB but that was fixed by commit:
commit 18f38132528c3e603c66ea464727b29e9bbcb91b
Author: David S. Miller <[email protected]> Date: Mon Aug 4 16:34:01 2014 -0700
sparc64: Do not insert non-valid PTEs into the TSB hash table.
Do you know if the kernels that triggered those ITLB BADRA errors had that fix or not?
David Miller
maintainerFrom: Bob Picco <[email protected]>
Date: Tue, 16 Sep 2014 09:26:47 -0400
Quoted 8 lines
Normally
sun4v_itlb_error_report()andsun4v_dtlb_error_report()would callprom_halt()and drop us to OF command prompt "ok". This isn't the case for LDOMs and the machine powers off.For the HV reported error of
HV_ENORADDRfor HVHV_MMU_MAP_ADDR_TRAPwe cause a SIGBUS error by qualifying it withindo_sparc64_fault()for fault code mask ofFAULT_CODE_BAD_RA. This is done when trap level (%tl) is less or equalone("1"). Otherwise, for %tl > 1, we proceed eventually todie_if_kernel().
This patch is great, applied, thanks Bob.
I think I'd like to request a follow-on patch to this if you don't mind.
If %tl > 1, the only situation we can possibly be in is to be taking a TLB miss during a window spill/fill trap, in which case we should up-trap back up through the spill/fill handler and into the long path fault code via winfix_trampoline.
Then you can kill all of the {i,d}tlb error code paths.
Thanks!
David Miller wrote: [Tue Sep 16 2014, 05:47:25PM EDT]
Quoted 4 lines
From: Bob Picco <[email protected]>
Date: Tue, 16 Sep 2014 09:26:47 -0400I just looked into this a bit while reviewing this patch.
Okay.
Quoted 6 lines
Quoted 3 lines
A more recent one is an ITLB issue with a bad pagesize which could be a hardware bug.
The error is signalled solely by the hypervisor. Let's look at the PTE it doesn't like.
I agree and forgot about this in the context of bus error.
Note, I've seen this issue once or twice. Perhaps only on my ~P0 T4-2. I couldn't find the issue in my T5-2 log.
Quoted 9 lines
Quoted 6 lines
This is recent mainline for ITLB:
[ 3708.179864] SUN4V-ITLB: TPC<0xfffffc010071cefc> [ 3708.188866] SUN4V-ITLB: O7[fffffc010071cee8] [ 3708.197377] SUN4V-ITLB: O7<0xfffffc010071cee8> [ 3708.206539] SUN4V-ITLB: vaddr[e0003] ctx[1a3c] pte[2900000dcc800eeb] error[4]Indeed bad pagesize is signalled. The page size bits are in the low 3 bits of the PTE, which here is
0x3which should encode a 4MB page.
Yes.
Quoted 4 lines
However I notice that
0x8is set, which is a reserved bit. That shouldn't happen, and could be what the hypervisor really doesn't like.
Also _PAGE_VALID isn't set.
Quoted 4 lines
--
To unsubscribe from this list: send the line "unsubscribe sparclinux" in the body of a message to [email protected]
More majordomo info at http://vger.kernel.org/majordomo-info.html
David Miller wrote: [Tue Sep 16 2014, 08:35:15PM EDT]
Quoted 8 lines
From: David Miller <[email protected]>
Date: Tue, 16 Sep 2014 20:16:51 -0400 (EDT)Quoted 2 lines
Anyways, we have to figure out why
0x8is set in the PTE that we try to load into ITLB. I haven't found any smoking guns yet.Looking more closely, the PTE
0x2900000dcc800eebdoesn't even have the valid bit set.
Yes.
Quoted 7 lines
Non-valid PTEs should not even get into the TSB, and non-valid PTEs found via page table lookup should vector us to the full fault path.
There used to be a bug where we'd put non-valid TTEs into the TSB but that was fixed by commit:
Quoted 8 lines
commit
18f38132528c3e603c66ea464727b29e9bbcb91bAuthor: David S. Miller <[email protected]> Date: Mon Aug 4 16:34:01 2014 -0700sparc64: Do not insert non-valid PTEs into the TSB hash table.
Do you know if the kernels that triggered those ITLB BADRA errors had that fix or not?
I doubt it. I just checked my log for T4-2 and none of the kernels appears close to this:
[bpicco@zareason linus.git]$ git describe --contains 18f38132528c3e603c66ea464727b29e9bbcb91b v3.17-rc1~105^2~1^2~6
when the issue was revealed.
David Miller wrote: [Tue Sep 16 2014, 08:49:43PM EDT]
Quoted 12 lines
From: Bob Picco <[email protected]>
Date: Tue, 16 Sep 2014 09:26:47 -0400Quoted 8 lines
Normally
sun4v_itlb_error_report()andsun4v_dtlb_error_report()would callprom_halt()and drop us to OF command prompt "ok". This isn't the case for LDOMs and the machine powers off.For the HV reported error of
HV_ENORADDRfor HVHV_MMU_MAP_ADDR_TRAPwe cause a SIGBUS error by qualifying it withindo_sparc64_fault()for fault code mask ofFAULT_CODE_BAD_RA. This is done when trap level (%tl) is less or equalone("1"). Otherwise, for %tl > 1, we proceed eventually todie_if_kernel().
Quoted 4 lines
This patch is great, applied, thanks Bob.
I think I'd like to request a follow-on patch to this if you don't mind.
Fine by me.
Quoted 5 lines
If %tl > 1, the only situation we can possibly be in is to be taking a TLB miss during a window spill/fill trap, in which case we should up-trap back up through the spill/fill handler and into the long path fault code via
winfix_trampoline.
Yes I agree.
Quoted 2 lines
Then you can kill all of the {i,d}tlb error code paths.
Let me look at this.
Quoted 2 lines
Thanks!
you're welcome and thanx!
David Miller
maintainerFrom: Bob Picco <[email protected]>
Date: Wed, 17 Sep 2014 08:24:23 -0400
Quoted 10 lines
David Miller wrote: [Tue Sep 16 2014, 08:35:15PM EDT]
Quoted 9 lines
From: David Miller <[email protected]>
Date: Tue, 16 Sep 2014 20:16:51 -0400 (EDT)commit
18f38132528c3e603c66ea464727b29e9bbcb91bAuthor: David S. Miller <[email protected]> Date: Mon Aug 4 16:34:01 2014 -0700sparc64: Do not insert non-valid PTEs into the TSB hash table.
Quoted 7 lines
Quoted 2 lines
Do you know if the kernels that triggered those ITLB BADRA errors had that fix or not?
I doubt it. I just checked my log for T4-2 and none of the kernels appears close to this:
[bpicco@zareason linus.git]$ git describe --contains18f38132528c3e603c66ea464727b29e9bbcb91bv3.17-rc1~105^2~1^2~6
when the issue was revealed.
So I believe this bug should be fixed by the above commit, let me know if you see this ever again.