Pin the kernel stack's SLB entry
Andrew, please apply. This has been given basic testing on Power4 pSeries and iSeries machines.
At present, the SLB miss handler has to check the SLB slot it is about to use to ensure that it does not contain the SLBE for the current kernel stack - throwing out the SLBE for the kernel stack can trigger the "megabug": we have no SLBE for the stack, but we don't fault immediately because we have an ERAT entry for it. The ERAT entry is then lost due to a tlbie on another CPU during the unrecoverable section of the exception return path.
This patch implements a different approach - with this patch SLB slot 2 always (well, nearly always) contains an SLBE for the stack. This slot is never cast out by the normal SLB miss path. On context switch, an SLBE for the new stack is pinned into this slot, unless the new stack is the the bolted segment.
For iSeries we need a special workaround because there is no way of ensuring the stack SLBE is preserved an a shared processor switch. So, we still need to handle taking an SLB miss on the stack, in which case we must make sure it is loaded into slot 2, rather than using the normal round-robin.
This approach shaves a few ns off the slb miss time (on pSeries), but more importantly makes it easier to experiment with different SLB castout aporoaches without worrying about reinstating the megabug.
arch/ppc64/kernel/asm-offsets.c +1 -0
@@ -58,6 +58,7 @@ DEFINE(THREAD_FPR0, offsetof(struct thread_struct, fpr[0]));DEFINE (THREAD_FPSCR , offsetof(struct thread_struct , fpscr));DEFINE (KSP , offsetof(struct thread_struct , ksp)); +DEFINE (KSP_VSID , offsetof(struct thread_struct , ksp_vsid));#ifdef CONFIG_ALTIVEC DEFINE(THREAD_VR0, offsetof(struct thread_struct, vr[0]));
arch/ppc64/kernel/entry.S +11 -3
@@ -393,9 +393,17 @@ cmpd cr1,r6,r9 /* or is new ESID the same as current ESID? */ cror eq,4*cr1+eq,eq beq 2f /* if yes, don't slbie it */ - oris r6,r6,0x0800 /* set C (class) bit */ - slbie r6 - slbie r6 /* Workaround POWER5 < DD2.1 issue */ + oris r0,r6,0x0800 /* set C (class) bit */ + + /* Bolt in the new stack SLB entry */ + ld r7,KSP_VSID(r4) /* Get new stack's VSID */ + oris r6,r6,(SLB_ESID_V)@h + ori r6,r6,(SLB_NUM_BOLTED-1)@l + slbie r0 + slbie r0 /* Workaround POWER5 < DD2.1 issue */ + slbmte r7,r6 + isync + 2: END_FTR_SECTION_IFSET(CPU_FTR_SLB) clrrdi r7,r8,THREAD_SHIFT /* base of new stack */
arch/ppc64/kernel/process.c +10 -0
@@ -356,6 +356,16 @@ kregs = (struct pt_regs *) sp; sp -=STACK_FRAME_OVERHEAD ; p->thread.ksp = sp; +if (cur_cpu_spec->cpu_features &CPU_FTR_SLB ) { +unsigned long sp_vsid =get_kernel_vsid (sp); + + sp_vsid <<=SLB_VSID_SHIFT ; + sp_vsid |=SLB_VSID_KERNEL ; +if (cur_cpu_spec->cpu_features &CPU_FTR_16M_PAGE ) + sp_vsid |=SLB_VSID_L ; + + p->thread.ksp_vsid = sp_vsid; + }/* * The PPC64 ABI makes use of a TOC to contain function
arch/ppc64/mm/slb.c +57 -28
117 lines not on this page · raw message
arch/ppc64/mm/slb_low.S +16 -30
60 lines not on this page · raw message
include/asm-ppc64/mmu.h +1 -1
@@ -198,7 +198,7 @@#define STAB0_PHYS_ADDR (STAB0_PAGE<<PAGE_SHIFT)#define STAB0_VIRT_ADDR (KERNELBASE+STAB0_PHYS_ADDR) -#define SLB_NUM_BOLTED 2 +#define SLB_NUM_BOLTED 3#define SLB_CACHE_ENTRIES 8/* Bits in the SLB ESID word */
include/asm-ppc64/processor.h +1 -0
@@ -538,6 +538,7 @@struct thread_struct {unsigned long ksp;/* Kernel stack pointer */ +unsigned long ksp_vsid;struct pt_regs *regs;/* Pointer to saved register state */ mm_segment_t fs;/* for get_fs() validation */ double fpr[32 ];/* Complete floating point set */
Signature · 4 lines
David Gibson | For every complex problem there is a david AT gibson.dropbear.id.au | solution which is simple, neat and | wrong. http://www.ozlabs.org/people/dgibson