Enable CONFIG_VMAP_STACK on PPC32
Subsystems (2)
v4 cover Enable CONFIG_VMAP_STACK on PPC32
The purpose of this serie is to enable CONFIG_VMAP_STACK on PPC32.
rfc v1: initial support on 8xx
rfc v2: added stack overflow detection.
v3:
- Stack overflow detection works, tested with LKDTM STACK_EXHAUST test
- Support for book3s32 added
v4:
- Fixed build failure with CONFIG_KVM (patch 14)
- Fixed boot failure with pmac32_defconfig+VMAP_STACK+KVM (new patch 15)
- Fixed crash with altivec and fpu (patch 16)
Christophe Leroy (16):
powerpc/32: replace MTMSRD() by mtmsr
powerpc/32: Add EXCEPTION_PROLOG_0 in head_32.h powerpc/32: save DEAR/DAR before calling handle_page_fault powerpc/32: move MSR_PR test into EXCEPTION_PROLOG_0 powerpc/32: add a macro to get and/or save DAR and DSISR on stack. powerpc/32: prepare for CONFIG_VMAP_STACK
powerpc: align stack to 2 * THREAD_SIZE with VMAP_STACK powerpc/32: Add early stack overflow detection with VMAP stack. powerpc/8xx: Use alternative scratch registers in DTLB miss handler powerpc/8xx: drop exception entries for non-existing exceptions powerpc/8xx: move DataStoreTLBMiss perf handler powerpc/8xx: split breakpoint exception
powerpc/8xx: Enable CONFIG_VMAP_STACK
powerpc/32s: reorganise DSI handler.
powerpc/32s: avoid crossing page boundary while changing SRR0/1. powerpc/32s: Activate CONFIG_VMAP_STACK
arch/powerpc/include/asm/irq.h | 1 + arch/powerpc/include/asm/processor.h | 6 ++ arch/powerpc/include/asm/thread_info.h | 18 ++++ arch/powerpc/kernel/asm-offsets.c | 6 ++ arch/powerpc/kernel/entry_32.S | 55 ++++++++-- arch/powerpc/kernel/fpu.S | 3 + arch/powerpc/kernel/head_32.S | 65 ++++++++---- arch/powerpc/kernel/head_32.h | 129 ++++++++++++++++++++--- arch/powerpc/kernel/head_40x.S | 2 + arch/powerpc/kernel/head_8xx.S | 186 +++++++++++++++------------------ arch/powerpc/kernel/head_booke.h | 2 + arch/powerpc/kernel/head_fsl_booke.S | 1 + arch/powerpc/kernel/irq.c | 1 + arch/powerpc/kernel/setup_32.c | 3 +- arch/powerpc/kernel/setup_64.c | 2 +- arch/powerpc/kernel/traps.c | 15 ++- arch/powerpc/kernel/vector.S | 3 + arch/powerpc/kernel/vmlinux.lds.S | 2 +- arch/powerpc/mm/book3s32/hash_low.S | 46 +++++--- arch/powerpc/mm/book3s32/mmu.c | 9 +- arch/powerpc/perf/8xx-pmu.c | 12 ++- arch/powerpc/platforms/Kconfig.cputype | 3 + 22 files changed, 393 insertions(+), 177 deletions(-)
Signature · 1 line
2.13.3
PATCH v4 1/16 powerpc/32: replace MTMSRD() by mtmsr
On PPC32, MTMSRD() is simply defined as mtmsr.
Replace MTMSRD(reg) by mtmsr reg in files dedicated to PPC32, this makes the code less obscure.
--- arch/powerpc/kernel/entry_32.S | 18 +++++++++--------- arch/powerpc/kernel/head_32.h | 4 ++-- 2 files changed, 11 insertions(+), 11 deletions(-)
arch/powerpc/kernel/entry_32.S +9 -9
@@ -397,7 +397,7 @@ ret_from_syscall: LOAD_REG_IMMEDIATE(r10,MSR_KERNEL) /* doesn't include MSR_EE */ /* Note: We don't bother telling lockdep about it */ SYNC - MTMSRD(r10) + mtmsr r10 lwz r9,TI_FLAGS(r2) li r8,-MAX_ERRNO andi. r0,r9,(_TIF_SYSCALL_DOTRACE|_TIF_SINGLESTEP|_TIF_USER_WORK_MASK|_TIF_PERSYSCALL_MASK)
@@ -554,7 +554,7 @@ syscall_exit_work: */ ori r10,r10,MSR_EE SYNC - MTMSRD(r10) + mtmsr r10 /* Save NVGPRS if they're not saved already */ lwz r4,_TRAP(r1)
@@ -697,7 +697,7 @@ END_FTR_SECTION_IFSET(CPU_FTR_SPE) and. r0,r0,r11 /* FP or altivec or SPE enabled? */ beq+ 1f andc r11,r11,r0 - MTMSRD(r11) + mtmsr r11 isync 1: stw r11,_MSR(r1) mfcr r10
@@ -831,7 +831,7 @@ ret_from_except: /* Note: We don't bother telling lockdep about it */ LOAD_REG_IMMEDIATE(r10,MSR_KERNEL) SYNC /* Some chip revs have problems here... */ - MTMSRD(r10) /* disable interrupts */ + mtmsr r10 /* disable interrupts */ lwz r3,_MSR(r1) /* Returning to user mode? */ andi. r0,r3,MSR_PR
@@ -998,7 +998,7 @@ END_FTR_SECTION_IFSET(CPU_FTR_NEED_PAIRED_STWCX) */ LOAD_REG_IMMEDIATE(r10,MSR_KERNEL & ~MSR_RI) SYNC - MTMSRD(r10) /* clear the RI bit */ + mtmsr r10 /* clear the RI bit */ .globl exc_exit_restart exc_exit_restart: lwz r12,_NIP(r1)
@@ -1234,7 +1234,7 @@ do_resched: /* r10 contains MSR_KERNEL here */ #endif ori r10,r10,MSR_EE SYNC - MTMSRD(r10) /* hard-enable interrupts */ + mtmsr r10 /* hard-enable interrupts */ bl schedule recheck: /* Note: And we don't tell it we are disabling them again
@@ -1243,7 +1243,7 @@ recheck: */ LOAD_REG_IMMEDIATE(r10,MSR_KERNEL) SYNC - MTMSRD(r10) /* disable interrupts */ + mtmsr r10 /* disable interrupts */ lwz r9,TI_FLAGS(r2) andi. r0,r9,_TIF_NEED_RESCHED bne- do_resched
@@ -1252,7 +1252,7 @@ recheck: do_user_signal: /* r10 contains MSR_KERNEL here */ ori r10,r10,MSR_EE SYNC - MTMSRD(r10) /* hard-enable interrupts */ + mtmsr r10 /* hard-enable interrupts */ /* save r13-r31 in the exception frame, if not already done */ lwz r3,_TRAP(r1) andi. r0,r3,1
@@ -1341,7 +1341,7 @@ _GLOBAL(enter_rtas) stw r9,8(r1) LOAD_REG_IMMEDIATE(r0,MSR_KERNEL) SYNC /* disable interrupts so SRR0/1 */ - MTMSRD(r0) /* don't get trashed */ + mtmsr r0 /* don't get trashed */ li r9,MSR_KERNEL & ~(MSR_IR|MSR_DR) mtlr r6 stw r7, THREAD + RTAS_SP(r2)
arch/powerpc/kernel/head_32.h +2 -2
@@ -50,7 +50,7 @@ rlwinm r9,r9,0,14,12 /* clear MSR_WE (necessary?) */ #else li r10,MSR_KERNEL & ~(MSR_IR|MSR_DR) /* can take exceptions */ - MTMSRD(r10) /* (except for mach check in rtas) */ + mtmsr r10 /* (except for mach check in rtas) */ #endif stw r0,GPR0(r11) lis r10,STACK_FRAME_REGS_MARKER@ha /* exception frame marker */
@@ -80,7 +80,7 @@ rlwinm r9,r9,0,14,12 /* clear MSR_WE (necessary?) */ #else LOAD_REG_IMMEDIATE(r10, MSR_KERNEL & ~(MSR_IR|MSR_DR)) /* can take exceptions */ - MTMSRD(r10) /* (except for mach check in rtas) */ + mtmsr r10 /* (except for mach check in rtas) */ #endif lis r10,STACK_FRAME_REGS_MARKER@ha /* exception frame marker */ stw r2,GPR2(r11)
Signature · 1 line
2.13.3
PATCH v4 2/16 powerpc/32: Add EXCEPTION_PROLOG_0 in head_32.h
In reply to Christophe LeroyThis patch creates a macro for the very first part of exception prolog, this will help when implementing CONFIG_VMAP_STACK
--- arch/powerpc/kernel/head_32.S | 4 +--- arch/powerpc/kernel/head_32.h | 9 ++++++--- arch/powerpc/kernel/head_8xx.S | 9 ++------- 3 files changed, 9 insertions(+), 13 deletions(-)
arch/powerpc/kernel/head_32.S +1 -3
@@ -272,9 +272,7 @@ __secondary_hold_acknowledge: */ . = 0x200 DO_KVM 0x200 - mtspr SPRN_SPRG_SCRATCH0,r10 - mtspr SPRN_SPRG_SCRATCH1,r11 - mfcr r10 + EXCEPTION_PROLOG_0 #ifdef CONFIG_PPC_CHRP mfspr r11, SPRN_SPRG_THREAD lwz r11, RTAS_SP(r11)
arch/powerpc/kernel/head_32.h +6 -3
@@ -10,13 +10,16 @@* We assume sprg3 has the physical address of the current * task's thread_struct. */ - .macro EXCEPTION_PROLOG +EXCEPTION_PROLOG_0 + EXCEPTION_PROLOG_1 + EXCEPTION_PROLOG_2 +.endm + +.macro EXCEPTION_PROLOG_0mtspr SPRN_SPRG_SCRATCH0 ,r10 mtspr SPRN_SPRG_SCRATCH1,r11 mfcr r10 - EXCEPTION_PROLOG_1 - EXCEPTION_PROLOG_2 .endm .macro EXCEPTION_PROLOG_1
arch/powerpc/kernel/head_8xx.S +2 -7
@@ -494,10 +494,7 @@ InstructionTLBError: */ . = 0x1400 DataTLBError: - mtspr SPRN_SPRG_SCRATCH0, r10 - mtspr SPRN_SPRG_SCRATCH1, r11 - mfcr r10 - + EXCEPTION_PROLOG_0 mfspr r11, SPRN_DAR cmpwi cr0, r11, RPN_PATTERN beq- FixupDAR /* must be a buggy dcbX, icbi insn. */
@@ -530,9 +527,7 @@ DARFixed:/* Return from dcbx instruction bug workaround */ */ . = 0x1c00 DataBreakpoint: - mtspr SPRN_SPRG_SCRATCH0, r10 - mtspr SPRN_SPRG_SCRATCH1, r11 - mfcr r10 + EXCEPTION_PROLOG_0 mfspr r11, SPRN_SRR0 cmplwi cr0, r11, (.Ldtlbie - PAGE_OFFSET)@l cmplwi cr7, r11, (.Litlbie - PAGE_OFFSET)@l
Signature · 1 line
2.13.3
PATCH v4 3/16 powerpc/32: save DEAR/DAR before calling handle_page_fault
In reply to Christophe Leroyhandle_page_fault() is the only function that save DAR/DEAR itself.
Save DAR/DEAR before calling handle_page_fault() to prepare for VMAP stack which will require to save even before.
--- arch/powerpc/kernel/entry_32.S | 1 - arch/powerpc/kernel/head_32.S | 2 ++ arch/powerpc/kernel/head_40x.S | 2 ++ arch/powerpc/kernel/head_8xx.S | 2 ++ arch/powerpc/kernel/head_booke.h | 2 ++ arch/powerpc/kernel/head_fsl_booke.S | 1 + 6 files changed, 9 insertions(+), 1 deletion(-)
arch/powerpc/kernel/entry_32.S +0 -1
@@ -621,7 +621,6 @@ ppc_swapcontext: */ .globl handle_page_fault handle_page_fault: - stw r4,_DAR(r1) addi r3,r1,STACK_FRAME_OVERHEAD #ifdef CONFIG_PPC_BOOK3S_32 andis. r0,r5,DSISR_DABRMATCH@h
arch/powerpc/kernel/head_32.S +2 -0
@@ -310,6 +310,7 @@ BEGIN_MMU_FTR_SECTION END_MMU_FTR_SECTION_IFSET(MMU_FTR_HPTE_TABLE) 1: lwz r5,_DSISR(r11) /* get DSISR value */ mfspr r4,SPRN_DAR + stw r4, _DAR(r11) EXC_XFER_LITE(0x300, handle_page_fault)
@@ -327,6 +328,7 @@ BEGIN_MMU_FTR_SECTION END_MMU_FTR_SECTION_IFSET(MMU_FTR_HPTE_TABLE) 1: mr r4,r12 andis. r5,r9,DSISR_SRR1_MATCH_32S@h /* Filter relevant SRR1 bits */ + stw r4, _DAR(r11) EXC_XFER_LITE(0x400, handle_page_fault) /* External interrupt */
arch/powerpc/kernel/head_40x.S +2 -0
@@ -313,6 +313,7 @@ _ENTRY(saved_ksp_limit) START_EXCEPTION(0x0400, InstructionAccess) EXCEPTION_PROLOG mr r4,r12 /* Pass SRR0 as arg2 */ + stw r4, _DEAR(r11) li r5,0 /* Pass zero as arg3 */ EXC_XFER_LITE(0x400, handle_page_fault)
@@ -676,6 +677,7 @@ DataAccess: mfspr r5,SPRN_ESR /* Grab the ESR, save it, pass arg3 */ stw r5,_ESR(r11) mfspr r4,SPRN_DEAR /* Grab the DEAR, save it, pass arg2 */ + stw r4, _DEAR(r11) EXC_XFER_LITE(0x300, handle_page_fault) /* Other PowerPC processors, namely those derived from the 6xx-series
arch/powerpc/kernel/head_8xx.S +2 -0
@@ -486,6 +486,7 @@ InstructionTLBError: tlbie r4 /* 0x400 is InstructionAccess exception, needed by bad_page_fault() */ .Litlbie: + stw r4, _DAR(r11) EXC_XFER_LITE(0x400, handle_page_fault) /* This is the data TLB error on the MPC8xx. This could be due to
@@ -504,6 +505,7 @@ DARFixed:/* Return from dcbx instruction bug workaround */ mfspr r5,SPRN_DSISR stw r5,_DSISR(r11) mfspr r4,SPRN_DAR + stw r4, _DAR(r11) andis. r10,r5,DSISR_NOHPTE@h beq+ .Ldtlbie tlbie r4
arch/powerpc/kernel/head_booke.h +2 -0
@@ -467,6 +467,7 @@ ALT_FTR_SECTION_END_IFSET(CPU_FTR_EMB_HV)mfspr r5,SPRN_ESR ;/* Grab the ESR and save it */ \stw r5,_ESR (r11 ); \mfspr r4,SPRN_DEAR ;/* Grab the DEAR */ \ +stw r4,_DEAR (r11 ); \ EXC_XFER_LITE(0x0300, handle_page_fault)#define INSTRUCTION_STORAGE_EXCEPTION \
@@ -475,6 +476,7 @@ ALT_FTR_SECTION_END_IFSET(CPU_FTR_EMB_HV)mfspr r5,SPRN_ESR ;/* Grab the ESR and save it */ \stw r5,_ESR (r11 ); \mr r4,r12;/* Pass SRR0 as arg2 */ \ +stw r4,_DEAR (r11 ); \ li r5,0; /* Pass zero as arg3 */ \EXC_XFER_LITE (0x0400 , handle_page_fault)
arch/powerpc/kernel/head_fsl_booke.S +1 -0
@@ -378,6 +378,7 @@ interrupt_base: mfspr r4,SPRN_DEAR /* Grab the DEAR, save it, pass arg2 */ andis. r10,r5,(ESR_ILK|ESR_DLK)@h bne 1f + stw r4, _DEAR(r11) EXC_XFER_LITE(0x0300, handle_page_fault) 1: addi r3,r1,STACK_FRAME_OVERHEAD
Signature · 1 line
2.13.3
PATCH v4 4/16 powerpc/32: move MSR_PR test into EXCEPTION_PROLOG_0
In reply to Christophe LeroyIn order to simplify VMAP stack implementation, move MSR_PR test into EXCEPTION_PROLOG_0.
This requires to not modify cr0 between EXCEPTION_PROLOG_0 and EXCEPTION_PROLOG_1.
--- arch/powerpc/kernel/head_32.h | 4 ++-- arch/powerpc/kernel/head_8xx.S | 39 ++++++++++++++++++++------------------- 2 files changed, 22 insertions(+), 21 deletions(-)
arch/powerpc/kernel/head_32.h +2 -2
@@ -19,12 +19,12 @@ .macro EXCEPTION_PROLOG_0mtspr SPRN_SPRG_SCRATCH0 ,r10 mtspr SPRN_SPRG_SCRATCH1,r11 + mfspr r11, SPRN_SRR1 /* check whether user or kernel */ mfcr r10 + andi. r11, r11, MSR_PR .endm .macro EXCEPTION_PROLOG_1 - mfspr r11,SPRN_SRR1 /* check whether user or kernel */ - andi. r11,r11,MSR_PR tophys(r11,r1) /* use tophys(r1) if kernel */ beq 1f mfspr r11,SPRN_SPRG_THREAD
arch/powerpc/kernel/head_8xx.S +20 -19
@@ -497,8 +497,8 @@ InstructionTLBError: DataTLBError: EXCEPTION_PROLOG_0 mfspr r11, SPRN_DAR - cmpwi cr0, r11, RPN_PATTERN - beq- FixupDAR /* must be a buggy dcbX, icbi insn. */ + cmpwi cr1, r11, RPN_PATTERN + beq- cr1, FixupDAR /* must be a buggy dcbX, icbi insn. */ DARFixed:/* Return from dcbx instruction bug workaround */ EXCEPTION_PROLOG_1 EXCEPTION_PROLOG_2
@@ -531,9 +531,9 @@ DARFixed:/* Return from dcbx instruction bug workaround */ DataBreakpoint: EXCEPTION_PROLOG_0 mfspr r11, SPRN_SRR0 - cmplwi cr0, r11, (.Ldtlbie - PAGE_OFFSET)@l + cmplwi cr1, r11, (.Ldtlbie - PAGE_OFFSET)@l cmplwi cr7, r11, (.Litlbie - PAGE_OFFSET)@l - beq- cr0, 11f + beq- cr1, 11f beq- cr7, 11f EXCEPTION_PROLOG_1 EXCEPTION_PROLOG_2
@@ -578,9 +578,9 @@ FixupDAR:/* Entry point for dcbx workaround. */ mfspr r10, SPRN_SRR0 mtspr SPRN_MD_EPN, r10 rlwinm r11, r10, 16, 0xfff8 - cmpli cr0, r11, PAGE_OFFSET@h + cmpli cr1, r11, PAGE_OFFSET@h mfspr r11, SPRN_M_TWB /* Get level 1 table */ - blt+ 3f + blt+ cr1, 3f rlwinm r11, r10, 16, 0xfff8 0: cmpli cr7, r11, (PAGE_OFFSET + 0x1800000)@h
@@ -595,7 +595,7 @@ FixupDAR:/* Entry point for dcbx workaround. */ 3: lwz r11, (swapper_pg_dir-PAGE_OFFSET)@l(r11) /* Get the level 1 entry */ mtspr SPRN_MD_TWC, r11 - mtcr r11 + mtcrf 0x01, r11 mfspr r11, SPRN_MD_TWC lwz r11, 0(r11) /* Get the pte */ bt 28,200f /* bit 28 = Large page (8M) */
@@ -608,16 +608,16 @@ FixupDAR:/* Entry point for dcbx workaround. */ * no need to include them here */ xoris r10, r11, 0x7c00 /* check if major OP code is 31 */ rlwinm r10, r10, 0, 21, 5 - cmpwi cr0, r10, 2028 /* Is dcbz? */ - beq+ 142f - cmpwi cr0, r10, 940 /* Is dcbi? */ - beq+ 142f - cmpwi cr0, r10, 108 /* Is dcbst? */ - beq+ 144f /* Fix up store bit! */ - cmpwi cr0, r10, 172 /* Is dcbf? */ - beq+ 142f - cmpwi cr0, r10, 1964 /* Is icbi? */ - beq+ 142f + cmpwi cr1, r10, 2028 /* Is dcbz? */ + beq+ cr1, 142f + cmpwi cr1, r10, 940 /* Is dcbi? */ + beq+ cr1, 142f + cmpwi cr1, r10, 108 /* Is dcbst? */ + beq+ cr1, 144f /* Fix up store bit! */ + cmpwi cr1, r10, 172 /* Is dcbf? */ + beq+ cr1, 142f + cmpwi cr1, r10, 1964 /* Is icbi? */ + beq+ cr1, 142f 141: mfspr r10,SPRN_M_TW b DARFixed /* Nope, go back to normal TLB processing */
@@ -676,8 +676,9 @@ FixupDAR:/* Entry point for dcbx workaround. */ add r10, r10, r30 ;b 151f add r10, r10, r31 151: - rlwinm. r11,r11,19,24,28 /* offset into jump table for reg RA */ - beq 152f /* if reg RA is zero, don't add it */ + rlwinm r11,r11,19,24,28 /* offset into jump table for reg RA */ + cmpwi cr1, r11, 0 + beq cr1, 152f /* if reg RA is zero, don't add it */ addi r11, r11, 150b@l /* add start of table */ mtctr r11 /* load ctr with jump address */ rlwinm r11,r11,0,16,10 /* make sure we don't execute this more than once */
Signature · 1 line
2.13.3
PATCH v4 5/16 powerpc/32: add a macro to get and/or save DAR and DSISR on stack.
In reply to Christophe LeroyRefactor reading and saving of DAR and DSISR in exception vectors.
This will ease the implementation of VMAP stack.
--- arch/powerpc/kernel/head_32.S | 5 +---- arch/powerpc/kernel/head_32.h | 11 +++++++++++ arch/powerpc/kernel/head_8xx.S | 23 +++++++---------------- 3 files changed, 19 insertions(+), 20 deletions(-)
arch/powerpc/kernel/head_32.S +1 -4
@@ -339,10 +339,7 @@ END_MMU_FTR_SECTION_IFSET(MMU_FTR_HPTE_TABLE) DO_KVM 0x600 Alignment: EXCEPTION_PROLOG - mfspr r4,SPRN_DAR - stw r4,_DAR(r11) - mfspr r5,SPRN_DSISR - stw r5,_DSISR(r11) + save_dar_dsisr_on_stack r4, r5, r11 addi r3,r1,STACK_FRAME_OVERHEAD EXC_XFER_STD(0x600, alignment_exception)
arch/powerpc/kernel/head_32.h +11 -0
@@ -144,6 +144,17 @@RFI /* jump to handler, enable MMU */ .endm +.macro save_dar_dsisr_on_stack reg1, reg2, sp + mfspr \reg1, SPRN_DAR + mfspr \reg2, SPRN_DSISR + stw \reg1, _DAR(\sp) + stw \reg2, _DSISR(\sp) +.endm + +.macro get_and_save_dar_dsisr_on_stack reg1, reg2, sp + save_dar_dsisr_on_stack \reg1, \reg2, \sp +.endm +/* * Note: code which follows this uses cr0.eq (set if from kernel), * r11, r12 (SRR0), and r9 (SRR1).
arch/powerpc/kernel/head_8xx.S +7 -16
@@ -128,12 +128,9 @@ instruction_counter: . = 0x200 MachineCheck: EXCEPTION_PROLOG - mfspr r4,SPRN_DAR - stw r4,_DAR(r11) - li r5,RPN_PATTERN - mtspr SPRN_DAR,r5 /* Tag DAR, to be used in DTLB Error */ - mfspr r5,SPRN_DSISR - stw r5,_DSISR(r11) + save_dar_dsisr_on_stack r4, r5, r11 + li r6, RPN_PATTERN + mtspr SPRN_DAR, r6 /* Tag DAR, to be used in DTLB Error */ addi r3,r1,STACK_FRAME_OVERHEAD EXC_XFER_STD(0x200, machine_check_exception)
@@ -156,12 +153,9 @@ InstructionAccess: . = 0x600 Alignment: EXCEPTION_PROLOG - mfspr r4,SPRN_DAR - stw r4,_DAR(r11) - li r5,RPN_PATTERN - mtspr SPRN_DAR,r5 /* Tag DAR, to be used in DTLB Error */ - mfspr r5,SPRN_DSISR - stw r5,_DSISR(r11) + save_dar_dsisr_on_stack r4, r5, r11 + li r6, RPN_PATTERN + mtspr SPRN_DAR, r6 /* Tag DAR, to be used in DTLB Error */ addi r3,r1,STACK_FRAME_OVERHEAD EXC_XFER_STD(0x600, alignment_exception)
@@ -502,10 +496,7 @@ DataTLBError: DARFixed:/* Return from dcbx instruction bug workaround */ EXCEPTION_PROLOG_1 EXCEPTION_PROLOG_2 - mfspr r5,SPRN_DSISR - stw r5,_DSISR(r11) - mfspr r4,SPRN_DAR - stw r4, _DAR(r11) + get_and_save_dar_dsisr_on_stack r4, r5, r11 andis. r10,r5,DSISR_NOHPTE@h beq+ .Ldtlbie tlbie r4
Signature · 1 line
2.13.3
PATCH v4 6/16 powerpc/32: prepare for CONFIG_VMAP_STACK
In reply to Christophe LeroyTo support CONFIG_VMAP_STACK, the kernel has to activate Data MMU Translation for accessing the stack. Before doing that it must save SRR0, SRR1 and also DAR and DSISR when relevant, in order to not loose them in case there is a Data TLB Miss once the translation is reactivated.
This patch adds fields in thread struct for saving those registers. It prepares entry_32.S to handle exception entry with Data MMU Translation enabled and alters EXCEPTION_PROLOG macros to save SRR0, SRR1, DAR and DSISR then reenables Data MMU.
--- arch/powerpc/include/asm/processor.h | 6 ++ arch/powerpc/include/asm/thread_info.h | 5 ++ arch/powerpc/kernel/asm-offsets.c | 6 ++ arch/powerpc/kernel/entry_32.S | 7 +++ arch/powerpc/kernel/head_32.h | 101 +++++++++++++++++++++++++++++---- 5 files changed, 115 insertions(+), 10 deletions(-)
arch/powerpc/include/asm/processor.h +6 -0
@@ -163,6 +163,12 @@ struct thread_struct {#if defined(CONFIG_PPC_BOOK3S_32 ) && defined(CONFIG_PPC_KUAP )unsigned long kuap;/* opened segments for user access */ #endif +#ifdef CONFIG_VMAP_STACK +unsigned long srr0; +unsigned long srr1; +unsigned long dar; +unsigned long dsisr; +#endif /* Debug Registers */ struct debug_reg debug;struct thread_fp_state fp_state;
arch/powerpc/include/asm/thread_info.h +5 -0
@@ -10,10 +10,15 @@#define _ASM_POWERPC_THREAD_INFO_H#include <asm/asm-const.h> +#include <asm/page.h> #ifdef __KERNEL__ +#if defined(CONFIG_VMAP_STACK ) &&CONFIG_THREAD_SHIFT <PAGE_SHIFT +#define THREAD_SHIFT PAGE_SHIFT +#else #define THREAD_SHIFT CONFIG_THREAD_SHIFT +#endif #define THREAD_SIZE (1 << THREAD_SHIFT)
arch/powerpc/kernel/asm-offsets.c +6 -0
@@ -127,6 +127,12 @@ int main(void)OFFSET (KSP_VSID , thread_struct, ksp_vsid);#else /* CONFIG_PPC64 */ OFFSET (PGDIR , thread_struct, pgdir); +#ifdef CONFIG_VMAP_STACK +OFFSET (SRR0 , thread_struct, srr0); +OFFSET (SRR1 , thread_struct, srr1); +OFFSET (DAR , thread_struct, dar); +OFFSET (DSISR , thread_struct, dsisr); +#endif #ifdef CONFIG_SPE OFFSET (THREAD_EVR0 , thread_struct, evr[0 ]);OFFSET (THREAD_ACC , thread_struct, acc);
arch/powerpc/kernel/entry_32.S +7 -0
@@ -140,6 +140,9 @@ transfer_to_handler: stw r12,_CTR(r11) stw r2,_XER(r11) mfspr r12,SPRN_SPRG_THREAD +#ifdef CONFIG_VMAP_STACK + tovirt(r12, r12) +#endif beq 2f /* if from user, fix up THREAD.regs */ addi r2, r12, -THREAD addi r11,r1,STACK_FRAME_OVERHEAD
@@ -195,7 +198,11 @@ transfer_to_handler: transfer_to_handler_cont: 3: mflr r9 +#ifdef CONFIG_VMAP_STACK + tovirt(r9, r9) +#else tovirt(r2, r2) /* set r2 to current */ +#endif lwz r11,0(r9) /* virtual address of handler */ lwz r9,4(r9) /* where to go when done */ #if defined(CONFIG_PPC_8xx) && defined(CONFIG_PERF_EVENTS)
arch/powerpc/kernel/head_32.h +91 -10
182 lines not on this page · raw message
Signature · 1 line
2.13.3
PATCH v4 7/16 powerpc: align stack to 2 * THREAD_SIZE with VMAP_STACK
In reply to Christophe LeroyIn order to ease stack overflow detection, align stack to 2 * THREAD_SIZE when using VMAP_STACK. This allows overflow detection using a single bit check.
--- arch/powerpc/include/asm/thread_info.h | 13 +++++++++++++ arch/powerpc/kernel/setup_32.c | 2 +- arch/powerpc/kernel/setup_64.c | 2 +- arch/powerpc/kernel/vmlinux.lds.S | 2 +- 4 files changed, 16 insertions(+), 3 deletions(-)
arch/powerpc/include/asm/thread_info.h +13 -0
@@ -22,6 +22,19 @@#define THREAD_SIZE (1 << THREAD_SHIFT) +/* +* By aligning VMAP'd stacks to 2 * THREAD_SIZE, we can detect overflow by +* checking sp & (1 << THREAD_SHIFT), which we can do cheaply in the entry +* assembly. +*/ +#ifdef CONFIG_VMAP_STACK +#define THREAD_ALIGN_SHIFT (THREAD_SHIFT + 1) +#else +#define THREAD_ALIGN_SHIFT THREAD_SHIFT +#endif + +#define THREAD_ALIGN (1 << THREAD_ALIGN_SHIFT) +#ifndef __ASSEMBLY__#include <linux/cache.h> #include <asm/processor.h>
arch/powerpc/kernel/setup_32.c +1 -1
@@ -140,7 +140,7 @@ arch_initcall(ppc_init);static void *__init alloc_stack (void ) { -void *ptr =memblock_alloc (THREAD_SIZE ,THREAD_SIZE ); +void *ptr =memblock_alloc (THREAD_SIZE ,THREAD_ALIGN );if (!ptr) panic("cannot allocate %d bytes for stack at %pS\n",
arch/powerpc/kernel/setup_64.c +1 -1
@@ -633,7 +633,7 @@ static void *__init alloc_stack(unsigned long limit, int cpu)BUILD_BUG_ON (STACK_INT_FRAME_SIZE %16 ); - ptr =memblock_alloc_try_nid (THREAD_SIZE ,THREAD_SIZE , + ptr =memblock_alloc_try_nid (THREAD_SIZE ,THREAD_ALIGN ,MEMBLOCK_LOW_LIMIT , limit,early_cpu_to_node (cpu));if (!ptr)
arch/powerpc/kernel/vmlinux.lds.S +1 -1
@@ -346,7 +346,7 @@ SECTIONS #endif /* The initial task and kernel stack */ - INIT_TASK_DATA_SECTION(THREAD_SIZE) + INIT_TASK_DATA_SECTION(THREAD_ALIGN) .data..page_aligned : AT(ADDR(.data..page_aligned) - LOAD_OFFSET) { PAGE_ALIGNED_DATA(PAGE_SIZE)
Signature · 1 line
2.13.3
PATCH v4 8/16 powerpc/32: Add early stack overflow detection with VMAP stack.
In reply to Christophe LeroyTo avoid recursive faults, stack overflow detection has to be performed before writing in the stack in exception prologs.
Do it by checking the alignment. If the stack pointer alignment is wrong, it means it is pointing to the following or preceding page.
Without VMAP stack, a stack overflow is catastrophic. With VMAP stack, a stack overflow isn't destructive, so don't panic. Kill the task with SIGSEGV instead.
A dedicated overflow stack is set up for each CPU.
lkdtm: Performing direct entry EXHAUST_STACK
lkdtm: Calling function with 512 frame size to depth 32 ...
lkdtm: loop 32/32 ...
lkdtm: loop 31/32 ...
lkdtm: loop 30/32 ...
lkdtm: loop 29/32 ...
lkdtm: loop 28/32 ...
lkdtm: loop 27/32 ...
lkdtm: loop 26/32 ...
lkdtm: loop 25/32 ...
lkdtm: loop 24/32 ...
lkdtm: loop 23/32 ...
lkdtm: loop 22/32 ...
lkdtm: loop 21/32 ...
lkdtm: loop 20/32 ...
Kernel stack overflow in process test[359], r1=c900c008
Oops: Kernel stack overflow, sig: 6 [#1] BE PAGE_SIZE=4K MMU=Hash PowerMac Modules linked in: CPU: 0 PID: 359 Comm: test Not tainted 5.3.0-rc7+ #2225 NIP: c0622060 LR: c0626710 CTR: 00000000 REGS: c0895f48 TRAP: 0000 Not tainted (5.3.0-rc7+) MSR: 00001032 <ME,IR,DR,RI> CR: 28004224 XER: 00000000
GPR00: c0626ca4 c900c008 c783c000 c07335cc c900c010 c07335cc c900c0f0 c07335cc
GPR08: c900c0f0 00000001 00000000 00000000 28008222 00000000 00000000 00000000
GPR16: 00000000 00000000 10010128 10010000 b799c245 10010158 c07335cc 00000025
GPR24: c0690000 c08b91d4 c068f688 00000020 c900c0f0 c068f668 c08b95b4 c08b91d4 NIP [c0622060] format_decode+0x0/0x4d4
LR [c0626710] vsnprintf+0x80/0x5fc
Call Trace:
[c900c068] [c0626ca4] vscnprintf+0x18/0x48
[c900c078] [c007b944] vprintk_store+0x40/0x214
[c900c0b8] [c007bf50] vprintk_emit+0x90/0x1dc
[c900c0e8] [c007c5cc] printk+0x50/0x60
[c900c128] [c03da5b0] recursive_loop+0x44/0x6c
[c900c338] [c03da5c4] recursive_loop+0x58/0x6c
[c900c548] [c03da5c4] recursive_loop+0x58/0x6c
[c900c758] [c03da5c4] recursive_loop+0x58/0x6c
[c900c968] [c03da5c4] recursive_loop+0x58/0x6c
[c900cb78] [c03da5c4] recursive_loop+0x58/0x6c
[c900cd88] [c03da5c4] recursive_loop+0x58/0x6c
[c900cf98] [c03da5c4] recursive_loop+0x58/0x6c
[c900d1a8] [c03da5c4] recursive_loop+0x58/0x6c
[c900d3b8] [c03da5c4] recursive_loop+0x58/0x6c
[c900d5c8] [c03da5c4] recursive_loop+0x58/0x6c
[c900d7d8] [c03da5c4] recursive_loop+0x58/0x6c
[c900d9e8] [c03da5c4] recursive_loop+0x58/0x6c
[c900dbf8] [c03da5c4] recursive_loop+0x58/0x6c
[c900de08] [c03da67c] lkdtm_EXHAUST_STACK+0x30/0x4c
[c900de18] [c03da3e8] direct_entry+0xc8/0x140
[c900de48] [c029fb40] full_proxy_write+0x64/0xcc
[c900de68] [c01500f8] __vfs_write+0x30/0x1d0
[c900dee8] [c0152cb8] vfs_write+0xb8/0x1d4
[c900df08] [c0152f7c] ksys_write+0x58/0xe8
[c900df38] [c0014208] ret_from_syscall+0x0/0x34
--- interrupt: c01 at 0xf806664
LR = 0x1000c868Instruction dump:4bffff91 80010014 7c832378 7c0803a6 38210010 4e800020 3d20c08a 3ca0c089 8089a0cc 38a58f0c 38600001 4ba2d494 <9421ffe0> 7c0802a6 bfc10018 7c9f2378
--- arch/powerpc/include/asm/irq.h | 1 + arch/powerpc/kernel/entry_32.S | 25 +++++++++++++++++++++++++ arch/powerpc/kernel/head_32.h | 4 ++++ arch/powerpc/kernel/irq.c | 1 + arch/powerpc/kernel/setup_32.c | 1 + arch/powerpc/kernel/traps.c | 15 ++++++++++++--- 6 files changed, 44 insertions(+), 3 deletions(-)
arch/powerpc/include/asm/irq.h +1 -0
@@ -55,6 +55,7 @@ extern void *mcheckirq_ctx[NR_CPUS];*/ extern void *hardirq_ctx[NR_CPUS ];extern void *softirq_ctx[NR_CPUS ]; +extern void *stackovf_ctx[NR_CPUS ];void call_do_softirq (void *sp);void call_do_irq (struct pt_regs *regs,void *sp);
arch/powerpc/kernel/entry_32.S +25 -0
@@ -184,9 +184,11 @@ transfer_to_handler: */ kuap_save_and_lock r11, r12, r9, r2, r0 addi r2, r12, -THREAD +#ifndef CONFIG_VMAP_STACK lwz r9,KSP_LIMIT(r12) cmplw r1,r9 /* if r1 <= ksp_limit */ ble- stack_ovf /* then the kernel stack overflowed */ +#endif 5: #if defined(CONFIG_PPC_BOOK3S_32) || defined(CONFIG_E500) lwz r12,TI_LOCAL_FLAGS(r2)
@@ -298,6 +300,28 @@ reenable_mmu: * On kernel stack overflow, load up an initial stack pointer * and call StackOverflow(regs), which should not return. */ +#ifdef CONFIG_VMAP_STACK +_GLOBAL(stack_ovf) + li r11, 0 +#ifdef CONFIG_SMP + mfspr r11, SPRN_SPRG_THREAD + tovirt(r11, r11) + lwz r11, TASK_CPU - THREAD(r11) + slwi r11, r11, 3 +#endif + addis r11, r11, stackovf_ctx@ha + addi r11, r11, stackovf_ctx@l + lwz r11, 0(r11) + cmpwi cr1, r11, 0 + addi r11, r11, THREAD_SIZE - INT_FRAME_SIZE + bne cr1, 1f + lis r11, init_thread_union + THREAD_SIZE - INT_FRAME_SIZE@ha + addi r11, r11, init_thread_union + THREAD_SIZE - INT_FRAME_SIZE@l +1: EXCEPTION_PROLOG_2 + SAVE_NVGPRS(r11) + addi r3, r1, STACK_FRAME_OVERHEAD + EXC_XFER_STD(0, StackOverflow) +#else stack_ovf: /* sometimes we use a statically-allocated stack, which is OK. */ lis r12,_end@h
@@ -319,6 +343,7 @@ stack_ovf: mtspr SPRN_SRR1,r10 SYNC RFI +#endif #ifdef CONFIG_TRACE_IRQFLAGS trace_syscall_entry_irq_off:
arch/powerpc/kernel/head_32.h +4 -0
@@ -58,6 +58,10 @@ tophys(r11,r11)#endif 1: +#ifdef CONFIG_VMAP_STACK + mtcrf 0x7f, r11 + bt 32 - THREAD_ALIGN_SHIFT, stack_ovf_trampoline +#endif .endm .macro EXCEPTION_PROLOG_2 ext
arch/powerpc/kernel/irq.c +1 -0
@@ -678,6 +678,7 @@ void *mcheckirq_ctx[NR_CPUS] __read_mostly;void *softirq_ctx[NR_CPUS ]__read_mostly ;void *hardirq_ctx[NR_CPUS ]__read_mostly ; +void *stackovf_ctx[NR_CPUS ]__read_mostly ;void do_softirq_own_stack (void ) {
arch/powerpc/kernel/setup_32.c +1 -0
@@ -158,6 +158,7 @@ void __init irqstack_early_init(void)for_each_possible_cpu (i) { softirq_ctx[i] =alloc_stack (); hardirq_ctx[i] =alloc_stack (); + stackovf_ctx[i] =alloc_stack (); } }
arch/powerpc/kernel/traps.c +12 -3
@@ -1630,11 +1630,20 @@ void alignment_exception(struct pt_regs *regs)void StackOverflow (struct pt_regs *regs) { +enum ctx_state prev_state =exception_enter (); +pr_crit ("Kernel stack overflow in process %s[%d], r1=%lx\n" , current->comm,task_pid_nr (current), regs->gpr[1 ]); -debugger (regs); -show_regs (regs); -panic ("kernel stack overflow" ); + +if (IS_ENABLED (CONFIG_VMAP_STACK )) { +die ("Kernel stack overflow" , regs,SIGSEGV ); + }else { +debugger (regs); +show_regs (regs); +panic ("kernel stack overflow" ); + } + +exception_exit (prev_state); }void kernel_fp_unavailable_exception (struct pt_regs *regs)
Signature · 1 line
2.13.3
PATCH v4 9/16 powerpc/8xx: Use alternative scratch registers in DTLB miss handler
In reply to Christophe LeroyIn preparation of handling CONFIG_VMAP_STACK, DTLB miss handler need to use different scratch registers than other exception handlers in order to not jeopardise exception entry on stack DTLB misses.
--- arch/powerpc/kernel/head_8xx.S | 27 ++++++++++++++------------- arch/powerpc/perf/8xx-pmu.c | 12 ++++++++---- 2 files changed, 22 insertions(+), 17 deletions(-)
arch/powerpc/kernel/head_8xx.S +14 -13
@@ -193,8 +193,9 @@ SystemCall: 0: lwz r10, (dtlb_miss_counter - PAGE_OFFSET)@l(0) addi r10, r10, 1 stw r10, (dtlb_miss_counter - PAGE_OFFSET)@l(0) - mfspr r10, SPRN_SPRG_SCRATCH0 - mfspr r11, SPRN_SPRG_SCRATCH1 + mfspr r10, SPRN_DAR + mtspr SPRN_DAR, r11 /* Tag DAR */ + mfspr r11, SPRN_M_TW rfi #endif
@@ -337,8 +338,8 @@ ITLBMissLinear: . = 0x1200 DataStoreTLBMiss: - mtspr SPRN_SPRG_SCRATCH0, r10 - mtspr SPRN_SPRG_SCRATCH1, r11 + mtspr SPRN_DAR, r10 + mtspr SPRN_M_TW, r11 mfcr r11 /* If we are faulting a kernel address, we have to use the
@@ -403,10 +404,10 @@ DataStoreTLBMiss: mtspr SPRN_MD_RPN, r10 /* Update TLB entry */ /* Restore registers */ - mtspr SPRN_DAR, r11 /* Tag DAR */ -0: mfspr r10, SPRN_SPRG_SCRATCH0 - mfspr r11, SPRN_SPRG_SCRATCH1 +0: mfspr r10, SPRN_DAR + mtspr SPRN_DAR, r11 /* Tag DAR */ + mfspr r11, SPRN_M_TW rfi patch_site 0b, patch__dtlbmiss_exit_1
@@ -422,10 +423,10 @@ DTLBMissIMMR: mtspr SPRN_MD_RPN, r10 /* Update TLB entry */ li r11, RPN_PATTERN - mtspr SPRN_DAR, r11 /* Tag DAR */ -0: mfspr r10, SPRN_SPRG_SCRATCH0 - mfspr r11, SPRN_SPRG_SCRATCH1 +0: mfspr r10, SPRN_DAR + mtspr SPRN_DAR, r11 /* Tag DAR */ + mfspr r11, SPRN_M_TW rfi patch_site 0b, patch__dtlbmiss_exit_2
@@ -459,10 +460,10 @@ DTLBMissLinear: mtspr SPRN_MD_RPN, r10 /* Update TLB entry */ li r11, RPN_PATTERN - mtspr SPRN_DAR, r11 /* Tag DAR */ -0: mfspr r10, SPRN_SPRG_SCRATCH0 - mfspr r11, SPRN_SPRG_SCRATCH1 +0: mfspr r10, SPRN_DAR + mtspr SPRN_DAR, r11 /* Tag DAR */ + mfspr r11, SPRN_M_TW rfi patch_site 0b, patch__dtlbmiss_exit_3
arch/powerpc/perf/8xx-pmu.c +8 -4
@@ -157,10 +157,6 @@ static void mpc8xx_pmu_read(struct perf_event *event)static void mpc8xx_pmu_del (struct perf_event *event,int flags) { -/* mfspr r10, SPRN_SPRG_SCRATCH0 */ -unsigned int insn =PPC_INST_MFSPR |__PPC_RS (R10 ) | -__PPC_SPR (SPRN_SPRG_SCRATCH0 ); -mpc8xx_pmu_read (event);/* If it was the last user, stop counting to avoid useles overhead */
@@ -173,6 +169,10 @@ static void mpc8xx_pmu_del(struct perf_event *event, int flags)break ;case PERF_8xx_ID_ITLB_LOAD_MISS:if (atomic_dec_return (&itlb_miss_ref) ==0 ) { +/* mfspr r10, SPRN_SPRG_SCRATCH0 */ +unsigned int insn =PPC_INST_MFSPR |__PPC_RS (R10 ) | +__PPC_SPR (SPRN_SPRG_SCRATCH0 ); +patch_instruction_site (&patch__itlbmiss_exit_1, insn);#ifndef CONFIG_PIN_TLB_TEXT patch_instruction_site (&patch__itlbmiss_exit_2, insn);
@@ -181,6 +181,10 @@ static void mpc8xx_pmu_del(struct perf_event *event, int flags)break ;case PERF_8xx_ID_DTLB_LOAD_MISS:if (atomic_dec_return (&dtlb_miss_ref) ==0 ) { +/* mfspr r10, SPRN_DAR */ +unsigned int insn =PPC_INST_MFSPR |__PPC_RS (R10 ) | +__PPC_SPR (SPRN_DAR ); +patch_instruction_site (&patch__dtlbmiss_exit_1, insn);patch_instruction_site (&patch__dtlbmiss_exit_2, insn);patch_instruction_site (&patch__dtlbmiss_exit_3, insn);
Signature · 1 line
2.13.3
PATCH v4 10/16 powerpc/8xx: drop exception entries for non-existing exceptions
In reply to Christophe Leroyhead_8xx.S has entries for all exceptions from 0x100 to 0x1f00. Several of them do not exist and are never generated by the 8xx in accordance with the documentation.
Remove those entry points to make some room for future growing exception code.
--- arch/powerpc/kernel/head_8xx.S | 29 ----------------------------- 1 file changed, 29 deletions(-)
arch/powerpc/kernel/head_8xx.S +0 -29
@@ -134,18 +134,6 @@ MachineCheck: addi r3,r1,STACK_FRAME_OVERHEAD EXC_XFER_STD(0x200, machine_check_exception) -/* Data access exception. - * This is "never generated" by the MPC8xx. - */ - . = 0x300 -DataAccess: - -/* Instruction access exception. - * This is "never generated" by the MPC8xx. - */ - . = 0x400 -InstructionAccess: - /* External interrupt */ EXCEPTION(0x500, HardwareInterrupt, do_IRQ, EXC_XFER_LITE)
@@ -162,16 +150,9 @@ Alignment: /* Program check exception */ EXCEPTION(0x700, ProgramCheck, program_check_exception, EXC_XFER_STD) -/* No FPU on MPC8xx. This exception is not supposed to happen. -*/ - EXCEPTION(0x800, FPUnavailable, unknown_exception, EXC_XFER_STD) - /* Decrementer */ EXCEPTION(0x900, Decrementer, timer_interrupt, EXC_XFER_LITE) - EXCEPTION(0xa00, Trap_0a, unknown_exception, EXC_XFER_STD) - EXCEPTION(0xb00, Trap_0b, unknown_exception, EXC_XFER_STD) - /* System call */ . = 0xc00 SystemCall:
@@ -179,8 +160,6 @@ SystemCall: /* Single step - not used on 601 */ EXCEPTION(0xd00, SingleStep, single_step_exception, EXC_XFER_STD) - EXCEPTION(0xe00, Trap_0e, unknown_exception, EXC_XFER_STD) - EXCEPTION(0xf00, Trap_0f, unknown_exception, EXC_XFER_STD) /* On the MPC8xx, this is a software emulation interrupt. It occurs * for all unimplemented and illegal instructions.
@@ -507,14 +486,6 @@ DARFixed:/* Return from dcbx instruction bug workaround */ /* 0x300 is DataAccess exception, needed by bad_page_fault() */ EXC_XFER_LITE(0x300, handle_page_fault) - EXCEPTION(0x1500, Trap_15, unknown_exception, EXC_XFER_STD) - EXCEPTION(0x1600, Trap_16, unknown_exception, EXC_XFER_STD) - EXCEPTION(0x1700, Trap_17, unknown_exception, EXC_XFER_STD) - EXCEPTION(0x1800, Trap_18, unknown_exception, EXC_XFER_STD) - EXCEPTION(0x1900, Trap_19, unknown_exception, EXC_XFER_STD) - EXCEPTION(0x1a00, Trap_1a, unknown_exception, EXC_XFER_STD) - EXCEPTION(0x1b00, Trap_1b, unknown_exception, EXC_XFER_STD) - /* On the MPC8xx, these next four traps are used for development * support of breakpoints and such. Someday I will get around to * using them.
Signature · 1 line
2.13.3
PATCH v4 11/16 powerpc/8xx: move DataStoreTLBMiss perf handler
In reply to Christophe LeroyMove DataStoreTLBMiss perf handler in order to cope with future growing exception prolog.
--- arch/powerpc/kernel/head_8xx.S | 24 ++++++++++++------------ 1 file changed, 12 insertions(+), 12 deletions(-)
arch/powerpc/kernel/head_8xx.S +12 -12
@@ -166,18 +166,6 @@ SystemCall: */ EXCEPTION(0x1000, SoftEmu, program_check_exception, EXC_XFER_STD) -/* Called from DataStoreTLBMiss when perf TLB misses events are activated */ -#ifdef CONFIG_PERF_EVENTS - patch_site 0f, patch__dtlbmiss_perf -0: lwz r10, (dtlb_miss_counter - PAGE_OFFSET)@l(0) - addi r10, r10, 1 - stw r10, (dtlb_miss_counter - PAGE_OFFSET)@l(0) - mfspr r10, SPRN_DAR - mtspr SPRN_DAR, r11 /* Tag DAR */ - mfspr r11, SPRN_M_TW - rfi -#endif - . = 0x1100 /* * For the MPC8xx, this is a software tablewalk to load the instruction
@@ -486,6 +474,18 @@ DARFixed:/* Return from dcbx instruction bug workaround */ /* 0x300 is DataAccess exception, needed by bad_page_fault() */ EXC_XFER_LITE(0x300, handle_page_fault) +/* Called from DataStoreTLBMiss when perf TLB misses events are activated */ +#ifdef CONFIG_PERF_EVENTS + patch_site 0f, patch__dtlbmiss_perf +0: lwz r10, (dtlb_miss_counter - PAGE_OFFSET)@l(0) + addi r10, r10, 1 + stw r10, (dtlb_miss_counter - PAGE_OFFSET)@l(0) + mfspr r10, SPRN_DAR + mtspr SPRN_DAR, r11 /* Tag DAR */ + mfspr r11, SPRN_M_TW + rfi +#endif + /* On the MPC8xx, these next four traps are used for development * support of breakpoints and such. Someday I will get around to * using them.
Signature · 1 line
2.13.3
PATCH v4 12/16 powerpc/8xx: split breakpoint exception
In reply to Christophe LeroyBreakpoint exception is big.
Split it to support future growth on exception prolog.
--- arch/powerpc/kernel/head_8xx.S | 19 ++++++++++--------- 1 file changed, 10 insertions(+), 9 deletions(-)
arch/powerpc/kernel/head_8xx.S +10 -9
@@ -490,14 +490,7 @@ DARFixed:/* Return from dcbx instruction bug workaround */ * support of breakpoints and such. Someday I will get around to * using them. */ - . = 0x1c00 -DataBreakpoint: - EXCEPTION_PROLOG_0 - mfspr r11, SPRN_SRR0 - cmplwi cr1, r11, (.Ldtlbie - PAGE_OFFSET)@l - cmplwi cr7, r11, (.Litlbie - PAGE_OFFSET)@l - beq- cr1, 11f - beq- cr7, 11f +do_databreakpoint: EXCEPTION_PROLOG_1 EXCEPTION_PROLOG_2 addi r3,r1,STACK_FRAME_OVERHEAD
@@ -505,7 +498,15 @@ DataBreakpoint: stw r4,_DAR(r11) mfspr r5,SPRN_DSISR EXC_XFER_STD(0x1c00, do_break) -11: + + . = 0x1c00 +DataBreakpoint: + EXCEPTION_PROLOG_0 + mfspr r11, SPRN_SRR0 + cmplwi cr1, r11, (.Ldtlbie - PAGE_OFFSET)@l + cmplwi cr7, r11, (.Litlbie - PAGE_OFFSET)@l + cror 4*cr1+eq, 4*cr1+eq, 4*cr7+eq + bne cr1, do_databreakpoint mtcr r10 mfspr r10, SPRN_SPRG_SCRATCH0 mfspr r11, SPRN_SPRG_SCRATCH1
Signature · 1 line
2.13.3
PATCH v4 13/16 powerpc/8xx: Enable CONFIG_VMAP_STACK
In reply to Christophe LeroyThis patch enables CONFIG_VMAP_STACK. For that, a few changes are done in head_8xx.S.
--- arch/powerpc/kernel/head_8xx.S | 34 ++++++++++++++++++++++++++++------ arch/powerpc/platforms/Kconfig.cputype | 1 + 2 files changed, 29 insertions(+), 6 deletions(-)
arch/powerpc/kernel/head_8xx.S +28 -6
@@ -127,7 +127,7 @@ instruction_counter: /* Machine check */ . = 0x200 MachineCheck: - EXCEPTION_PROLOG + EXCEPTION_PROLOG dar save_dar_dsisr_on_stack r4, r5, r11 li r6, RPN_PATTERN mtspr SPRN_DAR, r6 /* Tag DAR, to be used in DTLB Error */
@@ -140,7 +140,7 @@ MachineCheck: /* Alignment exception */ . = 0x600 Alignment: - EXCEPTION_PROLOG + EXCEPTION_PROLOG dar save_dar_dsisr_on_stack r4, r5, r11 li r6, RPN_PATTERN mtspr SPRN_DAR, r6 /* Tag DAR, to be used in DTLB Error */
@@ -457,20 +457,26 @@ InstructionTLBError: */ . = 0x1400 DataTLBError: - EXCEPTION_PROLOG_0 + EXCEPTION_PROLOG_0 dar mfspr r11, SPRN_DAR cmpwi cr1, r11, RPN_PATTERN beq- cr1, FixupDAR /* must be a buggy dcbX, icbi insn. */ DARFixed:/* Return from dcbx instruction bug workaround */ +#ifdef CONFIG_VMAP_STACK + li r11, RPN_PATTERN + mtspr SPRN_DAR, r11 /* Tag DAR, to be used in DTLB Error */ +#endif EXCEPTION_PROLOG_1 - EXCEPTION_PROLOG_2 + EXCEPTION_PROLOG_2 dar get_and_save_dar_dsisr_on_stack r4, r5, r11 andis. r10,r5,DSISR_NOHPTE@h beq+ .Ldtlbie tlbie r4 .Ldtlbie: +#ifndef CONFIG_VMAP_STACK li r10,RPN_PATTERN mtspr SPRN_DAR,r10 /* Tag DAR, to be used in DTLB Error */ +#endif /* 0x300 is DataAccess exception, needed by bad_page_fault() */ EXC_XFER_LITE(0x300, handle_page_fault)
@@ -492,16 +498,20 @@ DARFixed:/* Return from dcbx instruction bug workaround */ */ do_databreakpoint: EXCEPTION_PROLOG_1 - EXCEPTION_PROLOG_2 + EXCEPTION_PROLOG_2 dar addi r3,r1,STACK_FRAME_OVERHEAD mfspr r4,SPRN_BAR stw r4,_DAR(r11) +#ifdef CONFIG_VMAP_STACK + lwz r5,_DSISR(r11) +#else mfspr r5,SPRN_DSISR +#endif EXC_XFER_STD(0x1c00, do_break) . = 0x1c00 DataBreakpoint: - EXCEPTION_PROLOG_0 + EXCEPTION_PROLOG_0 dar mfspr r11, SPRN_SRR0 cmplwi cr1, r11, (.Ldtlbie - PAGE_OFFSET)@l cmplwi cr7, r11, (.Litlbie - PAGE_OFFSET)@l
@@ -530,6 +540,11 @@ InstructionBreakpoint: EXCEPTION(0x1e00, Trap_1e, unknown_exception, EXC_XFER_STD) EXCEPTION(0x1f00, Trap_1f, unknown_exception, EXC_XFER_STD) +#ifdef CONFIG_VMAP_STACK +stack_ovf_trampoline: + b stack_ovf +#endif + . = 0x2000 /* This is the procedure to calculate the data EA for buggy dcbx,dcbi instructions
@@ -650,7 +665,14 @@ FixupDAR:/* Entry point for dcbx workaround. */ 152: mfdar r11 mtctr r11 /* restore ctr reg from DAR */ +#ifdef CONFIG_VMAP_STACK + mfspr r11, SPRN_SPRG_THREAD + stw r10, DAR(r11) + mfspr r10, SPRN_DSISR + stw r10, DSISR(r11) +#else mtdar r10 /* save fault EA to DAR */ +#endif mfspr r10,SPRN_M_TW b DARFixed /* Go back to normal TLB handling */
arch/powerpc/platforms/Kconfig.cputype +1 -0
@@ -49,6 +49,7 @@ config PPC_8xx select PPC_HAVE_KUEP select PPC_HAVE_KUAP select PPC_MM_SLICES if HUGETLB_PAGE + select HAVE_ARCH_VMAP_STACK config 40x bool "AMCC 40x"
Signature · 1 line
2.13.3
PATCH v4 14/16 powerpc/32s: reorganise DSI handler.
In reply to Christophe LeroyThe part decidated to handling hash_page() is fully unneeded for processors not having real hash pages like the 603.
Lets enlarge the content of the feature fixup, and provide an alternative which jumps directly instead of getting NIPs.
Also, in preparation of VMAP stacks, the end of DSI handler has moved to later in the code as it won't fit anymore once VMAP stacks are there.
--- arch/powerpc/kernel/head_32.S | 31 +++++++++++++++++-------------- 1 file changed, 17 insertions(+), 14 deletions(-)
arch/powerpc/kernel/head_32.S +17 -14
@@ -295,24 +295,20 @@ __secondary_hold_acknowledge: DO_KVM 0x300 DataAccess: EXCEPTION_PROLOG - mfspr r10,SPRN_DSISR - stw r10,_DSISR(r11) + get_and_save_dar_dsisr_on_stack r4, r5, r11 +BEGIN_MMU_FTR_SECTION #ifdef CONFIG_PPC_KUAP - andis. r0,r10,(DSISR_BAD_FAULT_32S | DSISR_DABRMATCH | DSISR_PROTFAULT)@h + andis. r0, r5, (DSISR_BAD_FAULT_32S | DSISR_DABRMATCH | DSISR_PROTFAULT)@h #else - andis. r0,r10,(DSISR_BAD_FAULT_32S|DSISR_DABRMATCH)@h + andis. r0, r5, (DSISR_BAD_FAULT_32S | DSISR_DABRMATCH)@h #endif - bne 1f /* if not, try to put a PTE */ - mfspr r4,SPRN_DAR /* into the hash table */ - rlwinm r3,r10,32-15,21,21 /* DSISR_STORE -> _PAGE_RW */ -BEGIN_MMU_FTR_SECTION + bne handle_page_fault_tramp_2 /* if not, try to put a PTE */ + rlwinm r3, r5, 32 - 15, 21, 21 /* DSISR_STORE -> _PAGE_RW */ bl hash_page -END_MMU_FTR_SECTION_IFSET(MMU_FTR_HPTE_TABLE) -1: lwz r5,_DSISR(r11) /* get DSISR value */ - mfspr r4,SPRN_DAR - stw r4, _DAR(r11) - EXC_XFER_LITE(0x300, handle_page_fault) - + b handle_page_fault_tramp_1 +FTR_SECTION_ELSE + b handle_page_fault_tramp_2 +ALT_MMU_FTR_SECTION_END_IFSET(MMU_FTR_HPTE_TABLE) /* Instruction access exception. */ . = 0x400
@@ -642,6 +638,13 @@ END_MMU_FTR_SECTION_IFSET(MMU_FTR_NEED_DTLB_SW_LRU) . = 0x3000 +handle_page_fault_tramp_1: + lwz r4, _DAR(r11) + lwz r5, _DSISR(r11) + /* fall through */ +handle_page_fault_tramp_2: + EXC_XFER_LITE(0x300, handle_page_fault) + AltiVecUnavailable: EXCEPTION_PROLOG #ifdef CONFIG_ALTIVEC
Signature · 1 line
2.13.3
PATCH v4 15/16 powerpc/32s: avoid crossing page boundary while changing SRR0/1.
In reply to Christophe LeroyTrying VMAP_STACK with KVM, vmlinux was not starting. This was due to SRR0 and SRR1 clobbered by an ISI due to the rfi being in a different page than the mtsrr0/1:
c0003fe0 <mmu_off>:
c0003fe0: 38 83 00 54 addi r4,r3,84 c0003fe4: 7c 60 00 a6 mfmsr r3 c0003fe8: 70 60 00 30 andi. r0,r3,48 c0003fec: 4d 82 00 20 beqlr c0003ff0: 7c 63 00 78 andc r3,r3,r0 c0003ff4: 7c 9a 03 a6 mtsrr0 r4 c0003ff8: 7c 7b 03 a6 mtsrr1 r3 c0003ffc: 7c 00 04 ac hwsync c0004000: 4c 00 00 64 rfi
Align the 4 instruction block used to deactivate MMU to order 4, so that the block never crosses a page boundary.
--- arch/powerpc/kernel/head_32.S | 6 ++++++ 1 file changed, 6 insertions(+)
arch/powerpc/kernel/head_32.S +6 -0
@@ -917,6 +917,8 @@ END_MMU_FTR_SECTION_IFSET(MMU_FTR_HPTE_TABLE) ori r4,r4,2f@l tophys(r4,r4) li r3,MSR_KERNEL & ~(MSR_IR|MSR_DR) + + .align 4 mtspr SPRN_SRR0,r4 mtspr SPRN_SRR1,r3 SYNC
@@ -1058,6 +1060,8 @@ _ENTRY(update_bats) rlwinm r0, r6, 0, ~MSR_RI rlwinm r0, r0, 0, ~MSR_EE mtmsr r0 + + .align 4 mtspr SPRN_SRR0, r4 mtspr SPRN_SRR1, r3 SYNC
@@ -1097,6 +1101,8 @@ mmu_off: andi. r0,r3,MSR_DR|MSR_IR /* MMU enabled? */ beqlr andc r3,r3,r0 + + .align 4 mtspr SPRN_SRR0,r4 mtspr SPRN_SRR1,r3 sync
Signature · 1 line
2.13.3
PATCH v4 16/16 powerpc/32s: Activate CONFIG_VMAP_STACK
In reply to Christophe LeroyA few changes to retrieve DAR and DSISR from struct regs instead of retrieving them directly, as they may have changed due to a TLB miss.
Also modifies hash_page() and friends to work with virtual data addresses instead of physical ones. Same on load_up_fpu() and load_up_altivec().
--- arch/powerpc/kernel/entry_32.S | 4 +++ arch/powerpc/kernel/fpu.S | 3 +++ arch/powerpc/kernel/head_32.S | 19 +++++++++++--- arch/powerpc/kernel/head_32.h | 4 ++- arch/powerpc/kernel/vector.S | 3 +++ arch/powerpc/mm/book3s32/hash_low.S | 46 +++++++++++++++++++++------------- arch/powerpc/mm/book3s32/mmu.c | 9 +++++-- arch/powerpc/platforms/Kconfig.cputype | 2 ++ 8 files changed, 67 insertions(+), 23 deletions(-)
arch/powerpc/kernel/entry_32.S +4 -0
@@ -1365,7 +1365,11 @@ _GLOBAL(enter_rtas) lis r6,1f@ha /* physical return address for rtas */ addi r6,r6,1f@l tophys(r6,r6) +#ifdef CONFIG_VMAP_STACK + mr r7, r1 +#else tophys(r7,r1) +#endif lwz r8,RTASENTRY(r4) lwz r4,RTASBASE(r4) mfmsr r9
arch/powerpc/kernel/fpu.S +3 -0
@@ -94,6 +94,9 @@ END_FTR_SECTION_IFSET(CPU_FTR_VSX) /* enable use of FP after return */ #ifdef CONFIG_PPC32 mfspr r5,SPRN_SPRG_THREAD /* current task's THREAD (phys) */ +#ifdef CONFIG_VMAP_STACK + tovirt(r5, r5) +#endif lwz r4,THREAD_FPEXC_MODE(r5) ori r9,r9,MSR_FP /* enable FP for current */ or r9,r9,r4
arch/powerpc/kernel/head_32.S +16 -3
@@ -272,14 +272,22 @@ __secondary_hold_acknowledge: */ . = 0x200 DO_KVM 0x200 +MachineCheck: EXCEPTION_PROLOG_0 +#ifdef CONFIG_VMAP_STACK + li r11, MSR_KERNEL & ~(MSR_IR | MSR_RI) /* can take DTLB miss */ + mtmsr r11 +#endif #ifdef CONFIG_PPC_CHRP mfspr r11, SPRN_SPRG_THREAD +#ifdef CONFIG_VMAP_STACK + tovirt(r11, r11) +#endif lwz r11, RTAS_SP(r11) cmpwi cr1, r11, 0 bne cr1, 7f #endif /* CONFIG_PPC_CHRP */ - EXCEPTION_PROLOG_1 + EXCEPTION_PROLOG_1 rtas 7: EXCEPTION_PROLOG_2 addi r3,r1,STACK_FRAME_OVERHEAD #ifdef CONFIG_PPC_CHRP
@@ -294,7 +302,7 @@ __secondary_hold_acknowledge: . = 0x300 DO_KVM 0x300 DataAccess: - EXCEPTION_PROLOG + EXCEPTION_PROLOG dar get_and_save_dar_dsisr_on_stack r4, r5, r11 BEGIN_MMU_FTR_SECTION #ifdef CONFIG_PPC_KUAP
@@ -334,7 +342,7 @@ END_MMU_FTR_SECTION_IFSET(MMU_FTR_HPTE_TABLE) . = 0x600 DO_KVM 0x600 Alignment: - EXCEPTION_PROLOG + EXCEPTION_PROLOG dar save_dar_dsisr_on_stack r4, r5, r11 addi r3,r1,STACK_FRAME_OVERHEAD EXC_XFER_STD(0x600, alignment_exception)
@@ -645,6 +653,11 @@ handle_page_fault_tramp_1: handle_page_fault_tramp_2: EXC_XFER_LITE(0x300, handle_page_fault) +#ifdef CONFIG_VMAP_STACK +stack_ovf_trampoline: + b stack_ovf +#endif + AltiVecUnavailable: EXCEPTION_PROLOG #ifdef CONFIG_ALTIVEC
arch/powerpc/kernel/head_32.h +3 -1
@@ -38,10 +38,12 @@ andi. r11, r11,MSR_PR .endm -.macro EXCEPTION_PROLOG_1 +.macro EXCEPTION_PROLOG_1 rtas#ifdef CONFIG_VMAP_STACK + .ifb \rtas li r11, MSR_KERNEL & ~(MSR_IR | MSR_RI) /* can take DTLB miss */ mtmsr r11 + .endif subi r11, r1, INT_FRAME_SIZE /* use r1 if kernel */#else tophys (r11,r1)/* use tophys(r1) if kernel */
arch/powerpc/kernel/vector.S +3 -0
@@ -67,6 +67,9 @@ _GLOBAL(load_up_altivec) #ifdef CONFIG_PPC32 mfspr r5,SPRN_SPRG_THREAD /* current task's THREAD (phys) */ oris r9,r9,MSR_VEC@h +#ifdef CONFIG_VMAP_STACK + tovirt(r5, r5) +#endif #else ld r4,PACACURRENT(r13) addi r5,r4,THREAD /* Get THREAD */
arch/powerpc/mm/book3s32/hash_low.S +29 -17
111 lines not on this page · raw message
arch/powerpc/mm/book3s32/mmu.c +7 -2
@@ -413,6 +413,7 @@ void __init MMU_init_hw(void)void __init MMU_init_hw_patch (void ) {unsigned int hmask = Hash_mask >> (16 -LG_HPTEG_SIZE ); +unsigned int hash;if (ppc_md.progress) ppc_md.progress ("hash:patch" ,0x345 );
@@ -424,8 +425,12 @@ void __init MMU_init_hw_patch(void)/* * Patch up the instructions in hashtable.S:create_hpte */ -modify_instruction_site (&patch__hash_page_A0,0xffff , - ((unsigned int )Hash -PAGE_OFFSET ) >>16 ); +if (IS_ENABLED (CONFIG_VMAP_STACK )) + hash = (unsigned int )Hash; +else + hash = (unsigned int )Hash -PAGE_OFFSET ; + +modify_instruction_site (&patch__hash_page_A0,0xffff , hash >>16 );modify_instruction_site (&patch__hash_page_A1,0x7c0 , hash_mb <<6 );modify_instruction_site (&patch__hash_page_A2,0x7c0 , hash_mb2 <<6 );modify_instruction_site (&patch__hash_page_B,0xffff , hmask);
arch/powerpc/platforms/Kconfig.cputype +2 -0
@@ -31,12 +31,14 @@ config PPC_BOOK3S_6xx select PPC_HAVE_PMU_SUPPORT select PPC_HAVE_KUEP select PPC_HAVE_KUAP + select HAVE_ARCH_VMAP_STACK config PPC_BOOK3S_601 bool "PowerPC 601" select PPC_BOOK3S_32 select PPC_FPU select PPC_HAVE_KUAP + select HAVE_ARCH_VMAP_STACK config PPC_85xx bool "Freescale 85xx"
Signature · 1 line
2.13.3