powerpc/32: Reduce head complexity and re-activate MMU earlier
Subsystems (2)
PATCH v2 4/14 powerpc/40x: Change CRITICAL_EXCEPTION_PROLOG macro to a gas macro
In reply to Christophe LeroyChange CRITICAL_EXCEPTION_PROLOG macro to a gas macro to remove the ugly ; and \ on each line.
--- arch/powerpc/kernel/head_40x.S | 71 +++++++++++++++++----------------- 1 file changed, 36 insertions(+), 35 deletions(-)
arch/powerpc/kernel/head_40x.S +36 -35
@@ -100,42 +100,43 @@ _ENTRY(saved_ksp_limit) * Instead we use a couple of words of memory at low physical addresses. * This is OK since we don't support SMP on these processors. */ -#define CRITICAL_EXCEPTION_PROLOG \ - stw r10,crit_r10@l(0); /* save two registers to work with */\ - stw r11,crit_r11@l(0); \ - mfcr r10; /* save CR in r10 for now */\ - mfspr r11,SPRN_SRR3; /* check whether user or kernel */\ - andi. r11,r11,MSR_PR; \ - lis r11,critirq_ctx@ha; \ - tophys(r11,r11); \ - lwz r11,critirq_ctx@l(r11); \ - beq 1f; \ - /* COMING FROM USER MODE */ \ - mfspr r11,SPRN_SPRG_THREAD; /* if from user, start at top of */\ - lwz r11,TASK_STACK-THREAD(r11); /* this thread's kernel stack */\ -1: addi r11,r11,THREAD_SIZE-INT_FRAME_SIZE; /* Alloc an excpt frm */\ - tophys(r11,r11); \ - stw r10,_CCR(r11); /* save various registers */\ - stw r12,GPR12(r11); \ - stw r9,GPR9(r11); \ - mflr r10; \ - stw r10,_LINK(r11); \ - mfspr r12,SPRN_DEAR; /* save DEAR and ESR in the frame */\ - stw r12,_DEAR(r11); /* since they may have had stuff */\ - mfspr r9,SPRN_ESR; /* in them at the point where the */\ - stw r9,_ESR(r11); /* exception was taken */\ - mfspr r12,SPRN_SRR2; \ - stw r1,GPR1(r11); \ - mfspr r9,SPRN_SRR3; \ - stw r1,0(r11); \ - tovirt(r1,r11); \ - rlwinm r9,r9,0,14,12; /* clear MSR_WE (necessary?) */\ - stw r0,GPR0(r11); \ - lis r10, STACK_FRAME_REGS_MARKER@ha; /* exception frame marker */\ - addi r10, r10, STACK_FRAME_REGS_MARKER@l; \ - stw r10, 8(r11); \ - SAVE_4GPRS(3, r11); \ +.macro CRITICAL_EXCEPTION_PROLOG + stw r10,crit_r10@l(0) /* save two registers to work with */ + stw r11,crit_r11@l(0) + mfcr r10 /* save CR in r10 for now */ + mfspr r11,SPRN_SRR3 /* check whether user or kernel */ + andi. r11,r11,MSR_PR + lis r11,critirq_ctx@ha + tophys(r11,r11) + lwz r11,critirq_ctx@l(r11) + beq 1f + /* COMING FROM USER MODE */ + mfspr r11,SPRN_SPRG_THREAD /* if from user, start at top of */ + lwz r11,TASK_STACK-THREAD(r11) /* this thread's kernel stack */ +1: addi r11,r11,THREAD_SIZE-INT_FRAME_SIZE /* Alloc an excpt frm */ + tophys(r11,r11) + stw r10,_CCR(r11) /* save various registers */ + stw r12,GPR12(r11) + stw r9,GPR9(r11) + mflr r10 + stw r10,_LINK(r11) + mfspr r12,SPRN_DEAR /* save DEAR and ESR in the frame */ + stw r12,_DEAR(r11) /* since they may have had stuff */ + mfspr r9,SPRN_ESR /* in them at the point where the */ + stw r9,_ESR(r11) /* exception was taken */ + mfspr r12,SPRN_SRR2 + stw r1,GPR1(r11) + mfspr r9,SPRN_SRR3 + stw r1,0(r11) + tovirt(r1,r11) + rlwinm r9,r9,0,14,12 /* clear MSR_WE (necessary?) */ + stw r0,GPR0(r11) + lis r10, STACK_FRAME_REGS_MARKER@ha /* exception frame marker */ + addi r10, r10, STACK_FRAME_REGS_MARKER@l + stw r10, 8(r11) + SAVE_4GPRS(3, r11) SAVE_2GPRS(7, r11) +.endm /* * State at this point:
Signature · 1 line
2.25.0
PATCH v2 2/14 NOT TO BE MERGED - Squash of patches 2-7 of v6 series "powerpc: interrupt wrappers"
In reply to Christophe Leroypowerpc/32s: move DABR match out of handle_page_fault
handle_page_fault() has some code dedicated to book3s/32 to call do_break() when the DSI is a DABR match.
On other platforms, do_break() is handled separately.
Do the same for book3s/32, do it earlier in the process of DSI.
This change also avoid doing the test on ISI.
powerpc/64s: move DABR match out of handle_page_fault
Similar to the 32/s change, move the test and call to the do_break handler to the DSI.
powerpc/64s: move the hash fault handling logic to C
The fault handling still has some complex logic particularly around hash table handling, in asm. Implement most of this in C.
powerpc: remove arguments from fault handler functions
Make mm fault handlers all just take the pt_regs * argument and load DAR/DSISR from that. Make those that return a value return long.
This is done to make the function signatures match other handlers, which will help with a future patch to add wrappers. Explicit arguments could be added for performance but that would require more wrapper macro variants.
powerpc: do_break get registers from regs
Similar to the previous patch this makes interrupt handler function types more regular so they can be wrapped with the next patch.
powerpc: bad_page_fault get registers from regs
Similar to the previous patch this makes interrupt handler function types more regular so they can be wrapped with the next patch.
--- arch/powerpc/include/asm/asm-prototypes.h | 4 +- arch/powerpc/include/asm/book3s/64/mmu-hash.h | 1 + arch/powerpc/include/asm/bug.h | 7 +- arch/powerpc/include/asm/debug.h | 3 +- arch/powerpc/kernel/entry_32.S | 25 +-- arch/powerpc/kernel/exceptions-64e.S | 5 +- arch/powerpc/kernel/exceptions-64s.S | 164 +++++------------- arch/powerpc/kernel/head_40x.S | 10 +- arch/powerpc/kernel/head_8xx.S | 11 +- arch/powerpc/kernel/head_book3s_32.S | 8 +- arch/powerpc/kernel/head_booke.h | 4 +- arch/powerpc/kernel/process.c | 7 +- arch/powerpc/kernel/traps.c | 2 +- arch/powerpc/mm/book3s64/hash_utils.c | 79 ++++++--- arch/powerpc/mm/book3s64/slb.c | 11 +- arch/powerpc/mm/fault.c | 18 +- arch/powerpc/platforms/8xx/machine_check.c | 2 +- 17 files changed, 142 insertions(+), 219 deletions(-)
arch/powerpc/include/asm/asm-prototypes.h +2 -2
@@ -82,8 +82,8 @@ void kernel_bad_stack(struct pt_regs *regs);void system_reset_exception (struct pt_regs *regs);void machine_check_exception (struct pt_regs *regs);void emulation_assist_interrupt (struct pt_regs *regs); -long do_slb_fault (struct pt_regs *regs,unsigned long ea); -void do_bad_slb_fault (struct pt_regs *regs,unsigned long ea,long err); +long do_slb_fault (struct pt_regs *regs); +void do_bad_slb_fault (struct pt_regs *regs);/* signals, syscalls and interrupts */ long sys_swapcontext(struct ucontext __user *old_ctx,
arch/powerpc/include/asm/book3s/64/mmu-hash.h +1 -0
@@ -454,6 +454,7 @@ static inline unsigned long hpt_hash(unsigned long vpn,#define HPTE_NOHPTE_UPDATE 0x2#define HPTE_USE_KERNEL_KEY 0x4 +long do_hash_fault (struct pt_regs *regs);extern int __hash_page_4K (unsigned long ea,unsigned long access,unsigned long vsid,pte_t *ptep,unsigned long trap,unsigned long flags,int ssize,int subpage_prot);
arch/powerpc/include/asm/bug.h +4 -3
@@ -111,9 +111,10 @@#ifndef __ASSEMBLY__struct pt_regs ; -extern int do_page_fault (struct pt_regs *,unsigned long ,unsigned long ); -extern void bad_page_fault (struct pt_regs *,unsigned long ,int ); -void __bad_page_fault (struct pt_regs *regs,unsigned long address,int sig); +long do_page_fault (struct pt_regs *); +void bad_page_fault (struct pt_regs *,int ); +void __bad_page_fault (struct pt_regs *regs,int sig); +void do_bad_page_fault_segv (struct pt_regs *regs);extern void _exception (int ,struct pt_regs *,int ,unsigned long );extern void _exception_pkey (struct pt_regs *,unsigned long ,int );extern void die (const char *,struct pt_regs *,long );
arch/powerpc/include/asm/debug.h +1 -2
@@ -52,8 +52,7 @@ extern void do_send_trap(struct pt_regs *regs, unsigned long address, unsigned long error_code, int brkpt);#else -extern void do_break (struct pt_regs *regs,unsigned long address, -unsigned long error_code); +void do_break (struct pt_regs *regs);#endif #endif /* _ASM_POWERPC_DEBUG_H */
arch/powerpc/kernel/entry_32.S +2 -23
@@ -276,8 +276,7 @@ reenable_mmu: * We save a bunch of GPRs, * r3 can be different from GPR3(r1) at this point, r9 and r11 * contains the old MSR and handler address respectively, - * r4 & r5 can contain page fault arguments that need to be passed - * along as well. r0, r6-r8, r12, CCR, CTR, XER etc... are left + * r0, r4-r8, r12, CCR, CTR, XER etc... are left * clobbered as they aren't useful past this point. */
@@ -285,15 +284,11 @@ reenable_mmu: stw r9,8(r1) stw r11,12(r1) stw r3,16(r1) - stw r4,20(r1) - stw r5,24(r1) /* If we are disabling interrupts (normal case), simply log it with * lockdep */ 1: bl trace_hardirqs_off - lwz r5,24(r1) - lwz r4,20(r1) lwz r3,16(r1) lwz r11,12(r1) lwz r9,8(r1)
@@ -670,10 +665,6 @@ ppc_swapcontext: .globl handle_page_fault handle_page_fault: addi r3,r1,STACK_FRAME_OVERHEAD -#ifdef CONFIG_PPC_BOOK3S_32 - andis. r0,r5,DSISR_DABRMATCH@h - bne- handle_dabr_fault -#endif bl do_page_fault cmpwi r3,0 beq+ ret_from_except
@@ -681,23 +672,11 @@ handle_page_fault: lwz r0,_TRAP(r1) clrrwi r0,r0,1 stw r0,_TRAP(r1) - mr r5,r3 + mr r4,r3 /* err arg for bad_page_fault */ addi r3,r1,STACK_FRAME_OVERHEAD - lwz r4,_DAR(r1) bl __bad_page_fault b ret_from_except_full -#ifdef CONFIG_PPC_BOOK3S_32 - /* We have a data breakpoint exception - handle it */ -handle_dabr_fault: - SAVE_NVGPRS(r1) - lwz r0,_TRAP(r1) - clrrwi r0,r0,1 - stw r0,_TRAP(r1) - bl do_break - b ret_from_except_full -#endif - /* * This routine switches between two different tasks. The process * state of one is saved on its kernel stack. Then the state
arch/powerpc/kernel/exceptions-64e.S +1 -4
@@ -1011,8 +1011,6 @@ storage_fault_common: std r14,_DAR(r1) std r15,_DSISR(r1) addi r3,r1,STACK_FRAME_OVERHEAD - mr r4,r14 - mr r5,r15 ld r14,PACA_EXGEN+EX_R14(r13) ld r15,PACA_EXGEN+EX_R15(r13) bl do_page_fault
@@ -1020,9 +1018,8 @@ storage_fault_common: bne- 1f b ret_from_except_lite 1: bl save_nvgprs - mr r5,r3 + mr r4,r3 addi r3,r1,STACK_FRAME_OVERHEAD - ld r4,_DAR(r1) bl __bad_page_fault b ret_from_except
arch/powerpc/kernel/exceptions-64s.S +40 -124
228 lines not on this page · raw message
arch/powerpc/kernel/head_40x.S +5 -5
@@ -179,9 +179,9 @@ _ENTRY(saved_ksp_limit) */ START_EXCEPTION(0x0300, DataStorage) EXCEPTION_PROLOG - mfspr r5, SPRN_ESR /* Grab the ESR, save it, pass arg3 */ + mfspr r5, SPRN_ESR /* Grab the ESR, save it */ stw r5, _ESR(r11) - mfspr r4, SPRN_DEAR /* Grab the DEAR, save it, pass arg2 */ + mfspr r4, SPRN_DEAR /* Grab the DEAR, save it */ stw r4, _DEAR(r11) EXC_XFER_LITE(0x300, handle_page_fault)
@@ -191,9 +191,9 @@ _ENTRY(saved_ksp_limit) */ START_EXCEPTION(0x0400, InstructionAccess) EXCEPTION_PROLOG - mr r4,r12 /* Pass SRR0 as arg2 */ - stw r4, _DEAR(r11) - li r5,0 /* Pass zero as arg3 */ + li r5,0 + stw r5, _ESR(r11) /* Zero ESR */ + stw r12, _DEAR(r11) /* SRR0 as DEAR */ EXC_XFER_LITE(0x400, handle_page_fault) /* 0x0500 - External Interrupt Exception */
arch/powerpc/kernel/head_8xx.S +5 -6
@@ -312,14 +312,14 @@ DataStoreTLBMiss: . = 0x1300 InstructionTLBError: EXCEPTION_PROLOG - mr r4,r12 andis. r5,r9,DSISR_SRR1_MATCH_32S@h /* Filter relevant SRR1 bits */ andis. r10,r9,SRR1_ISI_NOPT@h beq+ .Litlbie - tlbie r4 + tlbie r12 /* 0x400 is InstructionAccess exception, needed by bad_page_fault() */ .Litlbie: - stw r4, _DAR(r11) + stw r12, _DAR(r11) + stw r5, _DSISR(r11) EXC_XFER_LITE(0x400, handle_page_fault) /* This is the data TLB error on the MPC8xx. This could be due to
@@ -364,10 +364,9 @@ do_databreakpoint: addi r3,r1,STACK_FRAME_OVERHEAD mfspr r4,SPRN_BAR stw r4,_DAR(r11) -#ifdef CONFIG_VMAP_STACK - lwz r5,_DSISR(r11) -#else +#ifndef CONFIG_VMAP_STACK mfspr r5,SPRN_DSISR + stw r5,_DSISR(r11) #endif EXC_XFER_STD(0x1c00, do_break)
arch/powerpc/kernel/head_book3s_32.S +5 -3
@@ -371,9 +371,9 @@ BEGIN_MMU_FTR_SECTION END_MMU_FTR_SECTION_IFSET(MMU_FTR_HPTE_TABLE) #endif #endif /* CONFIG_VMAP_STACK */ -1: mr r4,r12 andis. r5,r9,DSISR_SRR1_MATCH_32S@h /* Filter relevant SRR1 bits */ - stw r4, _DAR(r11) + stw r5, _DSISR(r11) + stw r12, _DAR(r11) EXC_XFER_LITE(0x400, handle_page_fault) /* External interrupt */
@@ -695,11 +695,13 @@ handle_page_fault_tramp_1: #ifdef CONFIG_VMAP_STACK EXCEPTION_PROLOG_2 handle_dar_dsisr=1 #endif - lwz r4, _DAR(r11) lwz r5, _DSISR(r11) /* fall through */ handle_page_fault_tramp_2: + andis. r0, r5, DSISR_DABRMATCH@h + bne- 1f EXC_XFER_LITE(0x300, handle_page_fault) +1: EXC_XFER_STD(0x300, do_break) #ifdef CONFIG_VMAP_STACK #ifdef CONFIG_PPC_BOOK3S_604
arch/powerpc/kernel/head_booke.h +1 -3
@@ -476,9 +476,7 @@ ALT_FTR_SECTION_END_IFSET(CPU_FTR_EMB_HV)NORMAL_EXCEPTION_PROLOG (INST_STORAGE ); \mfspr r5,SPRN_ESR ;/* Grab the ESR and save it */ \stw r5,_ESR (r11 ); \ -mr r4,r12;/* Pass SRR0 as arg2 */ \ -stw r4,_DEAR (r11 ); \ - li r5,0; /* Pass zero as arg3 */ \ +stw r12,_DEAR (r11 );/* Pass SRR0 as arg2 */ \ EXC_XFER_LITE(0x0400, handle_page_fault)#define ALIGNMENT_EXCEPTION \
arch/powerpc/kernel/process.c +3 -4
@@ -659,11 +659,10 @@ static void do_break_handler(struct pt_regs *regs) } } -void do_break (struct pt_regs *regs,unsigned long address, -unsigned long error_code) +void do_break (struct pt_regs *regs) { current->thread.trap_nr =TRAP_HWBKPT ; -if (notify_die (DIE_DABR_MATCH ,"dabr_match" , regs, error_code, +if (notify_die (DIE_DABR_MATCH ,"dabr_match" , regs, regs->dsisr,11 ,SIGSEGV ) ==NOTIFY_STOP )return ;
@@ -681,7 +680,7 @@ void do_break (struct pt_regs *regs, unsigned long address,do_break_handler (regs);/* Deliver the signal to userspace */ - force_sig_fault(SIGTRAP, TRAP_HWBKPT, (void __user *)address); + force_sig_fault(SIGTRAP, TRAP_HWBKPT, (void __user *)regs->dar); }#endif /* CONFIG_PPC_ADV_DEBUG_REGS */
arch/powerpc/kernel/traps.c +1 -1
@@ -1612,7 +1612,7 @@ void alignment_exception(struct pt_regs *regs)if (user_mode (regs))_exception (sig, regs, code, regs->dar);else -bad_page_fault (regs, regs->dar, sig); +bad_page_fault (regs, sig); bail:exception_exit (prev_state);
arch/powerpc/mm/book3s64/hash_utils.c +51 -28
107 lines not on this page · raw message
arch/powerpc/mm/book3s64/slb.c +7 -4
@@ -813,8 +813,9 @@ static long slb_allocate_user(struct mm_struct *mm, unsigned long ea)return slb_insert_entry (ea, context, flags, ssize, false); } -long do_slb_fault (struct pt_regs *regs,unsigned long ea) +long do_slb_fault (struct pt_regs *regs) { +unsigned long ea = regs->dar;unsigned long id =get_region_id (ea);/* IRQs are not reconciled here, so can't check irqs_disabled */
@@ -865,13 +866,15 @@ long do_slb_fault(struct pt_regs *regs, unsigned long ea) } } -void do_bad_slb_fault (struct pt_regs *regs,unsigned long ea,long err) +void do_bad_slb_fault (struct pt_regs *regs) { +int err = regs->result; +if (err == -EFAULT ) {if (user_mode (regs)) -_exception (SIGSEGV , regs,SEGV_BNDERR , ea); +_exception (SIGSEGV , regs,SEGV_BNDERR , regs->dar);else -bad_page_fault (regs, ea,SIGSEGV ); +bad_page_fault (regs,SIGSEGV ); }else if (err == -EINVAL ) {unrecoverable_exception (regs); }else {
arch/powerpc/mm/fault.c +12 -6
@@ -542,12 +542,11 @@ static int __do_page_fault(struct pt_regs *regs, unsigned long address, }NOKPROBE_SYMBOL (__do_page_fault); -int do_page_fault (struct pt_regs *regs,unsigned long address, -unsigned long error_code) +long do_page_fault (struct pt_regs *regs) {const struct exception_table_entry *entry;enum ctx_state prev_state =exception_enter (); -int rc =__do_page_fault (regs, address, error_code); +int rc =__do_page_fault (regs, regs->dar, regs->dsisr);exception_exit (prev_state);if (likely (!rc))return 0 ;
@@ -567,7 +566,7 @@ NOKPROBE_SYMBOL(do_page_fault);* It is called from the DSI and ISI handlers in head.S and from some * of the procedures in traps.c. */ -void __bad_page_fault (struct pt_regs *regs,unsigned long address,int sig) +void __bad_page_fault (struct pt_regs *regs,int sig) {int is_write =page_fault_is_write (regs->dsisr);
@@ -605,7 +604,7 @@ void __bad_page_fault(struct pt_regs *regs, unsigned long address, int sig)die ("Kernel access of bad area" , regs, sig); } -void bad_page_fault (struct pt_regs *regs,unsigned long address,int sig) +void bad_page_fault (struct pt_regs *regs,int sig) {const struct exception_table_entry *entry;
@@ -614,5 +613,12 @@ void bad_page_fault(struct pt_regs *regs, unsigned long address, int sig)if (entry)instruction_pointer_set (regs,extable_fixup (entry));else -__bad_page_fault (regs, address, sig); +__bad_page_fault (regs, sig); } + +#ifdef CONFIG_PPC_BOOK3S_64 +void do_bad_page_fault_segv (struct pt_regs *regs) +{ +bad_page_fault (regs,SIGSEGV ); +} +#endif
arch/powerpc/platforms/8xx/machine_check.c +1 -1
@@ -26,7 +26,7 @@ int machine_check_8xx(struct pt_regs *regs)* to deal with that than having a wart in the mcheck handler. * -- BenH */ -bad_page_fault (regs, regs->dar,SIGBUS ); +bad_page_fault (regs,SIGBUS );return 1 ;#else return 0 ;
Signature · 1 line
2.25.0
PATCH v2 3/14 powerpc/40x: Don't use SPRN_SPRG_SCRATCH0/1 in TLB miss handlers
In reply to Christophe LeroySPRN_SPRG_SCRATCH5 is used to save SPRN_PID.SPRN_SPRG_SCRATCH6 is already available.
SPRN_PID is only 8 bits. We have r12 that contains CR. We only need to preserve CR0, so we have space available in r12 to save PID.
Keep PID in r12 and free up SPRN_SPRG_SCRATCH5.
Then In TLB miss handlers, instead of using SPRN_SPRG_SCRATCH0 and SPRN_SPRG_SCRATCH1, use SPRN_SPRG_SCRATCH5 and SPRN_SPRG_SCRATCH6 to avoid future conflicts with normal exception prologs.
--- arch/powerpc/kernel/head_40x.S | 39 ++++++++++++++++------------------ 1 file changed, 18 insertions(+), 21 deletions(-)
arch/powerpc/kernel/head_40x.S +18 -21
@@ -249,13 +249,13 @@ _ENTRY(saved_ksp_limit) * load TLB entries from the page table if they exist. */ START_EXCEPTION(0x1100, DTLBMiss) - mtspr SPRN_SPRG_SCRATCH0, r10 /* Save some working registers */ - mtspr SPRN_SPRG_SCRATCH1, r11 + mtspr SPRN_SPRG_SCRATCH5, r10 /* Save some working registers */ + mtspr SPRN_SPRG_SCRATCH6, r11 mtspr SPRN_SPRG_SCRATCH3, r12 mtspr SPRN_SPRG_SCRATCH4, r9 mfcr r12 mfspr r9, SPRN_PID - mtspr SPRN_SPRG_SCRATCH5, r9 + rlwimi r12, r9, 0, 0xff mfspr r10, SPRN_DEAR /* Get faulting address */ /* If we are faulting a kernel address, we have to use the
@@ -316,13 +316,12 @@ _ENTRY(saved_ksp_limit) /* The bailout. Restore registers to pre-exception conditions * and call the heavyweights to help us out. */ - mfspr r9, SPRN_SPRG_SCRATCH5 - mtspr SPRN_PID, r9 - mtcr r12 + mtspr SPRN_PID, r12 + mtcrf 0x80, r12 mfspr r9, SPRN_SPRG_SCRATCH4 mfspr r12, SPRN_SPRG_SCRATCH3 - mfspr r11, SPRN_SPRG_SCRATCH1 - mfspr r10, SPRN_SPRG_SCRATCH0 + mfspr r11, SPRN_SPRG_SCRATCH6 + mfspr r10, SPRN_SPRG_SCRATCH5 b DataStorage /* 0x1200 - Instruction TLB Miss Exception
@@ -330,13 +329,13 @@ _ENTRY(saved_ksp_limit) * registers and bailout to a different point. */ START_EXCEPTION(0x1200, ITLBMiss) - mtspr SPRN_SPRG_SCRATCH0, r10 /* Save some working registers */ - mtspr SPRN_SPRG_SCRATCH1, r11 + mtspr SPRN_SPRG_SCRATCH5, r10 /* Save some working registers */ + mtspr SPRN_SPRG_SCRATCH6, r11 mtspr SPRN_SPRG_SCRATCH3, r12 mtspr SPRN_SPRG_SCRATCH4, r9 mfcr r12 mfspr r9, SPRN_PID - mtspr SPRN_SPRG_SCRATCH5, r9 + rlwimi r12, r9, 0, 0xff mfspr r10, SPRN_SRR0 /* Get faulting address */ /* If we are faulting a kernel address, we have to use the
@@ -397,13 +396,12 @@ _ENTRY(saved_ksp_limit) /* The bailout. Restore registers to pre-exception conditions * and call the heavyweights to help us out. */ - mfspr r9, SPRN_SPRG_SCRATCH5 - mtspr SPRN_PID, r9 - mtcr r12 + mtspr SPRN_PID, r12 + mtcrf 0x80, r12 mfspr r9, SPRN_SPRG_SCRATCH4 mfspr r12, SPRN_SPRG_SCRATCH3 - mfspr r11, SPRN_SPRG_SCRATCH1 - mfspr r10, SPRN_SPRG_SCRATCH0 + mfspr r11, SPRN_SPRG_SCRATCH6 + mfspr r10, SPRN_SPRG_SCRATCH5 b InstructionAccess EXCEPTION(0x1300, Trap_13, unknown_exception, EXC_XFER_STD)
@@ -542,13 +540,12 @@ finish_tlb_load: /* Done...restore registers and get out of here. */ - mfspr r9, SPRN_SPRG_SCRATCH5 - mtspr SPRN_PID, r9 - mtcr r12 + mtspr SPRN_PID, r12 + mtcrf 0x80, r12 mfspr r9, SPRN_SPRG_SCRATCH4 mfspr r12, SPRN_SPRG_SCRATCH3 - mfspr r11, SPRN_SPRG_SCRATCH1 - mfspr r10, SPRN_SPRG_SCRATCH0 + mfspr r11, SPRN_SPRG_SCRATCH6 + mfspr r10, SPRN_SPRG_SCRATCH5 rfi /* Should sync shadow TLBs */ b . /* prevent prefetch past rfi */
Signature · 1 line
2.25.0
PATCH v2 1/14 powerpc/32s: Only build hash code when CONFIG_PPC_BOOK3S_604 is selected
In reply to Christophe LeroyIt is now possible to only build book3s/32 kernel for CPUs without hash table.
Opt out hash related code when CONFIG_PPC_BOOK3S_604 is not selected.
--- arch/powerpc/kernel/head_book3s_32.S | 12 ++++++++++++ arch/powerpc/mm/book3s32/Makefile | 4 +++- 2 files changed, 15 insertions(+), 1 deletion(-)
arch/powerpc/kernel/head_book3s_32.S +12 -0
@@ -295,6 +295,7 @@ MachineCheck: DO_KVM 0x300 DataAccess: #ifdef CONFIG_VMAP_STACK +#ifdef CONFIG_PPC_BOOK3S_604 BEGIN_MMU_FTR_SECTION mtspr SPRN_SPRG_SCRATCH2,r10 mfspr r10, SPRN_SPRG_THREAD
@@ -311,12 +312,14 @@ BEGIN_MMU_FTR_SECTION MMU_FTR_SECTION_ELSE b 1f ALT_MMU_FTR_SECTION_END_IFSET(MMU_FTR_HPTE_TABLE) +#endif 1: EXCEPTION_PROLOG_0 handle_dar_dsisr=1 EXCEPTION_PROLOG_1 b handle_page_fault_tramp_1 #else /* CONFIG_VMAP_STACK */ EXCEPTION_PROLOG handle_dar_dsisr=1 get_and_save_dar_dsisr_on_stack r4, r5, r11 +#ifdef CONFIG_PPC_BOOK3S_604 BEGIN_MMU_FTR_SECTION andis. r0, r5, (DSISR_BAD_FAULT_32S | DSISR_DABRMATCH)@h bne handle_page_fault_tramp_2 /* if not, try to put a PTE */
@@ -324,8 +327,11 @@ BEGIN_MMU_FTR_SECTION bl hash_page b handle_page_fault_tramp_1 MMU_FTR_SECTION_ELSE +#endif b handle_page_fault_tramp_2 +#ifdef CONFIG_PPC_BOOK3S_604 ALT_MMU_FTR_SECTION_END_IFSET(MMU_FTR_HPTE_TABLE) +#endif #endif /* CONFIG_VMAP_STACK */ /* Instruction access exception. */
@@ -341,12 +347,14 @@ InstructionAccess: mfspr r11, SPRN_SRR1 /* check whether user or kernel */ stw r11, SRR1(r10) mfcr r10 +#ifdef CONFIG_PPC_BOOK3S_604 BEGIN_MMU_FTR_SECTION andis. r11, r11, SRR1_ISI_NOPT@h /* no pte found? */ bne hash_page_isi .Lhash_page_isi_cont: mfspr r11, SPRN_SRR1 /* check whether user or kernel */ END_MMU_FTR_SECTION_IFSET(MMU_FTR_HPTE_TABLE) +#endif andi. r11, r11, MSR_PR EXCEPTION_PROLOG_1
@@ -357,9 +365,11 @@ END_MMU_FTR_SECTION_IFSET(MMU_FTR_HPTE_TABLE) beq 1f /* if so, try to put a PTE */ li r3,0 /* into the hash table */ mr r4,r12 /* SRR0 is fault address */ +#ifdef CONFIG_PPC_BOOK3S_604 BEGIN_MMU_FTR_SECTION bl hash_page END_MMU_FTR_SECTION_IFSET(MMU_FTR_HPTE_TABLE) +#endif #endif /* CONFIG_VMAP_STACK */ 1: mr r4,r12 andis. r5,r9,DSISR_SRR1_MATCH_32S@h /* Filter relevant SRR1 bits */
@@ -692,6 +702,7 @@ handle_page_fault_tramp_2: EXC_XFER_LITE(0x300, handle_page_fault) #ifdef CONFIG_VMAP_STACK +#ifdef CONFIG_PPC_BOOK3S_604 .macro save_regs_thread thread stw r0, THR0(\thread) stw r3, THR3(\thread)
@@ -763,6 +774,7 @@ fast_hash_page_return: mfspr r11, SPRN_SPRG_SCRATCH1 mfspr r10, SPRN_SPRG_SCRATCH0 rfi +#endif /* CONFIG_PPC_BOOK3S_604 */ stack_overflow: vmap_stack_overflow_exception
arch/powerpc/mm/book3s32/Makefile +3 -1
@@ -6,4 +6,6 @@ ifdef CONFIG_KASAN CFLAGS_mmu.o += -DDISABLE_BRANCH_PROFILING endif -obj-y += mmu.o hash_low.o mmu_context.o tlb.o nohash_low.o +obj-y += mmu.o mmu_context.o +obj-$(CONFIG_PPC_BOOK3S_603) += nohash_low.o +obj-$(CONFIG_PPC_BOOK3S_604) += hash_low.o tlb.o
Signature · 1 line
2.25.0
v2 cover powerpc/32: Reduce head complexity and re-activate MMU earlier
This series is a first step on the way to C syscall/exception entry/exit.
This series aims at reducing exception/syscall prologs complexity. It also brings earlier MMU re-activation.
This series is based on Nick's v6 series "powerpc: interrupt wrappers". It takes benefit of the removal of traps arguments (patches 2-7 of that series). I have squashed those patches as second patch of my series in order to please test robots. My series cleanly applies on top of entire Nick's series.
v2 has been reworked in order to apply to all PPC32, including BOOKE and 40x.
At the time being, we have two pathes in the prologs: one for when we have VMAP stack and one when we don't.
When VMAP stack is supported, there is special prolog code to allow accessing stack with MMU on.
That code that accesses VM stack with MMU on is also able to access linear memory, so it can also access non VM stack with MMU on.
CONFIG_VMAP_STACK as been on by default on 6xx and 8xx for several kernel releases now, so it is known to work.
On the 8xx, null_syscall runs in 292 cycles with VMAP_STACK and in 296 cycles without VMAP stack.
On the 832x, null_syscall runs in 224 cycles with VMAP_STACK and in 213 cycles without VMAP stack.
By removing the old non VMAP stack code, and using the same prolog regardless of the activation of VMAP stacks, we make the code a lot simplier and reduce the number of test cases.
BOOKE has MMU always on, so there is no change needed for that. To allow removal of the old non VMAP stack code, only 40x need to get adapted to support earlier MMU activation. That's what patches 3-8 are for.
Once this is done, we easily go one step further and re-activate Instruction translation at the same time as data translation.
At the end, null_syscall runs in 286 cycles on the 8xx and in 216 cycles on the 832x
Changes in v2:
- Implemented early MMU activation also on 40x
- Added BOOKE in the loop
- Removed the patches that replace r11 by r1 all over the place (too much churn for very small benefit for now)
Christophe Leroy (14):
powerpc/32s: Only build hash code when CONFIG_PPC_BOOK3S_604 is
selected
NOT TO BE MERGED - Squash of patches 2-7 of v6 series "powerpc:
interrupt wrappers"
powerpc/40x: Don't use SPRN_SPRG_SCRATCH0/1 in TLB miss handlers powerpc/40x: Change CRITICAL_EXCEPTION_PROLOG macro to a gas macro powerpc/40x: Save SRR0/SRR1 and r10/r11 earlier in critical exception powerpc/40x: Reorder a few instructions in critical exception prolog powerpc/40x: Prepare for enabling MMU in critical exception prolog powerpc/40x: Prepare normal exception handler for enabling MMU early powerpc/32: Preserve cr1 in exception prolog stack check powerpc/32: Use LOAD_REG_IMMEDIATE() to load MSR values powerpc/32: Always enable data translation in exception prolog powerpc/32: Enable instruction translation at the same time as data
translation
powerpc/32: Remove msr argument in EXC_XFER_TEMPLATE() powerpc/32: Use fast instructions to change MSR EE/RI when available
arch/powerpc/include/asm/asm-prototypes.h | 4 +- arch/powerpc/include/asm/book3s/64/mmu-hash.h | 1 + arch/powerpc/include/asm/bug.h | 7 +- arch/powerpc/include/asm/debug.h | 3 +- arch/powerpc/include/asm/hw_irq.h | 46 +++++ arch/powerpc/include/asm/processor.h | 4 +- arch/powerpc/kernel/asm-offsets.c | 2 - arch/powerpc/kernel/entry_32.S | 128 ++++---------- arch/powerpc/kernel/exceptions-64e.S | 5 +- arch/powerpc/kernel/exceptions-64s.S | 164 +++++------------ arch/powerpc/kernel/fpu.S | 2 - arch/powerpc/kernel/head_32.h | 167 ++++-------------- arch/powerpc/kernel/head_40x.S | 161 +++++++++-------- arch/powerpc/kernel/head_8xx.S | 26 +-- arch/powerpc/kernel/head_book3s_32.S | 52 ++---- arch/powerpc/kernel/head_booke.h | 20 +-- arch/powerpc/kernel/idle_6xx.S | 12 +- arch/powerpc/kernel/idle_e500.S | 4 +- arch/powerpc/kernel/process.c | 7 +- arch/powerpc/kernel/traps.c | 2 +- arch/powerpc/kernel/vector.S | 2 - arch/powerpc/mm/book3s32/Makefile | 4 +- arch/powerpc/mm/book3s32/hash_low.S | 14 -- arch/powerpc/mm/book3s64/hash_utils.c | 79 ++++++--- arch/powerpc/mm/book3s64/slb.c | 11 +- arch/powerpc/mm/fault.c | 18 +- arch/powerpc/platforms/8xx/machine_check.c | 2 +- 27 files changed, 354 insertions(+), 593 deletions(-)
Signature · 1 line
2.25.0
PATCH v2 9/14 powerpc/32: Preserve cr1 in exception prolog stack check
THREAD_ALIGN_SHIFT = THREAD_SHIFT + 1 = PAGE_SHIFT + 1 Maximum PAGE_SHIFT is 18 for 256k pages so THREAD_ALIGN_SHIFT is 19 at the maximum.
No need to clobber cr1, it can be preserved when moving r1 into CR when we check stack overflow.
--- arch/powerpc/kernel/head_32.h | 2 +- arch/powerpc/kernel/head_book3s_32.S | 6 ------ 2 files changed, 1 insertion(+), 7 deletions(-)
arch/powerpc/kernel/head_32.h +1 -1
@@ -55,7 +55,7 @@ lwz r1,TASK_STACK-THREAD(r1) addi r1, r1, THREAD_SIZE - INT_FRAME_SIZE 1: - mtcrf 0x7f, r1 + mtcrf 0x3f, r1 bt 32 - THREAD_ALIGN_SHIFT, stack_overflow #else subi r11, r1, INT_FRAME_SIZE /* use r1 if kernel */
arch/powerpc/kernel/head_book3s_32.S +0 -6
@@ -278,12 +278,6 @@ MachineCheck: 7: EXCEPTION_PROLOG_2 addi r3,r1,STACK_FRAME_OVERHEAD #ifdef CONFIG_PPC_CHRP -#ifdef CONFIG_VMAP_STACK - mfspr r4, SPRN_SPRG_THREAD - tovirt(r4, r4) - lwz r4, RTAS_SP(r4) - cmpwi cr1, r4, 0 -#endif beq cr1, machine_check_tramp twi 31, 0, 0 #else
Signature · 1 line
2.25.0
PATCH v2 10/14 powerpc/32: Use LOAD_REG_IMMEDIATE() to load MSR values
In reply to Christophe Leroy40x MSR value doesn't fit on 15 bits.
LOAD_REG_IMMEDIATE() in places that will be used also with 40x in the next patch.
--- arch/powerpc/kernel/head_32.h | 4 ++-- 1 file changed, 2 insertions(+), 2 deletions(-)
arch/powerpc/kernel/head_32.h +2 -2
@@ -69,7 +69,7 @@ .macro EXCEPTION_PROLOG_2 handle_dar_dsisr=0#ifdef CONFIG_VMAP_STACK - li r11, MSR_KERNEL & ~(MSR_IR | MSR_RI) /* can take DTLB miss */ + LOAD_REG_IMMEDIATE(r11, MSR_KERNEL & ~(MSR_IR | MSR_RI)) /* can take DTLB miss */ mtmsr r11 isyncmfspr r11,SPRN_SPRG_SCRATCH2
@@ -134,7 +134,7 @@ lwz r1,TASK_STACK-THREAD(r12) beq- 99f addi r1, r1, THREAD_SIZE - INT_FRAME_SIZE - li r10, MSR_KERNEL & ~(MSR_IR | MSR_RI) /* can take DTLB miss */ + LOAD_REG_IMMEDIATE(r10, MSR_KERNEL & ~(MSR_IR | MSR_RI)) /* can take DTLB miss */ mtmsr r10 isync tovirt(r12, r12)
Signature · 1 line
2.25.0
PATCH v2 12/14 powerpc/32: Enable instruction translation at the same time as data translation
In reply to Christophe LeroyOn 40x and 8xx, kernel text is pinned.
On book3s/32, kernel text is mapped by BATs.
Enable instruction translation at the same time as data translation, it makes things simpler.
In syscall handler, MSR_RI can also be set at the same time because srr0/srr1 are already saved and r1 is set properly.
On booke, translation is always on, so at the end all PPC32 have translation on early.
Also update comment in power_save_ppc32_restore().
--- arch/powerpc/kernel/entry_32.S | 44 ++++++++++---------------------- arch/powerpc/kernel/head_32.h | 39 ++++++++++++---------------- arch/powerpc/kernel/head_40x.S | 10 +++++--- arch/powerpc/kernel/head_booke.h | 7 ++--- 4 files changed, 39 insertions(+), 61 deletions(-)
arch/powerpc/kernel/entry_32.S +13 -31
@@ -198,12 +198,8 @@ transfer_to_handler: transfer_to_handler_cont: 3: mflr r9 - tovirt(r9, r9) lwz r11,0(r9) /* virtual address of handler */ lwz r9,4(r9) /* where to go when done */ -#if defined(CONFIG_PPC_8xx) && defined(CONFIG_PERF_EVENTS) - mtspr SPRN_NRI, r0 -#endif #ifdef CONFIG_TRACE_IRQFLAGS /* * When tracing IRQ state (lockdep) we enable the MMU before we call
@@ -219,13 +215,9 @@ transfer_to_handler_cont: /* MSR isn't changing, just transition directly */ #endif - mtspr SPRN_SRR0,r11 - mtspr SPRN_SRR1,r10 + mtctr r11 mtlr r9 - rfi /* jump to handler, enable MMU */ -#ifdef CONFIG_40x - b . /* Prevent prefetch past rfi */ -#endif + bctr /* jump to handler */ #if defined (CONFIG_PPC_BOOK3S_32) || defined(CONFIG_E500) 4: rlwinm r12,r12,0,~_TLF_NAPPING
@@ -245,21 +237,7 @@ _ASM_NOKPROBE_SYMBOL(transfer_to_handler) _ASM_NOKPROBE_SYMBOL(transfer_to_handler_cont) #ifdef CONFIG_TRACE_IRQFLAGS -1: /* MSR is changing, re-enable MMU so we can notify lockdep. We need to - * keep interrupts disabled at this point otherwise we might risk - * taking an interrupt before we tell lockdep they are enabled. - */ - lis r12,reenable_mmu@h - ori r12,r12,reenable_mmu@l - LOAD_REG_IMMEDIATE(r0, MSR_KERNEL) - mtspr SPRN_SRR0,r12 - mtspr SPRN_SRR1,r0 - rfi -#ifdef CONFIG_40x - b . /* Prevent prefetch past rfi */ -#endif - -reenable_mmu: +1: /* * We save a bunch of GPRs, * r3 can be different from GPR3(r1) at this point, r9 and r11
@@ -1334,16 +1312,20 @@ _GLOBAL(enter_rtas) mtspr SPRN_SRR1,r9 rfi 1: - li r0, MSR_KERNEL & ~MSR_IR /* can take DTLB miss */ - mtmsr r0 - isync + lis r8, 1f@h + ori r8, r8, 1f@l + LOAD_REG_IMMEDIATE(r9,MSR_KERNEL) + mtspr SPRN_SRR0,r8 + mtspr SPRN_SRR1,r9 + rfi /* Reactivate MMU translation */ +1: lwz r8,INT_FRAME_SIZE+4(r1) /* get return address */ lwz r9,8(r1) /* original msr value */ addi r1,r1,INT_FRAME_SIZE li r0,0 stw r0, THREAD + RTAS_SP(r2) - mtspr SPRN_SRR0,r8 - mtspr SPRN_SRR1,r9 - rfi /* return to caller */ + mtlr r8 + mtmsr r9 + blr /* return to caller */ _ASM_NOKPROBE_SYMBOL(enter_rtas) #endif /* CONFIG_PPC_RTAS */
arch/powerpc/kernel/head_32.h +17 -22
@@ -57,10 +57,14 @@ .endm .macro EXCEPTION_PROLOG_2 handle_dar_dsisr=0 - LOAD_REG_IMMEDIATE(r11, MSR_KERNEL & ~(MSR_IR | MSR_RI)) /* can take DTLB miss */ - mtmsr r11 - isync + LOAD_REG_IMMEDIATE(r11, MSR_KERNEL & ~MSR_RI) /* re-enable MMU */ + mtspr SPRN_SRR1, r11 + lis r11, 1f@h + ori r11, r11, 1f@l + mtspr SPRN_SRR0, r11 mfspr r11, SPRN_SPRG_SCRATCH2 + rfi +1: stw r11,GPR1(r1) stw r11,0(r1) mr r11, r1
@@ -86,7 +90,7 @@#ifdef CONFIG_40x rlwinm r9,r9,0,14,12 /* clear MSR_WE (necessary?) */ #else - li r10, MSR_KERNEL & ~MSR_IR /* can take exceptions */ + li r10, MSR_KERNEL /* can take exceptions */ mtmsr r10 /* (except for mach check in rtas) */ #endif stw r0,GPR0(r11)
@@ -107,9 +111,13 @@ lwz r1,TASK_STACK-THREAD(r12) beq- 99f addi r1, r1, THREAD_SIZE - INT_FRAME_SIZE - LOAD_REG_IMMEDIATE(r10, MSR_KERNEL & ~(MSR_IR | MSR_RI)) /* can take DTLB miss */ - mtmsr r10 - isync + LOAD_REG_IMMEDIATE(r10, MSR_KERNEL) /* can take exceptions */ + mtspr SPRN_SRR1, r10 + lis r10, 1f@h + ori r10, r10, 1f@l + mtspr SPRN_SRR0, r10 + rfi +1: tovirt(r12, r12) stw r11,GPR1(r1) stw r11,0(r1)
@@ -123,9 +131,6 @@ stw r10,_CCR(r11) /* save registers */#ifdef CONFIG_40x rlwinm r9,r9,0,14,12 /* clear MSR_WE (necessary?) */ -#else - LOAD_REG_IMMEDIATE(r10, MSR_KERNEL & ~MSR_IR) /* can take exceptions */ - mtmsr r10 /* (except for mach check in rtas) */ #endif lis r10,STACK_FRAME_REGS_MARKER@ha /* exception frame marker */ stw r2,GPR2(r11)
@@ -162,8 +167,6 @@#endif 3: - lis r11, transfer_to_syscall@h - ori r11, r11, transfer_to_syscall@l#ifdef CONFIG_TRACE_IRQFLAGS /* * If MSR is changing we need to keep interrupts disabled at this point
@@ -175,15 +178,8 @@#else LOAD_REG_IMMEDIATE(r10, MSR_KERNEL | MSR_EE)#endif -#if defined(CONFIG_PPC_8xx) && defined(CONFIG_PERF_EVENTS ) - mtspr SPRN_NRI, r0 -#endif -mtspr SPRN_SRR1 ,r10 - mtspr SPRN_SRR0,r11 - rfi /* jump to handler, enable MMU */ -#ifdef CONFIG_40x - b . /* Prevent prefetch past rfi */ -#endif + mtmsr r10 +b transfer_to_syscall/* jump to handler */ 99: b ret_from_kernel_syscall .endm
@@ -220,7 +216,6 @@#define EXC_XFER_TEMPLATE (hdlr, trap, msr, tfer, ret) \ li r10,trap; \ stw r10,_TRAP(r11); \ - LOAD_REG_IMMEDIATE(r10, msr); \ bl tfer; \ .long hdlr; \ .long ret
arch/powerpc/kernel/head_40x.S +7 -3
@@ -128,9 +128,13 @@ _ENTRY(saved_ksp_limit) lwz r11,TASK_STACK-THREAD(r11) /* this thread's kernel stack */ 1: stw r1,crit_r1@l(0) addi r1,r11,THREAD_SIZE-INT_FRAME_SIZE /* Alloc an excpt frm */ - LOAD_REG_IMMEDIATE(r11,MSR_KERNEL & ~(MSR_IR | MSR_RI)) - mtmsr r11 - isync + LOAD_REG_IMMEDIATE(r11, MSR_KERNEL & ~(MSR_ME|MSR_DE|MSR_CE)) /* re-enable MMU */ + mtspr SPRN_SRR1, r11 + lis r11, 1f@h + ori r11, r11, 1f@l + mtspr SPRN_SRR0, r11 + rfi +1: lwz r11,crit_r1@l(0) stw r11,GPR1(r1) stw r11,0(r1)
arch/powerpc/kernel/head_booke.h +2 -5
@@ -157,8 +157,6 @@ ALT_FTR_SECTION_END_IFSET(CPU_FTR_EMB_HV) stw r12,4(r11) 3: - lis r11, transfer_to_syscall@h - ori r11, r11, transfer_to_syscall@l#ifdef CONFIG_TRACE_IRQFLAGS /* * If MSR is changing we need to keep interrupts disabled at this point
@@ -172,9 +170,8 @@ ALT_FTR_SECTION_END_IFSET(CPU_FTR_EMB_HV) lis r10, (MSR_KERNEL | MSR_EE)@h ori r10, r10, (MSR_KERNEL | MSR_EE)@l #endif - mtspr SPRN_SRR1,r10 - mtspr SPRN_SRR0,r11 - rfi /* jump to handler, enable MMU */ + mtmsr r10 + b transfer_to_syscall /* jump to handler */ 99: b ret_from_kernel_syscall .endm
Signature · 1 line
2.25.0
PATCH v2 13/14 powerpc/32: Remove msr argument in EXC_XFER_TEMPLATE()
In reply to Christophe Leroymsr argument is not used anymore, remove it.
--- arch/powerpc/kernel/head_32.h | 8 +++----- arch/powerpc/kernel/head_40x.S | 5 +---- 2 files changed, 4 insertions(+), 9 deletions(-)
arch/powerpc/kernel/head_32.h +3 -5
@@ -213,7 +213,7 @@addi r3,r1,STACK_FRAME_OVERHEAD ; \ xfer(n, hdlr) -#define EXC_XFER_TEMPLATE (hdlr, trap, msr, tfer, ret) \ +#define EXC_XFER_TEMPLATE (hdlr, trap, tfer, ret) \ li r10,trap; \ stw r10,_TRAP(r11); \ bl tfer; \
@@ -221,12 +221,10 @@ .long ret#define EXC_XFER_STD (n, hdlr) \ - EXC_XFER_TEMPLATE(hdlr, n, MSR_KERNEL, transfer_to_handler_full, \ - ret_from_except_full) + EXC_XFER_TEMPLATE(hdlr, n, transfer_to_handler_full, ret_from_except_full)#define EXC_XFER_LITE (n, hdlr) \ - EXC_XFER_TEMPLATE(hdlr, n+1, MSR_KERNEL, transfer_to_handler, \ - ret_from_except) + EXC_XFER_TEMPLATE(hdlr, n + 1, transfer_to_handler, ret_from_except) .macro vmap_stack_overflow_exception#ifdef CONFIG_SMP
arch/powerpc/kernel/head_40x.S +1 -4
@@ -184,8 +184,7 @@ _ENTRY(saved_ksp_limit) START_EXCEPTION(n, label); \ CRITICAL_EXCEPTION_PROLOG; \ addi r3,r1,STACK_FRAME_OVERHEAD; \ - EXC_XFER_TEMPLATE(hdlr, n+2, (MSR_KERNEL & ~(MSR_ME|MSR_DE|MSR_CE)), \ - crit_transfer_to_handler, ret_from_crit_exc) + EXC_XFER_TEMPLATE(hdlr, n+2, crit_transfer_to_handler, ret_from_crit_exc) /* * 0x0100 - Critical Interrupt Exception
@@ -495,7 +494,6 @@ _ENTRY(saved_ksp_limit) 2: mfspr r4,SPRN_DBSR addi r3,r1,STACK_FRAME_OVERHEAD EXC_XFER_TEMPLATE(DebugException, 0x2002, \ - (MSR_KERNEL & ~(MSR_ME|MSR_DE|MSR_CE)), \ crit_transfer_to_handler, ret_from_crit_exc) /* Programmable Interval Timer (PIT) Exception. (from 0x1000) */
@@ -517,7 +515,6 @@ WDTException: CRITICAL_EXCEPTION_PROLOG; addi r3,r1,STACK_FRAME_OVERHEAD; EXC_XFER_TEMPLATE(WatchdogException, 0x1020+2, - (MSR_KERNEL & ~(MSR_ME|MSR_DE|MSR_CE)), crit_transfer_to_handler, ret_from_crit_exc) /* Other PowerPC processors, namely those derived from the 6xx-series
Signature · 1 line
2.25.0
PATCH v2 14/14 powerpc/32: Use fast instructions to change MSR EE/RI when available
In reply to Christophe LeroyBooke and 40x have wrtee and wrteei to quickly change MSR EE.
8xx has registers SPRN_NRI, SPRN_EID and SPRN_EIE for changing MSR EE and RI.
Use them in syscall and exception handler when possible.
On an 8xx, it reduces the null_syscall test by 6 cycles (Two instances are changed in this patch, meaning we win 3 cycles per place).
--- arch/powerpc/include/asm/hw_irq.h | 46 +++++++++++++++++++++++++++++++ arch/powerpc/kernel/entry_32.S | 26 ++++++----------- arch/powerpc/kernel/head_32.h | 13 +++++---- arch/powerpc/kernel/head_booke.h | 9 ++---- 4 files changed, 66 insertions(+), 28 deletions(-)
arch/powerpc/include/asm/hw_irq.h +46 -0
@@ -368,6 +368,52 @@ static inline void may_hard_irq_enable(void) { }#define ARCH_IRQ_INIT_FLAGS IRQ_NOREQUEST +#else /* __ASSEMBLY__ */ + +.macro __hard_irq_enable tmp +#if defined(CONFIG_BOOKE ) || defined(CONFIG_40x) + wrteei 1 +#elif defined(CONFIG_PPC_8xx) + mtspr SPRN_EIE, r2 /* RI=1, EE=1 */ +#else + LOAD_REG_IMMEDIATE(\tmp, MSR_KERNEL | MSR_EE) + mtmsr \tmp +#endif +.endm + +.macro __hard_irq_disable tmp +#if defined(CONFIG_BOOKE) || defined(CONFIG_40x) + wrteei 0 +#elif defined(CONFIG_PPC_8xx) + mtspr SPRN_EID, r2 /* RI=1, EE=0 */ +#else + LOAD_REG_IMMEDIATE(\tmp, MSR_KERNEL) + mtmsr \tmp +#endif +.endm + +.macro __hard_EE_RI_disable tmp +#if defined(CONFIG_BOOKE) || defined(CONFIG_40x) + wrteei 0 +#elif defined(CONFIG_PPC_8xx) + mtspr SPRN_NRI, r2 /* RI=0, EE=0 */ +#else + LOAD_REG_IMMEDIATE(\tmp, MSR_KERNEL & ~MSR_RI) + mtmsr \tmp +#endif +.endm + +.macro __hard_RI_enable tmp +#if defined(CONFIG_BOOKE) || defined(CONFIG_40x) + /* nop */ +#elif defined(CONFIG_PPC_8xx) + mtspr SPRN_EID, r2 /* RI=1, EE=0 */ +#else + LOAD_REG_IMMEDIATE(\tmp, MSR_KERNEL) + mtmsr \tmp +#endif +.endm +#endif /* __ASSEMBLY__ */ #endif /* __KERNEL__ */ #endif /* _ASM_POWERPC_HW_IRQ_H */
arch/powerpc/kernel/entry_32.S +9 -17
@@ -303,8 +303,7 @@ trace_syscall_entry_irq_off: bl trace_hardirqs_on /* Now enable for real */ - LOAD_REG_IMMEDIATE(r10, MSR_KERNEL | MSR_EE) - mtmsr r10 + __hard_irq_enable r10 REST_GPR(0, r1) REST_4GPRS(3, r1)
@@ -373,9 +372,8 @@ ret_from_syscall: #endif mr r6,r3 /* disable interrupts so current_thread_info()->flags can't change */ - LOAD_REG_IMMEDIATE(r10,MSR_KERNEL) /* doesn't include MSR_EE */ + __hard_irq_disable r10 /* Note: We don't bother telling lockdep about it */ - mtmsr r10 lwz r9,TI_FLAGS(r2) li r8,-MAX_ERRNO andi. r0,r9,(_TIF_SYSCALL_DOTRACE|_TIF_SINGLESTEP|_TIF_USER_WORK_MASK|_TIF_PERSYSCALL_MASK)
@@ -529,8 +527,7 @@ syscall_exit_work: /* Re-enable interrupts. There is no need to trace that with * lockdep as we are supposed to have IRQs on at this point */ - ori r10,r10,MSR_EE - mtmsr r10 + __hard_irq_enable r10 /* Save NVGPRS if they're not saved already */ lwz r4,_TRAP(r1)
@@ -812,8 +809,7 @@ ret_from_except: * can't change between when we test it and when we return * from the interrupt. */ /* Note: We don't bother telling lockdep about it */ - LOAD_REG_IMMEDIATE(r10,MSR_KERNEL) - mtmsr r10 /* disable interrupts */ + __hard_irq_disable r10 lwz r3,_MSR(r1) /* Returning to user mode? */ andi. r0,r3,MSR_PR
@@ -971,8 +967,8 @@ END_FTR_SECTION_IFSET(CPU_FTR_NEED_PAIRED_STWCX) * can restart the exception exit path at the label * exc_exit_restart below. -- paulus */ - LOAD_REG_IMMEDIATE(r10,MSR_KERNEL & ~MSR_RI) - mtmsr r10 /* clear the RI bit */ + __hard_EE_RI_disable r10 + .globl exc_exit_restart exc_exit_restart: lwz r12,_NIP(r1)
@@ -1206,26 +1202,22 @@ do_work: /* r10 contains MSR_KERNEL here */ do_resched: /* r10 contains MSR_KERNEL here */ #ifdef CONFIG_TRACE_IRQFLAGS bl trace_hardirqs_on - mfmsr r10 #endif - ori r10,r10,MSR_EE - mtmsr r10 /* hard-enable interrupts */ + __hard_irq_enable r10 bl schedule recheck: /* Note: And we don't tell it we are disabling them again * neither. Those disable/enable cycles used to peek at * TI_FLAGS aren't advertised. */ - LOAD_REG_IMMEDIATE(r10,MSR_KERNEL) - mtmsr r10 /* disable interrupts */ + __hard_irq_disable r10 lwz r9,TI_FLAGS(r2) andi. r0,r9,_TIF_NEED_RESCHED bne- do_resched andi. r0,r9,_TIF_USER_WORK_MASK beq restore_user do_user_signal: /* r10 contains MSR_KERNEL here */ - ori r10,r10,MSR_EE - mtmsr r10 /* hard-enable interrupts */ + __hard_irq_enable r10 /* save r13-r31 in the exception frame, if not already done */ lwz r3,_TRAP(r1) andi. r0,r3,1
arch/powerpc/kernel/head_32.h +8 -5
@@ -3,6 +3,7 @@#define __HEAD_32_H__#include <asm/ptrace.h> /* for STACK_FRAME_REGS_MARKER */ +#include <asm/hw_irq.h> /* * Exception entry code. This code runs with address translation
@@ -89,10 +90,8 @@ lwz r12, SRR0(r12)#ifdef CONFIG_40x rlwinm r9,r9,0,14,12 /* clear MSR_WE (necessary?) */ -#else - li r10, MSR_KERNEL /* can take exceptions */ - mtmsr r10 /* (except for mach check in rtas) */ #endif + __hard_RI_enable r10 stw r0,GPR0(r11) lis r10,STACK_FRAME_REGS_MARKER@ha /* exception frame marker */ addi r10,r10,STACK_FRAME_REGS_MARKER@l
@@ -173,12 +172,16 @@* otherwise we might risk taking an interrupt before we tell lockdep * they are enabled. */ +#ifdef CONFIG_40x + wrtee r9 +#else LOAD_REG_IMMEDIATE(r10, MSR_KERNEL) rlwimi r10, r9, 0, MSR_EE + mtmsr r10 +#endif #else - LOAD_REG_IMMEDIATE(r10, MSR_KERNEL | MSR_EE) + __hard_irq_enable r10#endif - mtmsr r10b transfer_to_syscall/* jump to handler */ 99: b ret_from_kernel_syscall .endm
arch/powerpc/kernel/head_booke.h +3 -6
@@ -5,6 +5,7 @@#include <asm/ptrace.h> /* for STACK_FRAME_REGS_MARKER */ #include <asm/kvm_asm.h> #include <asm/kvm_booke_hv_asm.h> +#include <asm/hw_irq.h> #ifdef __ASSEMBLY__
@@ -163,14 +164,10 @@ ALT_FTR_SECTION_END_IFSET(CPU_FTR_EMB_HV)* otherwise we might risk taking an interrupt before we tell lockdep * they are enabled. */ - lis r10, MSR_KERNEL@h - ori r10, r10, MSR_KERNEL@l - rlwimi r10, r9, 0, MSR_EE + wrtee r9#else - lis r10, (MSR_KERNEL | MSR_EE)@h - ori r10, r10, (MSR_KERNEL | MSR_EE)@l + __hard_irq_enable r10#endif - mtmsr r10b transfer_to_syscall/* jump to handler */ 99: b ret_from_kernel_syscall .endm
Signature · 1 line
2.25.0
PATCH v2 8/14 powerpc/40x: Prepare normal exception handler for enabling MMU early
In reply to Christophe LeroyEnsure normal exception handler are able to manage stuff with MMU enabled. For that we use CONFIG_VMAP_STACK related code allthough there is no intention to really activate CONFIG_VMAP_STACK on powerpc 40x for the moment.
40x uses SPRN_DEAR instead of SPRN_DAR and SPRN_ESR instead of SPRN_DSISR. Take it into account in common macros.
--- arch/powerpc/kernel/entry_32.S | 2 +- arch/powerpc/kernel/head_32.h | 15 ++++++++++++++- arch/powerpc/kernel/head_40x.S | 17 ++++++----------- 3 files changed, 21 insertions(+), 13 deletions(-)
arch/powerpc/kernel/entry_32.S +1 -1
@@ -162,7 +162,7 @@ transfer_to_handler: li r12,-1 /* clear all pending debug events */ mtspr SPRN_DBSR,r12 lis r11,global_dbcr0@ha - tophys(r11,r11) + tophys_novmstack r11,r11 addi r11,r11,global_dbcr0@l #ifdef CONFIG_SMP lwz r9,TASK_CPU(r2)
arch/powerpc/kernel/head_32.h +14 -1
@@ -22,9 +22,17 @@ #ifdef CONFIG_VMAP_STACK mfspr r10, SPRN_SPRG_THREAD .if \handle_dar_dsisr +#ifdef CONFIG_40x + mfspr r11, SPRN_DEAR +#else mfspr r11, SPRN_DAR +#endif stw r11, DAR(r10) +#ifdef CONFIG_40x + mfspr r11, SPRN_ESR +#else mfspr r11, SPRN_DSISR +#endif stw r11, DSISR(r10) .endif mfspr r11, SPRN_SRR0
@@ -190,7 +198,7 @@ li r12,-1 /* clear all pending debug events */ mtspr SPRN_DBSR,r12 lis r11,global_dbcr0@ha - tophys(r11,r11) + tophys_novmstack(r11,r11) addi r11,r11,global_dbcr0@l lwz r12,0(r11) mtspr SPRN_DBCR0,r12
@@ -228,8 +236,13 @@ .macro save_dar_dsisr_on_stack reg1, reg2, sp #ifndef CONFIG_VMAP_STACK +#ifdef CONFIG_40x + mfspr \reg1, SPRN_DEAR + mfspr \reg2, SPRN_ESR +#else mfspr \reg1, SPRN_DAR mfspr \reg2, SPRN_DSISR +#endif stw \reg1, _DAR(\sp) stw \reg2, _DSISR(\sp) #endif
arch/powerpc/kernel/head_40x.S +6 -11
@@ -223,11 +223,8 @@ _ENTRY(saved_ksp_limit) * if they can't resolve the lightweight TLB fault. */ START_EXCEPTION(0x0300, DataStorage) - EXCEPTION_PROLOG - mfspr r5, SPRN_ESR /* Grab the ESR, save it */ - stw r5, _ESR(r11) - mfspr r4, SPRN_DEAR /* Grab the DEAR, save it */ - stw r4, _DEAR(r11) + EXCEPTION_PROLOG handle_dar_dsisr=1 + save_dar_dsisr_on_stack r4, r5, r11 EXC_XFER_LITE(0x300, handle_page_fault) /*
@@ -246,17 +243,15 @@ _ENTRY(saved_ksp_limit) /* 0x0600 - Alignment Exception */ START_EXCEPTION(0x0600, Alignment) - EXCEPTION_PROLOG - mfspr r4,SPRN_DEAR /* Grab the DEAR and save it */ - stw r4,_DEAR(r11) + EXCEPTION_PROLOG handle_dar_dsisr=1 + save_dar_dsisr_on_stack r4, r5, r11 addi r3,r1,STACK_FRAME_OVERHEAD EXC_XFER_STD(0x600, alignment_exception) /* 0x0700 - Program Exception */ START_EXCEPTION(0x0700, ProgramCheck) - EXCEPTION_PROLOG - mfspr r4,SPRN_ESR /* Grab the ESR and save it */ - stw r4,_ESR(r11) + EXCEPTION_PROLOG handle_dar_dsisr=1 + save_dar_dsisr_on_stack r4, r5, r11 addi r3,r1,STACK_FRAME_OVERHEAD EXC_XFER_STD(0x700, program_check_exception)
Signature · 1 line
2.25.0
PATCH v2 5/14 powerpc/40x: Save SRR0/SRR1 and r10/r11 earlier in critical exception
In reply to Christophe LeroyIn order to be able to switch MMU on in exception prolog, save SRR0 and SRR1 earlier.
Also save r10 and r11 into stack earlier to better match with the normal exception prolog.
--- arch/powerpc/kernel/entry_32.S | 9 --------- arch/powerpc/kernel/head_40x.S | 8 ++++++++ 2 files changed, 8 insertions(+), 9 deletions(-)
arch/powerpc/kernel/entry_32.S +0 -9
@@ -107,15 +107,6 @@ _ASM_NOKPROBE_SYMBOL(crit_transfer_to_handler) #ifdef CONFIG_40x .globl crit_transfer_to_handler crit_transfer_to_handler: - lwz r0,crit_r10@l(0) - stw r0,GPR10(r11) - lwz r0,crit_r11@l(0) - stw r0,GPR11(r11) - mfspr r0,SPRN_SRR0 - stw r0,crit_srr0@l(0) - mfspr r0,SPRN_SRR1 - stw r0,crit_srr1@l(0) - /* set the stack limit to the current stack */ mfspr r8,SPRN_SPRG_THREAD lwz r0,KSP_LIMIT(r8)
arch/powerpc/kernel/head_40x.S +8 -0
@@ -103,6 +103,10 @@ _ENTRY(saved_ksp_limit) .macro CRITICAL_EXCEPTION_PROLOG stw r10,crit_r10@l(0) /* save two registers to work with */ stw r11,crit_r11@l(0) + mfspr r10,SPRN_SRR0 + mfspr r11,SPRN_SRR1 + stw r10,crit_srr0@l(0) + stw r11,crit_srr1@l(0) mfcr r10 /* save CR in r10 for now */ mfspr r11,SPRN_SRR3 /* check whether user or kernel */ andi. r11,r11,MSR_PR
@@ -120,6 +124,10 @@ _ENTRY(saved_ksp_limit) stw r9,GPR9(r11) mflr r10 stw r10,_LINK(r11) + lwz r10,crit_r10@l(0) + lwz r12,crit_r11@l(0) + stw r10,GPR10(r11) + stw r12,GPR11(r11) mfspr r12,SPRN_DEAR /* save DEAR and ESR in the frame */ stw r12,_DEAR(r11) /* since they may have had stuff */ mfspr r9,SPRN_ESR /* in them at the point where the */
Signature · 1 line
2.25.0
PATCH v2 7/14 powerpc/40x: Prepare for enabling MMU in critical exception prolog
In reply to Christophe LeroyIn order the enable MMU early in exception prolog, implement CONFIG_VMAP_STACK principles in critical exception prolog.
There is no intention to use CONFIG_VMAP_STACK on 40x, but related code will be used to enable MMU early in exception in a later patch.
Also address (critirq_ctx-PAGE_OFFSET) directly instead of using tophys() in order to win one instruction.
--- arch/powerpc/kernel/head_40x.S | 42 +++++++++++++++++++++++++++++++--- 1 file changed, 39 insertions(+), 3 deletions(-)
arch/powerpc/kernel/head_40x.S +39 -3
@@ -89,6 +89,14 @@ _ENTRY(crit_srr0) .space 4 _ENTRY(crit_srr1) .space 4 +#ifdef CONFIG_VMAP_STACK +_ENTRY(crit_r1) + .space 4 +_ENTRY(crit_dear) + .space 4 +_ENTRY(crit_esr) + .space 4 +#endif _ENTRY(saved_ksp_limit) .space 4
@@ -107,32 +115,60 @@ _ENTRY(saved_ksp_limit) mfspr r11,SPRN_SRR1 stw r10,crit_srr0@l(0) stw r11,crit_srr1@l(0) +#ifdef CONFIG_VMAP_STACK + mfspr r10,SPRN_DEAR + mfspr r11,SPRN_ESR + stw r10,crit_dear@l(0) + stw r11,crit_esr@l(0) +#endif mfcr r10 /* save CR in r10 for now */ mfspr r11,SPRN_SRR3 /* check whether user or kernel */ andi. r11,r11,MSR_PR - lis r11,critirq_ctx@ha - tophys(r11,r11) - lwz r11,critirq_ctx@l(r11) + lis r11,(critirq_ctx-PAGE_OFFSET)@ha + lwz r11,(critirq_ctx-PAGE_OFFSET)@l(r11) beq 1f /* COMING FROM USER MODE */ mfspr r11,SPRN_SPRG_THREAD /* if from user, start at top of */ lwz r11,TASK_STACK-THREAD(r11) /* this thread's kernel stack */ +#ifdef CONFIG_VMAP_STACK +1: stw r1,crit_r1@l(0) + addi r1,r11,THREAD_SIZE-INT_FRAME_SIZE /* Alloc an excpt frm */ + LOAD_REG_IMMEDIATE(r11,MSR_KERNEL & ~(MSR_IR | MSR_RI)) + mtmsr r11 + isync + lwz r11,crit_r1@l(0) + stw r11,GPR1(r1) + stw r11,0(r1) + mr r11,r1 +#else 1: addi r11,r11,THREAD_SIZE-INT_FRAME_SIZE /* Alloc an excpt frm */ tophys(r11,r11) stw r1,GPR1(r11) stw r1,0(r11) tovirt(r1,r11) +#endif stw r10,_CCR(r11) /* save various registers */ stw r12,GPR12(r11) stw r9,GPR9(r11) mflr r10 stw r10,_LINK(r11) +#ifdef CONFIG_VMAP_STACK + lis r9,PAGE_OFFSET@ha + lwz r10,crit_r10@l(r9) + lwz r12,crit_r11@l(r9) +#else lwz r10,crit_r10@l(0) lwz r12,crit_r11@l(0) +#endif stw r10,GPR10(r11) stw r12,GPR11(r11) +#ifdef CONFIG_VMAP_STACK + lwz r12,crit_dear@l(r9) + lwz r9,crit_esr@l(r9) +#else mfspr r12,SPRN_DEAR /* save DEAR and ESR in the frame */ mfspr r9,SPRN_ESR /* in them at the point where the */ +#endif stw r12,_DEAR(r11) /* since they may have had stuff */ stw r9,_ESR(r11) /* exception was taken */ mfspr r12,SPRN_SRR2
Signature · 1 line
2.25.0
PATCH v2 6/14 powerpc/40x: Reorder a few instructions in critical exception prolog
In reply to Christophe LeroyIn order to ease preparation for CONFIG_VMAP_STACK, reorder a few instruction, especially save r1 into stack frame earlier.
--- arch/powerpc/kernel/head_40x.S | 8 ++++---- 1 file changed, 4 insertions(+), 4 deletions(-)
arch/powerpc/kernel/head_40x.S +4 -4
@@ -119,6 +119,9 @@ _ENTRY(saved_ksp_limit) lwz r11,TASK_STACK-THREAD(r11) /* this thread's kernel stack */ 1: addi r11,r11,THREAD_SIZE-INT_FRAME_SIZE /* Alloc an excpt frm */ tophys(r11,r11) + stw r1,GPR1(r11) + stw r1,0(r11) + tovirt(r1,r11) stw r10,_CCR(r11) /* save various registers */ stw r12,GPR12(r11) stw r9,GPR9(r11)
@@ -129,14 +132,11 @@ _ENTRY(saved_ksp_limit) stw r10,GPR10(r11) stw r12,GPR11(r11) mfspr r12,SPRN_DEAR /* save DEAR and ESR in the frame */ - stw r12,_DEAR(r11) /* since they may have had stuff */ mfspr r9,SPRN_ESR /* in them at the point where the */ + stw r12,_DEAR(r11) /* since they may have had stuff */ stw r9,_ESR(r11) /* exception was taken */ mfspr r12,SPRN_SRR2 - stw r1,GPR1(r11) mfspr r9,SPRN_SRR3 - stw r1,0(r11) - tovirt(r1,r11) rlwinm r9,r9,0,14,12 /* clear MSR_WE (necessary?) */ stw r0,GPR0(r11) lis r10, STACK_FRAME_REGS_MARKER@ha /* exception frame marker */
Signature · 1 line
2.25.0
PATCH v2 11/14 powerpc/32: Always enable data translation in exception prolog
In reply to Christophe LeroyIf the code can use a stack in vm area, it can also use a stack in linear space.
Simplify code by removing old non VMAP stack code on PPC32.
That means the data translation is now re-enabled early in exception prolog in all cases, not only when using VMAP stacks.
--- arch/powerpc/include/asm/processor.h | 4 +- arch/powerpc/kernel/asm-offsets.c | 2 - arch/powerpc/kernel/entry_32.S | 26 ++----- arch/powerpc/kernel/fpu.S | 2 - arch/powerpc/kernel/head_32.h | 106 +-------------------------- arch/powerpc/kernel/head_40x.S | 25 ------- arch/powerpc/kernel/head_8xx.S | 19 +---- arch/powerpc/kernel/head_book3s_32.S | 38 +--------- arch/powerpc/kernel/head_booke.h | 2 - arch/powerpc/kernel/idle_6xx.S | 12 +-- arch/powerpc/kernel/idle_e500.S | 4 +- arch/powerpc/kernel/vector.S | 2 - arch/powerpc/mm/book3s32/hash_low.S | 14 ---- 13 files changed, 17 insertions(+), 239 deletions(-)
arch/powerpc/include/asm/processor.h +1 -3
@@ -148,11 +148,9 @@ struct thread_struct {#ifdef CONFIG_PPC_RTAS unsigned long rtas_sp;/* stack pointer for when in RTAS */ #endif -#endif #if defined(CONFIG_PPC_BOOK3S_32 ) && defined(CONFIG_PPC_KUAP )unsigned long kuap;/* opened segments for user access */ #endif -#ifdef CONFIG_VMAP_STACK unsigned long srr0;unsigned long srr1;unsigned long dar;
@@ -161,7 +159,7 @@ struct thread_struct {unsigned long r0, r3, r4, r5, r6, r8, r9, r11;unsigned long lr, ctr;#endif -#endif +#endif /* CONFIG_PPC32 */ /* Debug Registers */ struct debug_reg debug;#ifdef CONFIG_PPC_FPU_REGS
arch/powerpc/kernel/asm-offsets.c +0 -2
@@ -132,7 +132,6 @@ int main(void)OFFSET (KSP_VSID , thread_struct, ksp_vsid);#else /* CONFIG_PPC64 */ OFFSET (PGDIR , thread_struct, pgdir); -#ifdef CONFIG_VMAP_STACK OFFSET (SRR0 , thread_struct, srr0);OFFSET (SRR1 , thread_struct, srr1);OFFSET (DAR , thread_struct, dar);
@@ -149,7 +148,6 @@ int main(void)OFFSET (THLR , thread_struct, lr);OFFSET (THCTR , thread_struct, ctr);#endif -#endif #ifdef CONFIG_SPE OFFSET (THREAD_EVR0 , thread_struct, evr[0 ]);OFFSET (THREAD_ACC , thread_struct, acc);
arch/powerpc/kernel/entry_32.S +8 -18
@@ -141,7 +141,7 @@ transfer_to_handler: stw r12,_CTR(r11) stw r2,_XER(r11) mfspr r12,SPRN_SPRG_THREAD - tovirt_vmstack r12, r12 + tovirt(r12, r12) beq 2f /* if from user, fix up THREAD.regs */ addi r2, r12, -THREAD addi r11,r1,STACK_FRAME_OVERHEAD
@@ -162,7 +162,6 @@ transfer_to_handler: li r12,-1 /* clear all pending debug events */ mtspr SPRN_DBSR,r12 lis r11,global_dbcr0@ha - tophys_novmstack r11,r11 addi r11,r11,global_dbcr0@l #ifdef CONFIG_SMP lwz r9,TASK_CPU(r2)
@@ -199,8 +198,7 @@ transfer_to_handler: transfer_to_handler_cont: 3: mflr r9 - tovirt_novmstack r2, r2 /* set r2 to current */ - tovirt_vmstack r9, r9 + tovirt(r9, r9) lwz r11,0(r9) /* virtual address of handler */ lwz r9,4(r9) /* where to go when done */ #if defined(CONFIG_PPC_8xx) && defined(CONFIG_PERF_EVENTS)
@@ -215,8 +213,7 @@ transfer_to_handler_cont: * To speed up the syscall path where interrupts stay on, let's check * first if we are changing the MSR value at all. */ - tophys_novmstack r12, r1 - lwz r12,_MSR(r12) + lwz r12,_MSR(r1) andi. r12,r12,MSR_EE bne 1f
@@ -308,9 +305,6 @@ stack_ovf: lis r9,StackOverflow@ha addi r9,r9,StackOverflow@l LOAD_REG_IMMEDIATE(r10,MSR_KERNEL) -#if defined(CONFIG_PPC_8xx) && defined(CONFIG_PERF_EVENTS) - mtspr SPRN_NRI, r0 -#endif mtspr SPRN_SRR0,r9 mtspr SPRN_SRR1,r10 rfi
@@ -1327,7 +1321,6 @@ _GLOBAL(enter_rtas) lis r6,1f@ha /* physical return address for rtas */ addi r6,r6,1f@l tophys(r6,r6) - tophys_novmstack r7, r1 lwz r8,RTASENTRY(r4) lwz r4,RTASBASE(r4) mfmsr r9
@@ -1336,22 +1329,19 @@ _GLOBAL(enter_rtas) mtmsr r0 /* disable interrupts so SRR0/1 don't get trashed */ li r9,MSR_KERNEL & ~(MSR_IR|MSR_DR) mtlr r6 - stw r7, THREAD + RTAS_SP(r2) + stw r1, THREAD + RTAS_SP(r2) mtspr SPRN_SRR0,r8 mtspr SPRN_SRR1,r9 rfi -1: tophys_novmstack r9, r1 -#ifdef CONFIG_VMAP_STACK +1: li r0, MSR_KERNEL & ~MSR_IR /* can take DTLB miss */ mtmsr r0 isync -#endif - lwz r8,INT_FRAME_SIZE+4(r9) /* get return address */ - lwz r9,8(r9) /* original msr value */ + lwz r8,INT_FRAME_SIZE+4(r1) /* get return address */ + lwz r9,8(r1) /* original msr value */ addi r1,r1,INT_FRAME_SIZE li r0,0 - tophys_novmstack r7, r2 - stw r0, THREAD + RTAS_SP(r7) + stw r0, THREAD + RTAS_SP(r2) mtspr SPRN_SRR0,r8 mtspr SPRN_SRR1,r9 rfi /* return to caller */
arch/powerpc/kernel/fpu.S +0 -2
@@ -92,9 +92,7 @@ END_FTR_SECTION_IFSET(CPU_FTR_VSX) /* enable use of FP after return */ #ifdef CONFIG_PPC32 mfspr r5,SPRN_SPRG_THREAD /* current task's THREAD (phys) */ -#ifdef CONFIG_VMAP_STACK tovirt(r5, r5) -#endif lwz r4,THREAD_FPEXC_MODE(r5) ori r9,r9,MSR_FP /* enable FP for current */ or r9,r9,r4
arch/powerpc/kernel/head_32.h +2 -104
210 lines not on this page · raw message
arch/powerpc/kernel/head_40x.S +0 -25
@@ -89,14 +89,12 @@ _ENTRY(crit_srr0) .space 4 _ENTRY(crit_srr1) .space 4 -#ifdef CONFIG_VMAP_STACK _ENTRY(crit_r1) .space 4 _ENTRY(crit_dear) .space 4 _ENTRY(crit_esr) .space 4 -#endif _ENTRY(saved_ksp_limit) .space 4
@@ -115,12 +113,10 @@ _ENTRY(saved_ksp_limit) mfspr r11,SPRN_SRR1 stw r10,crit_srr0@l(0) stw r11,crit_srr1@l(0) -#ifdef CONFIG_VMAP_STACK mfspr r10,SPRN_DEAR mfspr r11,SPRN_ESR stw r10,crit_dear@l(0) stw r11,crit_esr@l(0) -#endif mfcr r10 /* save CR in r10 for now */ mfspr r11,SPRN_SRR3 /* check whether user or kernel */ andi. r11,r11,MSR_PR
@@ -130,7 +126,6 @@ _ENTRY(saved_ksp_limit) /* COMING FROM USER MODE */ mfspr r11,SPRN_SPRG_THREAD /* if from user, start at top of */ lwz r11,TASK_STACK-THREAD(r11) /* this thread's kernel stack */ -#ifdef CONFIG_VMAP_STACK 1: stw r1,crit_r1@l(0) addi r1,r11,THREAD_SIZE-INT_FRAME_SIZE /* Alloc an excpt frm */ LOAD_REG_IMMEDIATE(r11,MSR_KERNEL & ~(MSR_IR | MSR_RI))
@@ -140,35 +135,18 @@ _ENTRY(saved_ksp_limit) stw r11,GPR1(r1) stw r11,0(r1) mr r11,r1 -#else -1: addi r11,r11,THREAD_SIZE-INT_FRAME_SIZE /* Alloc an excpt frm */ - tophys(r11,r11) - stw r1,GPR1(r11) - stw r1,0(r11) - tovirt(r1,r11) -#endif stw r10,_CCR(r11) /* save various registers */ stw r12,GPR12(r11) stw r9,GPR9(r11) mflr r10 stw r10,_LINK(r11) -#ifdef CONFIG_VMAP_STACK lis r9,PAGE_OFFSET@ha lwz r10,crit_r10@l(r9) lwz r12,crit_r11@l(r9) -#else - lwz r10,crit_r10@l(0) - lwz r12,crit_r11@l(0) -#endif stw r10,GPR10(r11) stw r12,GPR11(r11) -#ifdef CONFIG_VMAP_STACK lwz r12,crit_dear@l(r9) lwz r9,crit_esr@l(r9) -#else - mfspr r12,SPRN_DEAR /* save DEAR and ESR in the frame */ - mfspr r9,SPRN_ESR /* in them at the point where the */ -#endif stw r12,_DEAR(r11) /* since they may have had stuff */ stw r9,_ESR(r11) /* exception was taken */ mfspr r12,SPRN_SRR2
@@ -224,7 +202,6 @@ _ENTRY(saved_ksp_limit) */ START_EXCEPTION(0x0300, DataStorage) EXCEPTION_PROLOG handle_dar_dsisr=1 - save_dar_dsisr_on_stack r4, r5, r11 EXC_XFER_LITE(0x300, handle_page_fault) /*
@@ -244,14 +221,12 @@ _ENTRY(saved_ksp_limit) /* 0x0600 - Alignment Exception */ START_EXCEPTION(0x0600, Alignment) EXCEPTION_PROLOG handle_dar_dsisr=1 - save_dar_dsisr_on_stack r4, r5, r11 addi r3,r1,STACK_FRAME_OVERHEAD EXC_XFER_STD(0x600, alignment_exception) /* 0x0700 - Program Exception */ START_EXCEPTION(0x0700, ProgramCheck) EXCEPTION_PROLOG handle_dar_dsisr=1 - save_dar_dsisr_on_stack r4, r5, r11 addi r3,r1,STACK_FRAME_OVERHEAD EXC_XFER_STD(0x700, program_check_exception)
arch/powerpc/kernel/head_8xx.S +2 -17
@@ -124,7 +124,6 @@ instruction_counter: . = 0x200 MachineCheck: EXCEPTION_PROLOG handle_dar_dsisr=1 - save_dar_dsisr_on_stack r4, r5, r11 li r6, RPN_PATTERN mtspr SPRN_DAR, r6 /* Tag DAR, to be used in DTLB Error */ addi r3,r1,STACK_FRAME_OVERHEAD
@@ -137,7 +136,6 @@ MachineCheck: . = 0x600 Alignment: EXCEPTION_PROLOG handle_dar_dsisr=1 - save_dar_dsisr_on_stack r4, r5, r11 li r6, RPN_PATTERN mtspr SPRN_DAR, r6 /* Tag DAR, to be used in DTLB Error */ addi r3,r1,STACK_FRAME_OVERHEAD
@@ -333,21 +331,16 @@ DataTLBError: cmpwi cr1, r11, RPN_PATTERN beq- cr1, FixupDAR /* must be a buggy dcbX, icbi insn. */ DARFixed:/* Return from dcbx instruction bug workaround */ -#ifdef CONFIG_VMAP_STACK li r11, RPN_PATTERN mtspr SPRN_DAR, r11 /* Tag DAR, to be used in DTLB Error */ -#endif EXCEPTION_PROLOG_1 EXCEPTION_PROLOG_2 handle_dar_dsisr=1 - get_and_save_dar_dsisr_on_stack r4, r5, r11 + lwz r4, _DAR(r11) + lwz r5, _DSISR(r11) andis. r10,r5,DSISR_NOHPTE@h beq+ .Ldtlbie tlbie r4 .Ldtlbie: -#ifndef CONFIG_VMAP_STACK - li r10,RPN_PATTERN - mtspr SPRN_DAR,r10 /* Tag DAR, to be used in DTLB Error */ -#endif /* 0x300 is DataAccess exception, needed by bad_page_fault() */ EXC_XFER_LITE(0x300, handle_page_fault)
@@ -364,10 +357,6 @@ do_databreakpoint: addi r3,r1,STACK_FRAME_OVERHEAD mfspr r4,SPRN_BAR stw r4,_DAR(r11) -#ifndef CONFIG_VMAP_STACK - mfspr r5,SPRN_DSISR - stw r5,_DSISR(r11) -#endif EXC_XFER_STD(0x1c00, do_break) . = 0x1c00
@@ -510,14 +499,10 @@ FixupDAR:/* Entry point for dcbx workaround. */ 152: mfdar r11 mtctr r11 /* restore ctr reg from DAR */ -#ifdef CONFIG_VMAP_STACK mfspr r11, SPRN_SPRG_THREAD stw r10, DAR(r11) mfspr r10, SPRN_DSISR stw r10, DSISR(r11) -#else - mtdar r10 /* save fault EA to DAR */ -#endif mfspr r10,SPRN_M_TW b DARFixed /* Go back to normal TLB handling */
arch/powerpc/kernel/head_book3s_32.S +1 -37
@@ -288,7 +288,6 @@ MachineCheck: . = 0x300 DO_KVM 0x300 DataAccess: -#ifdef CONFIG_VMAP_STACK #ifdef CONFIG_PPC_BOOK3S_604 BEGIN_MMU_FTR_SECTION mtspr SPRN_SPRG_SCRATCH2,r10
@@ -310,29 +309,11 @@ ALT_MMU_FTR_SECTION_END_IFSET(MMU_FTR_HPTE_TABLE) 1: EXCEPTION_PROLOG_0 handle_dar_dsisr=1 EXCEPTION_PROLOG_1 b handle_page_fault_tramp_1 -#else /* CONFIG_VMAP_STACK */ - EXCEPTION_PROLOG handle_dar_dsisr=1 - get_and_save_dar_dsisr_on_stack r4, r5, r11 -#ifdef CONFIG_PPC_BOOK3S_604 -BEGIN_MMU_FTR_SECTION - andis. r0, r5, (DSISR_BAD_FAULT_32S | DSISR_DABRMATCH)@h - bne handle_page_fault_tramp_2 /* if not, try to put a PTE */ - rlwinm r3, r5, 32 - 15, 21, 21 /* DSISR_STORE -> _PAGE_RW */ - bl hash_page - b handle_page_fault_tramp_1 -MMU_FTR_SECTION_ELSE -#endif - b handle_page_fault_tramp_2 -#ifdef CONFIG_PPC_BOOK3S_604 -ALT_MMU_FTR_SECTION_END_IFSET(MMU_FTR_HPTE_TABLE) -#endif -#endif /* CONFIG_VMAP_STACK */ /* Instruction access exception. */ . = 0x400 DO_KVM 0x400 InstructionAccess: -#ifdef CONFIG_VMAP_STACK mtspr SPRN_SPRG_SCRATCH0,r10 mtspr SPRN_SPRG_SCRATCH1,r11 mfspr r10, SPRN_SPRG_THREAD
@@ -353,18 +334,6 @@ END_MMU_FTR_SECTION_IFSET(MMU_FTR_HPTE_TABLE) EXCEPTION_PROLOG_1 EXCEPTION_PROLOG_2 -#else /* CONFIG_VMAP_STACK */ - EXCEPTION_PROLOG - andis. r0,r9,SRR1_ISI_NOPT@h /* no pte found? */ - beq 1f /* if so, try to put a PTE */ - li r3,0 /* into the hash table */ - mr r4,r12 /* SRR0 is fault address */ -#ifdef CONFIG_PPC_BOOK3S_604 -BEGIN_MMU_FTR_SECTION - bl hash_page -END_MMU_FTR_SECTION_IFSET(MMU_FTR_HPTE_TABLE) -#endif -#endif /* CONFIG_VMAP_STACK */ andis. r5,r9,DSISR_SRR1_MATCH_32S@h /* Filter relevant SRR1 bits */ stw r5, _DSISR(r11) stw r12, _DAR(r11)
@@ -378,7 +347,6 @@ END_MMU_FTR_SECTION_IFSET(MMU_FTR_HPTE_TABLE) DO_KVM 0x600 Alignment: EXCEPTION_PROLOG handle_dar_dsisr=1 - save_dar_dsisr_on_stack r4, r5, r11 addi r3,r1,STACK_FRAME_OVERHEAD b alignment_exception_tramp
@@ -686,18 +654,13 @@ alignment_exception_tramp: EXC_XFER_STD(0x600, alignment_exception) handle_page_fault_tramp_1: -#ifdef CONFIG_VMAP_STACK EXCEPTION_PROLOG_2 handle_dar_dsisr=1 -#endif lwz r5, _DSISR(r11) - /* fall through */ -handle_page_fault_tramp_2: andis. r0, r5, DSISR_DABRMATCH@h bne- 1f EXC_XFER_LITE(0x300, handle_page_fault) 1: EXC_XFER_STD(0x300, do_break) -#ifdef CONFIG_VMAP_STACK #ifdef CONFIG_PPC_BOOK3S_604 .macro save_regs_thread thread stw r0, THR0(\thread)
@@ -772,6 +735,7 @@ fast_hash_page_return: rfi #endif /* CONFIG_PPC_BOOK3S_604 */ +#ifdef CONFIG_VMAP_STACK stack_overflow: vmap_stack_overflow_exception #endif
arch/powerpc/kernel/head_booke.h +0 -2
@@ -144,7 +144,6 @@ ALT_FTR_SECTION_END_IFSET(CPU_FTR_EMB_HV) li r12,-1 /* clear all pending debug events */ mtspr SPRN_DBSR,r12 lis r11,global_dbcr0@ha - tophys(r11,r11) addi r11,r11,global_dbcr0@l #ifdef CONFIG_SMP lwz r10, TASK_CPU(r2)
@@ -158,7 +157,6 @@ ALT_FTR_SECTION_END_IFSET(CPU_FTR_EMB_HV) stw r12,4(r11) 3: - tovirt(r2, r2) /* set r2 to current */ lis r11, transfer_to_syscall@h ori r11, r11, transfer_to_syscall@l #ifdef CONFIG_TRACE_IRQFLAGS
arch/powerpc/kernel/idle_6xx.S +1 -11
@@ -145,9 +145,7 @@ END_FTR_SECTION_IFSET(CPU_FTR_ALTIVEC) /* * Return from NAP/DOZE mode, restore some CPU specific registers, - * we are called with DR/IR still off and r2 containing physical - * address of current. R11 points to the exception frame (physical - * address). We have to preserve r10. + * R11 points to the exception frame. We have to preserve r10. */ _GLOBAL(power_save_ppc32_restore) lwz r9,_LINK(r11) /* interrupted in ppc6xx_idle: */
@@ -166,11 +164,7 @@ BEGIN_FTR_SECTION mfspr r9,SPRN_HID0 andis. r9,r9,HID0_NAP@h beq 1f -#ifdef CONFIG_VMAP_STACK addis r9, r11, nap_save_msscr0@ha -#else - addis r9,r11,(nap_save_msscr0-KERNELBASE)@ha -#endif lwz r9,nap_save_msscr0@l(r9) mtspr SPRN_MSSCR0, r9 sync
@@ -178,11 +172,7 @@ BEGIN_FTR_SECTION 1: END_FTR_SECTION_IFSET(CPU_FTR_NAP_DISABLE_L2_PR) BEGIN_FTR_SECTION -#ifdef CONFIG_VMAP_STACK addis r9, r11, nap_save_hid1@ha -#else - addis r9,r11,(nap_save_hid1-KERNELBASE)@ha -#endif lwz r9,nap_save_hid1@l(r9) mtspr SPRN_HID1, r9 END_FTR_SECTION_IFSET(CPU_FTR_DUAL_PLL_750FX)
arch/powerpc/kernel/idle_e500.S +2 -2
@@ -74,8 +74,8 @@ END_FTR_SECTION_IFSET(CPU_FTR_CAN_NAP) /* * Return from NAP/DOZE mode, restore some CPU specific registers, - * r2 containing physical address of current. - * r11 points to the exception frame (physical address). + * r2 containing address of current. + * r11 points to the exception frame. * We have to preserve r10. */ _GLOBAL(power_save_ppc32_restore)
arch/powerpc/kernel/vector.S +0 -2
@@ -67,9 +67,7 @@ _GLOBAL(load_up_altivec) #ifdef CONFIG_PPC32 mfspr r5,SPRN_SPRG_THREAD /* current task's THREAD (phys) */ oris r9,r9,MSR_VEC@h -#ifdef CONFIG_VMAP_STACK tovirt(r5, r5) -#endif #else ld r4,PACACURRENT(r13) addi r5,r4,THREAD /* Get THREAD */
arch/powerpc/mm/book3s32/hash_low.S +0 -14
@@ -140,10 +140,6 @@ _GLOBAL(hash_page) bne- .Lretry /* retry if someone got there first */ mfsrin r3,r4 /* get segment reg for segment */ -#ifndef CONFIG_VMAP_STACK - mfctr r0 - stw r0,_CTR(r11) -#endif bl create_hpte /* add the hash table entry */ #ifdef CONFIG_SMP
@@ -152,17 +148,7 @@ _GLOBAL(hash_page) li r0,0 stw r0, (mmu_hash_lock - PAGE_OFFSET)@l(r8) #endif - -#ifdef CONFIG_VMAP_STACK b fast_hash_page_return -#else - /* Return from the exception */ - lwz r5,_CTR(r11) - mtctr r5 - lwz r0,GPR0(r11) - lwz r8,GPR8(r11) - b fast_exception_return -#endif #ifdef CONFIG_SMP .Lhash_page_out:
Signature · 1 line
2.25.0