powerpc: merge _switch in 32/64
Nicholas Piggin
reviewerv3 cover powerpc: merge _switch in 32/64
Since v2:
- Add PPC_CREATE_STACK_FRAME() to abstract prologue differences.
- Build fix.
- Makefile tidy [Christophe]
- Fix a missing SOB.
Since v1:
- Don't re-order 32-bit prologue.
- Improve Kconfig conditional includes.
- Break out code changes into their own patches before merging, so merge patch leaves generated code unchanged.
- Change prom_entry.S to prom_entry_64.S.
Nicholas Piggin (6):
powerpc/64s: move stack SLB pinning out of line from _switch powerpc/64: Rearrange 64-bit _switch to prepare for 32/64 merge powerpc/32: Remove sync from _switch
powerpc/32: Rearrange _switch to prepare for 32/64 merge
powerpc: merge 32-bit and 64-bit _switch implementation powerpc/64: Rename entry_64.S to prom_entry_64.S
arch/powerpc/include/asm/ppc_asm.h | 14 +
arch/powerpc/kernel/Makefile | 14 +-
arch/powerpc/kernel/entry_32.S | 58 ----
arch/powerpc/kernel/prom_entry_64.S | 87 +++++
arch/powerpc/kernel/{entry_64.S => switch.S} | 331 ++++++++-----------
scripts/head-object-list.txt | 2 +-
6 files changed, 238 insertions(+), 268 deletions(-)
create mode 100644 arch/powerpc/kernel/prom_entry_64.S
rename arch/powerpc/kernel/{entry_64.S => switch.S} (59%)Signature · 1 line
2.40.1
Nicholas Piggin
reviewerPATCH v3 1/6 powerpc/64s: move stack SLB pinning out of line from _switch
The large hunk of SLB pinning in _switch asm code makes it more difficult to see everything else that's going on. It is a less important path now, so icache and fetch footprint overhead can be avoided.
Move context switch stack SLB pinning out of line.
--- arch/powerpc/kernel/entry_64.S | 113 ++++++++++++++++++--------------- 1 file changed, 62 insertions(+), 51 deletions(-)
arch/powerpc/kernel/entry_64.S +62 -51
@@ -105,6 +105,64 @@ flush_branch_caches: .endr blr + +#ifdef CONFIG_PPC_64S_HASH_MMU +.balign 32 +/* + * New stack pointer in r8, old stack pointer in r1, must not clobber r3 + */ +pin_stack_slb: +BEGIN_FTR_SECTION + clrrdi r6,r8,28 /* get its ESID */ + clrrdi r9,r1,28 /* get current sp ESID */ +FTR_SECTION_ELSE + clrrdi r6,r8,40 /* get its 1T ESID */ + clrrdi r9,r1,40 /* get current sp 1T ESID */ +ALT_MMU_FTR_SECTION_END_IFCLR(MMU_FTR_1T_SEGMENT) + clrldi. r0,r6,2 /* is new ESID c00000000? */ + cmpd cr1,r6,r9 /* or is new ESID the same as current ESID? */ + cror eq,4*cr1+eq,eq + beq 2f /* if yes, don't slbie it */ + + /* Bolt in the new stack SLB entry */ + ld r7,KSP_VSID(r4) /* Get new stack's VSID */ + oris r0,r6,(SLB_ESID_V)@h + ori r0,r0,(SLB_NUM_BOLTED-1)@l +BEGIN_FTR_SECTION + li r9,MMU_SEGSIZE_1T /* insert B field */ + oris r6,r6,(MMU_SEGSIZE_1T << SLBIE_SSIZE_SHIFT)@h + rldimi r7,r9,SLB_VSID_SSIZE_SHIFT,0 +END_MMU_FTR_SECTION_IFSET(MMU_FTR_1T_SEGMENT) + + /* Update the last bolted SLB. No write barriers are needed + * here, provided we only update the current CPU's SLB shadow + * buffer. + */ + ld r9,PACA_SLBSHADOWPTR(r13) + li r12,0 + std r12,SLBSHADOW_STACKESID(r9) /* Clear ESID */ + li r12,SLBSHADOW_STACKVSID + STDX_BE r7,r12,r9 /* Save VSID */ + li r12,SLBSHADOW_STACKESID + STDX_BE r0,r12,r9 /* Save ESID */ + + /* No need to check for MMU_FTR_NO_SLBIE_B here, since when + * we have 1TB segments, the only CPUs known to have the errata + * only support less than 1TB of system memory and we'll never + * actually hit this code path. + */ + + isync + slbie r6 +BEGIN_FTR_SECTION + slbie r6 /* Workaround POWER5 < DD2.1 issue */ +END_FTR_SECTION_IFCLR(CPU_FTR_ARCH_207S) + slbmte r7,r0 + isync +2: blr + .size pin_stack_slb,.-pin_stack_slb +#endif /* CONFIG_PPC_64S_HASH_MMU */ + #else #define FLUSH_COUNT_CACHE #endif /* CONFIG_PPC_BOOK3S_64 */
@@ -182,59 +240,12 @@ _GLOBAL(_switch) #endif ld r8,KSP(r4) /* new stack pointer */ + #ifdef CONFIG_PPC_64S_HASH_MMU BEGIN_MMU_FTR_SECTION - b 2f -END_MMU_FTR_SECTION_IFSET(MMU_FTR_TYPE_RADIX) -BEGIN_FTR_SECTION - clrrdi r6,r8,28 /* get its ESID */ - clrrdi r9,r1,28 /* get current sp ESID */ -FTR_SECTION_ELSE - clrrdi r6,r8,40 /* get its 1T ESID */ - clrrdi r9,r1,40 /* get current sp 1T ESID */ -ALT_MMU_FTR_SECTION_END_IFCLR(MMU_FTR_1T_SEGMENT) - clrldi. r0,r6,2 /* is new ESID c00000000? */ - cmpd cr1,r6,r9 /* or is new ESID the same as current ESID? */ - cror eq,4*cr1+eq,eq - beq 2f /* if yes, don't slbie it */ - - /* Bolt in the new stack SLB entry */ - ld r7,KSP_VSID(r4) /* Get new stack's VSID */ - oris r0,r6,(SLB_ESID_V)@h - ori r0,r0,(SLB_NUM_BOLTED-1)@l -BEGIN_FTR_SECTION - li r9,MMU_SEGSIZE_1T /* insert B field */ - oris r6,r6,(MMU_SEGSIZE_1T << SLBIE_SSIZE_SHIFT)@h - rldimi r7,r9,SLB_VSID_SSIZE_SHIFT,0 -END_MMU_FTR_SECTION_IFSET(MMU_FTR_1T_SEGMENT) - - /* Update the last bolted SLB. No write barriers are needed - * here, provided we only update the current CPU's SLB shadow - * buffer. - */ - ld r9,PACA_SLBSHADOWPTR(r13) - li r12,0 - std r12,SLBSHADOW_STACKESID(r9) /* Clear ESID */ - li r12,SLBSHADOW_STACKVSID - STDX_BE r7,r12,r9 /* Save VSID */ - li r12,SLBSHADOW_STACKESID - STDX_BE r0,r12,r9 /* Save ESID */ - - /* No need to check for MMU_FTR_NO_SLBIE_B here, since when - * we have 1TB segments, the only CPUs known to have the errata - * only support less than 1TB of system memory and we'll never - * actually hit this code path. - */ - - isync - slbie r6 -BEGIN_FTR_SECTION - slbie r6 /* Workaround POWER5 < DD2.1 issue */ -END_FTR_SECTION_IFCLR(CPU_FTR_ARCH_207S) - slbmte r7,r0 - isync -2: -#endif /* CONFIG_PPC_64S_HASH_MMU */ + bl pin_stack_slb +END_MMU_FTR_SECTION_IFCLR(MMU_FTR_TYPE_RADIX) +#endif clrrdi r7, r8, THREAD_SHIFT /* base of new stack */ /* Note: this uses SWITCH_FRAME_SIZE rather than INT_FRAME_SIZE
Signature · 1 line
2.40.1
Nicholas Piggin
reviewerPATCH v3 2/6 powerpc/64: Rearrange 64-bit _switch to prepare for 32/64 merge
In reply to Nicholas PigginMore some 64-bit specifics out from the function epilogue and rearrange this to be a bit neater, use 32-bit mem ops for CR save/restore, and change some register numbers.
This is preparation to consolidate 32-bit and 64-bit switch code.
--- arch/powerpc/kernel/entry_64.S | 38 ++++++++++++++++------------------ 1 file changed, 18 insertions(+), 20 deletions(-)
arch/powerpc/kernel/entry_64.S +18 -20
@@ -190,12 +190,13 @@ _GLOBAL(_switch) mflr r0 std r0,16(r1) stdu r1,-SWITCH_FRAME_SIZE(r1) + std r1,KSP(r3) /* Set old stack pointer */ /* r3-r13 are caller saved -- Cort */ SAVE_NVGPRS(r1) std r0,_NIP(r1) /* Return to switch caller */ - mfcr r23 - std r23,_CCR(r1) - std r1,KSP(r3) /* Set old stack pointer */ + mfcr r0 + stw r0,_CCR(r1) + ld r8,KSP(r4) /* Load new stack pointer */ kuap_check_amr r9, r10
@@ -232,14 +233,20 @@ _GLOBAL(_switch) DCBT_BOOK3S_STOP_ALL_STREAM_IDS(r6) #endif - addi r6,r4,-THREAD /* Convert THREAD to 'current' */ - std r6,PACACURRENT(r13) /* Set new 'current' */ + addi r3,r3,-THREAD /* old thread -> task_struct for return value */ + addi r6,r4,-THREAD /* new thread -> task_struct */ + std r6,PACACURRENT(r13) /* Set new task_struct to 'current' */ #if defined(CONFIG_STACKPROTECTOR) ld r6, TASK_CANARY(r6) std r6, PACA_CANARY(r13) #endif - - ld r8,KSP(r4) /* new stack pointer */ + /* Set the new PACAKSAVE */ + clrrdi r7, r8, THREAD_SHIFT /* base of new stack */ + /* Note: this uses SWITCH_FRAME_SIZE rather than INT_FRAME_SIZE + because we don't need to leave the 288-byte ABI gap at the + top of the kernel stack. */ + addi r7,r7,THREAD_SIZE-SWITCH_FRAME_SIZE + std r7,PACAKSAVE(r13) #ifdef CONFIG_PPC_64S_HASH_MMU BEGIN_MMU_FTR_SECTION
@@ -247,12 +254,6 @@ BEGIN_MMU_FTR_SECTION END_MMU_FTR_SECTION_IFCLR(MMU_FTR_TYPE_RADIX) #endif - clrrdi r7, r8, THREAD_SHIFT /* base of new stack */ - /* Note: this uses SWITCH_FRAME_SIZE rather than INT_FRAME_SIZE - because we don't need to leave the 288-byte ABI gap at the - top of the kernel stack. */ - addi r7,r7,THREAD_SIZE-SWITCH_FRAME_SIZE - /* * PMU interrupts in radix may come in here. They will use r1, not * PACAKSAVE, so this stack switch will not cause a problem. They
@@ -262,18 +263,15 @@ END_MMU_FTR_SECTION_IFCLR(MMU_FTR_TYPE_RADIX) * active on the new CPU, will order those stores. */ mr r1,r8 /* start using new stack pointer */ - std r7,PACAKSAVE(r13) - ld r6,_CCR(r1) - mtcrf 0xFF,r6 + lwz r0,_CCR(r1) + mtcrf 0xFF,r0 /* r3-r13 are destroyed -- Cort */ REST_NVGPRS(r1) - /* convert old thread to its task_struct for return value */ - addi r3,r3,-THREAD - ld r7,_NIP(r1) /* Return to _switch caller in new task */ - mtlr r7 + ld r0,_NIP(r1) /* Return to _switch caller in new task */ + mtlr r0 addi r1,r1,SWITCH_FRAME_SIZE blr
Signature · 1 line
2.40.1
Nicholas Piggin
reviewerPATCH v3 3/6 powerpc/32: Remove sync from _switch
In reply to Nicholas Piggin64-bit has removed the sync from _switch since commit 9145effd626d1 ("powerpc/64: Drop explicit hwsync in context switch"). The same logic there should apply to 32-bit. Remove the sync and replace with a placeholder comment (32 and 64 will be merged with a later change).
--- arch/powerpc/kernel/entry_32.S | 8 +------- 1 file changed, 1 insertion(+), 7 deletions(-)
arch/powerpc/kernel/entry_32.S +1 -7
@@ -243,13 +243,7 @@ _GLOBAL(_switch) stw r10,_CCR(r1) stw r1,KSP(r3) /* Set old stack pointer */ -#ifdef CONFIG_SMP - /* We need a sync somewhere here to make sure that if the - * previous task gets rescheduled on another CPU, it sees all - * stores it has performed on this one. - */ - sync -#endif /* CONFIG_SMP */ + /* The sync for SMP migration is taken care of, see entry_64.S */ tophys(r0,r4) mtspr SPRN_SPRG_THREAD,r0 /* Update current THREAD phys addr */
Signature · 1 line
2.40.1
Nicholas Piggin
reviewerPATCH v3 4/6 powerpc/32: Rearrange _switch to prepare for 32/64 merge
In reply to Nicholas PigginChange the order of some operations and change some register numbers in preparation to merge 32-bit and 64-bit switch.
--- arch/powerpc/kernel/entry_32.S | 10 +++++----- 1 file changed, 5 insertions(+), 5 deletions(-)
arch/powerpc/kernel/entry_32.S +5 -5
@@ -236,12 +236,12 @@ _GLOBAL(_switch) stwu r1,-SWITCH_FRAME_SIZE(r1) mflr r0 stw r0,SWITCH_FRAME_SIZE+4(r1) + stw r1,KSP(r3) /* Set old stack pointer */ /* r3-r12 are caller saved -- Cort */ SAVE_NVGPRS(r1) stw r0,_NIP(r1) /* Return to switch caller */ - mfcr r10 - stw r10,_CCR(r1) - stw r1,KSP(r3) /* Set old stack pointer */ + mfcr r0 + stw r0,_CCR(r1) /* The sync for SMP migration is taken care of, see entry_64.S */
@@ -258,8 +258,8 @@ _GLOBAL(_switch) /* r3-r12 are destroyed -- Cort */ REST_NVGPRS(r1) - lwz r4,_NIP(r1) /* Return to _switch caller in new task */ - mtlr r4 + lwz r0,_NIP(r1) /* Return to _switch caller in new task */ + mtlr r0 addi r1,r1,SWITCH_FRAME_SIZE blr
Signature · 1 line
2.40.1
Nicholas Piggin
reviewerPATCH v3 5/6 powerpc: merge 32-bit and 64-bit _switch implementation
In reply to Nicholas PigginThe _switch stack frame setup are substantially the same, so are the comments. The difference in how the stack and current are switched, and other hardware and software housekeeping is done is moved into macros.
Generated code should be unchanged.
--- arch/powerpc/include/asm/ppc_asm.h | 14 ++ arch/powerpc/kernel/Makefile | 2 +- arch/powerpc/kernel/entry_32.S | 52 ------ arch/powerpc/kernel/entry_64.S | 229 ------------------------- arch/powerpc/kernel/switch.S | 258 +++++++++++++++++++++++++++++ 5 files changed, 273 insertions(+), 282 deletions(-) create mode 100644 arch/powerpc/kernel/switch.S
arch/powerpc/include/asm/ppc_asm.h +14 -0
@@ -406,6 +406,15 @@ GLUE(.,name):/* offsets for stack frame layout */ #define LRSAVE 16 +/* +* GCC stack frames follow a different pattern on 32 vs 64. This can be used +* to make asm frames be consistent with C. +*/ +#define PPC_CREATE_STACK_FRAME (size) \ + mflr r0; \ + std r0,16(r1); \ + stdu r1,-(size)(r1) +#else /* 32-bit */ #define LOAD_REG_IMMEDIATE (reg, expr) __LOAD_REG_IMMEDIATE_32 reg, expr
@@ -422,6 +431,11 @@ GLUE(.,name):/* offsets for stack frame layout */ #define LRSAVE 4 +#define PPC_CREATE_STACK_FRAME (size) \ + stwu r1,-(size)(r1); \ + mflr r0; \ + stw r0,(size+4)(r1) +#endif /* various errata or part fixups */
arch/powerpc/kernel/Makefile +1 -1
@@ -68,7 +68,7 @@ CFLAGS_REMOVE_syscall.o = -fstack-protector -fstack-protector-strong CFLAGS_syscall.o += -fno-stack-protector #endif -obj-y := cputable.o syscalls.o \ +obj-y := cputable.o syscalls.o switch.o \ irq.o align.o signal_$(BITS).o pmc.o vdso.o \ process.o systbl.o idle.o \ signal.o sysfs.o cacheinfo.o time.o \
arch/powerpc/kernel/entry_32.S +0 -52
58 lines not on this page · raw message
arch/powerpc/kernel/entry_64.S +0 -229
243 lines not on this page · raw message
arch/powerpc/kernel/switch.S new file +258 -0
258 lines not on this page · raw message
Signature · 1 line
2.40.1
Nicholas Piggin
reviewerPATCH v3 6/6 powerpc/64: Rename entry_64.S to prom_entry_64.S
In reply to Nicholas PigginThis file contains only the enter_prom implementation now. Trim includes and update header comment while we're here.
---
arch/powerpc/kernel/Makefile | 12 ++++----
.../kernel/{entry_64.S => prom_entry_64.S} | 30 ++-----------------
scripts/head-object-list.txt | 2 +-
3 files changed, 10 insertions(+), 34 deletions(-)
rename arch/powerpc/kernel/{entry_64.S => prom_entry_64.S} (73%)arch/powerpc/kernel/Makefile +6 -6
@@ -165,9 +165,6 @@ endif obj64-$(CONFIG_PPC_TRANSACTIONAL_MEM) += tm.o -obj-$(CONFIG_PPC64) += $(obj64-y) -obj-$(CONFIG_PPC32) += $(obj32-y) - ifneq ($(CONFIG_XMON)$(CONFIG_KEXEC_CORE)(CONFIG_PPC_BOOK3S),) obj-y += ppc_save_regs.o endif
@@ -209,10 +206,13 @@ CFLAGS_paca.o += -fno-stack-protector obj-$(CONFIG_PPC_FPU) += fpu.o obj-$(CONFIG_ALTIVEC) += vector.o -obj-$(CONFIG_PPC64) += entry_64.o -obj-$(CONFIG_PPC_OF_BOOT_TRAMPOLINE) += prom_init.o -extra-$(CONFIG_PPC_OF_BOOT_TRAMPOLINE) += prom_init_check +obj-$(CONFIG_PPC_OF_BOOT_TRAMPOLINE) += prom_init.o +obj64-$(CONFIG_PPC_OF_BOOT_TRAMPOLINE) += prom_entry_64.o +extra-$(CONFIG_PPC_OF_BOOT_TRAMPOLINE) += prom_init_check + +obj-$(CONFIG_PPC64) += $(obj64-y) +obj-$(CONFIG_PPC32) += $(obj32-y) quiet_cmd_prom_init_check = PROMCHK $@ cmd_prom_init_check = $(CONFIG_SHELL) $< "$(NM)" $(obj)/prom_init.o; touch $@
arch/powerpc/kernel/prom_entry_64.S renamed from arch/powerpc/kernel/entry_64.S +3 -27
@@ -10,41 +10,17 @@ * Copyright (C) 1996 Paul Mackerras. * MPC8xx modifications Copyright (C) 1997 Dan Malek ([email protected]). * - * This file contains the system call entry code, context switch - * code, and exception/interrupt return code for PowerPC. + * This file contains the 64-bit prom entry code. */ - -#include <linux/errno.h> -#include <linux/err.h> -#include <asm/cache.h> -#include <asm/unistd.h> -#include <asm/processor.h> -#include <asm/page.h> -#include <asm/mmu.h> -#include <asm/thread_info.h> -#include <asm/code-patching-asm.h> -#include <asm/ppc_asm.h> #include <asm/asm-offsets.h> -#include <asm/cputable.h> -#include <asm/firmware.h> -#include <asm/bug.h> -#include <asm/ptrace.h> -#include <asm/irqflags.h> -#include <asm/hw_irq.h> -#include <asm/context_tracking.h> -#include <asm/ppc-opcode.h> -#include <asm/barrier.h> -#include <asm/export.h> -#include <asm/asm-compat.h> #ifdef CONFIG_PPC_BOOK3S #include <asm/exception-64s.h> #else #include <asm/exception-64e.h> #endif -#include <asm/feature-fixups.h> -#include <asm/kup.h> +#include <asm/ppc_asm.h> - .section ".text" +.section ".text","ax",@progbits _GLOBAL(enter_prom) mflr r0
scripts/head-object-list.txt +1 -1
@@ -34,7 +34,7 @@ arch/powerpc/kernel/head_64.o arch/powerpc/kernel/head_8xx.o arch/powerpc/kernel/head_85xx.o arch/powerpc/kernel/head_book3s_32.o -arch/powerpc/kernel/entry_64.o +arch/powerpc/kernel/prom_entry_64.o arch/powerpc/kernel/fpu.o arch/powerpc/kernel/vector.o arch/powerpc/kernel/prom_init.o
Signature · 1 line
2.40.1
Michael Ellerman
AppliedQuoted 6 lines · Nicholas Piggin
Since v2:
- AddPPC_CREATE_STACK_FRAME()to abstract prologue differences.
- Build fix.
-Makefiletidy [Christophe]
- Fix a missing SOB.
Quoted 8 lines
Since v1:
- Don't re-order 32-bit prologue.
- ImproveKconfigconditional includes.
- Break out code changes into their own patches before merging, so merge patch leaves generated code unchanged.
- Changeprom_entry.Stoprom_entry_64.S.[...]
Applied to powerpc/next.
[1/6] powerpc/64s: move stack SLB pinning out of line from _switch
https://git.kernel.org/powerpc/c/d6b87c3eb6b2e0b34ba747df549e08768b019fe9 [2/6] powerpc/64: Rearrange 64-bit _switch to prepare for 32/64 merge https://git.kernel.org/powerpc/c/0eb8088b5a7524f96cadfb27083f5bdd819d9d52 [3/6] powerpc/32: Remove sync from _switch https://git.kernel.org/powerpc/c/fc8562c9b69af9533c39903b1601c378742189b0 [4/6] powerpc/32: Rearrange _switch to prepare for 32/64 merge https://git.kernel.org/powerpc/c/6958ad05d5789a303afe4fa4495df43993d9b7cb [5/6] powerpc: merge 32-bit and 64-bit _switch implementation https://git.kernel.org/powerpc/c/afc6386815a88d067d9f567dcc6266800286f626 [6/6] powerpc/64: Rename entry_64.S to prom_entry_64.S https://git.kernel.org/powerpc/c/27be2456332dcd69907f086cda327ad923b222cf
cheers