SLB miss conversion to C, and SLB optimisations
Nicholas Piggin
reviewercover SLB miss conversion to C, and SLB optimisations
This is a repost of the SLB conversion to C, no real change since last post. But given that slows down the SLB miss handler, I promised some optimisations could be made to mitigate that.
The two main optimisations after the C conversion are the SLB alloation bitmaps, and the preload cache.
Thanks,
Nick
Nicholas Piggin (12):
powerpc/64s/hash: Fix stab_rr off by one initialization powerpc/64s/hash: avoid the POWER5 < DD2.1 slb invalidate workaround
on POWER8/9
powerpc/64s/hash: move POWER5 < DD2.1 slbie workaround where it is
needed
powerpc/64s/hash: remove the vmalloc segment from the bolted SLB powerpc/64s/hash: Use POWER6 SLBIA IH=1 variant in switch_slb powerpc/64s/hash: Use POWER9 SLBIA IH=3 variant in switch_slb powerpc/64s/hash: convert SLB miss handlers to C powerpc/64s/hash: remove user SLB data from the paca powerpc/64s/hash: SLB allocation status bitmaps powerpc/64s: xmon do not dump hash fields when using radix mode powerpc/64s/hash: provide arch_setup_exec hooks for hash slice setup powerpc/64s/hash: Add a SLB preload cache
arch/powerpc/include/asm/asm-prototypes.h | 2 + arch/powerpc/include/asm/book3s/64/mmu-hash.h | 5 +- arch/powerpc/include/asm/exception-64s.h | 8 - arch/powerpc/include/asm/paca.h | 19 +- arch/powerpc/include/asm/processor.h | 1 + arch/powerpc/include/asm/slice.h | 1 + arch/powerpc/include/asm/thread_info.h | 11 + arch/powerpc/kernel/asm-offsets.c | 11 +- arch/powerpc/kernel/entry_64.S | 2 + arch/powerpc/kernel/exceptions-64s.S | 202 ++---- arch/powerpc/kernel/paca.c | 21 - arch/powerpc/kernel/process.c | 16 + arch/powerpc/mm/Makefile | 2 +- arch/powerpc/mm/hash_utils_64.c | 46 +- arch/powerpc/mm/mmu_context.c | 3 +- arch/powerpc/mm/mmu_context_book3s64.c | 9 + arch/powerpc/mm/slb.c | 596 ++++++++++++------ arch/powerpc/mm/slb_low.S | 335 ---------- arch/powerpc/mm/slice.c | 43 +- arch/powerpc/xmon/xmon.c | 37 +- 20 files changed, 540 insertions(+), 830 deletions(-) delete mode 100644 arch/powerpc/mm/slb_low.S
Signature · 1 line
2.18.0
Nicholas Piggin
reviewerPATCH 1/12 powerpc/64s/hash: Fix stab_rr off by one initialization
This causes SLB alloation to start 1 beyond the start of the SLB. There is no real problem because after it wraps it stats behaving properly, it's just surprisig to see when looking at SLB traces.
--- arch/powerpc/mm/slb.c | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-)
arch/powerpc/mm/slb.c +1 -1
@@ -355,7 +355,7 @@ void slb_initialize(void)#endif } -get_paca ()->stab_rr =SLB_NUM_BOLTED ; +get_paca ()->stab_rr =SLB_NUM_BOLTED -1 ; lflags =SLB_VSID_KERNEL | linear_llp; vflags =SLB_VSID_KERNEL | vmalloc_llp;
Signature · 1 line
2.18.0
Nicholas Piggin
reviewerPATCH 2/12 powerpc/64s/hash: avoid the POWER5 < DD2.1 slb invalidate workaround on POWER8/9
In reply to Nicholas PigginI only have POWER8/9 to test, so just remove it for those.
--- arch/powerpc/kernel/entry_64.S | 2 ++ arch/powerpc/mm/slb.c | 8 +++++--- 2 files changed, 7 insertions(+), 3 deletions(-)
arch/powerpc/kernel/entry_64.S +2 -0
@@ -672,7 +672,9 @@ END_MMU_FTR_SECTION_IFSET(MMU_FTR_1T_SEGMENT) isync slbie r6 +BEGIN_FTR_SECTION slbie r6 /* Workaround POWER5 < DD2.1 issue */ +END_FTR_SECTION_IFCLR(CPU_FTR_ARCH_207S) slbmte r7,r0 isync 2:
arch/powerpc/mm/slb.c +5 -3
@@ -256,9 +256,11 @@ void switch_slb(struct task_struct *tsk, struct mm_struct *mm)__slb_flush_and_rebolt (); } -/* Workaround POWER5 < DD2.1 issue */ -if (offset ==1 || offset >SLB_CACHE_ENTRIES ) - asmvolatile ("slbie %0" : :"r" (slbie_data)); +if (!cpu_has_feature (CPU_FTR_ARCH_207S )) { +/* Workaround POWER5 < DD2.1 issue */ +if (offset ==1 || offset >SLB_CACHE_ENTRIES ) + asmvolatile ("slbie %0" : :"r" (slbie_data)); + }get_paca ()->slb_cache_ptr =0 ;copy_mm_to_paca (mm);
Signature · 1 line
2.18.0
Nicholas Piggin
reviewerPATCH 3/12 powerpc/64s/hash: move POWER5 < DD2.1 slbie workaround where it is needed
In reply to Nicholas PigginThe POWER5 < DD2.1 issue is that slbie needs to be issued more than once. It came in with this change:
[email protected], 2004-04-29 07:12:31-07:00, [email protected]
[PATCH] POWER5 erratum workaround
Early POWER5 revisions (<DD2.1) have a problem requiring slbie instructions to be repeated under some circumstances. The patch below adds a workaround (patch made by Anton Blanchard).
The extra slbie in switch_slb is done even for the case where slbia is called (slb_flush_and_rebolt). I don't believe that is required because there are other slb_flush_and_rebolt callers which do not issue the workaround slbie, which would be broken if it was required.
It also seems to be fine inside the isync with the first slbie, as it is in the kernel stack switch code.
So move this workaround to where it is required. This is not much of an optimisation because this is the fast path, but it makes the code more understandable and neater.
--- arch/powerpc/mm/slb.c | 14 +++++++------- 1 file changed, 7 insertions(+), 7 deletions(-)
arch/powerpc/mm/slb.c +7 -7
@@ -226,7 +226,6 @@ static inline int esids_match(unsigned long addr1, unsigned long addr2)void switch_slb (struct task_struct *tsk,struct mm_struct *mm) {unsigned long offset; -unsigned long slbie_data =0 ;unsigned long pc =KSTK_EIP (tsk);unsigned long stack =KSTK_ESP (tsk);unsigned long exec_base;
@@ -241,7 +240,9 @@ void switch_slb(struct task_struct *tsk, struct mm_struct *mm) offset =get_paca ()->slb_cache_ptr;if (!mmu_has_feature (MMU_FTR_NO_SLBIE_B ) && offset <=SLB_CACHE_ENTRIES ) { +unsigned long slbie_data;int i; + asmvolatile ("isync" : : :"memory" );for (i =0 ; i < offset; i++) { slbie_data = (unsigned long )get_paca ()->slb_cache[i]
@@ -251,15 +252,14 @@ void switch_slb(struct task_struct *tsk, struct mm_struct *mm) slbie_data |=SLBIE_C ;/* C set for user addresses */ asmvolatile ("slbie %0" : :"r" (slbie_data)); } - asmvolatile ("isync" : : :"memory" ); - }else { -__slb_flush_and_rebolt (); - } -if (!cpu_has_feature (CPU_FTR_ARCH_207S )) {/* Workaround POWER5 < DD2.1 issue */ -if (offset ==1 || offset >SLB_CACHE_ENTRIES ) +if (!cpu_has_feature (CPU_FTR_ARCH_207S ) && offset ==1 ) asmvolatile ("slbie %0" : :"r" (slbie_data)); + + asmvolatile ("isync" : : :"memory" ); + }else { +__slb_flush_and_rebolt (); }get_paca ()->slb_cache_ptr =0 ;
Signature · 1 line
2.18.0
Nicholas Piggin
reviewerPATCH 4/12 powerpc/64s/hash: remove the vmalloc segment from the bolted SLB
In reply to Nicholas PigginRemove the vmalloc segment from bolted SLBEs. This is not required to be bolted, and seems like it was added to help pre-load the SLB on context switch. However there are now other segments like the vmemmap segment and non-zero node memory that often take misses after a context switch, so it is better to solve this in a more general way.
A subsequent change will track free SLB entries and uses those rather than round-robin overwrite valid entries, which makes it far less likely for kernel SLBEs to be evicted after they are installed.
--- arch/powerpc/include/asm/book3s/64/mmu-hash.h | 2 +- arch/powerpc/mm/slb.c | 23 ++++--------------- 2 files changed, 6 insertions(+), 19 deletions(-)
arch/powerpc/include/asm/book3s/64/mmu-hash.h +1 -1
@@ -30,7 +30,7 @@* SLB */ -#define SLB_NUM_BOLTED 3 +#define SLB_NUM_BOLTED 2#define SLB_CACHE_ENTRIES 8#define SLB_MIN_SIZE 32
arch/powerpc/mm/slb.c +5 -18
@@ -30,8 +30,7 @@enum slb_index {LINEAR_INDEX =0 ,/* Kernel linear map (0xc000000000000000) */ -VMALLOC_INDEX =1 ,/* Kernel virtual map (0xd000000000000000) */ -KSTACK_INDEX =2 ,/* Kernel stack map */ +KSTACK_INDEX =1 ,/* Kernel stack map */ };extern void slb_allocate (unsigned long ea);
@@ -133,13 +132,11 @@ static void __slb_flush_and_rebolt(void) {/* If you change this make sure you change SLB_NUM_BOLTED * and PR KVM appropriately too. */ -unsigned long linear_llp, vmalloc_llp, lflags, vflags; +unsigned long linear_llp, lflags;unsigned long ksp_esid_data, ksp_vsid_data; linear_llp = mmu_psize_defs[mmu_linear_psize].sllp; - vmalloc_llp = mmu_psize_defs[mmu_vmalloc_psize].sllp; lflags =SLB_VSID_KERNEL | linear_llp; - vflags =SLB_VSID_KERNEL | vmalloc_llp; ksp_esid_data =mk_esid_data (get_paca ()->kstack, mmu_kernel_ssize,KSTACK_INDEX );if ((ksp_esid_data & ~0xfffffffUL ) <=PAGE_OFFSET ) {
@@ -157,14 +154,10 @@ static void __slb_flush_and_rebolt(void)* the stack between the slbia and rebolting it. */ asmvolatile ("isync\n" "slbia\n" -/* Slot 1 - first VMALLOC segment */ +/* Slot 1 - kernel stack */ "slbmte %0,%1\n" -/* Slot 2 - kernel stack */ -"slbmte %2,%3\n" "isync" - ::"r" (mk_vsid_data (VMALLOC_START , mmu_kernel_ssize, vflags)), -"r" (mk_esid_data (VMALLOC_START , mmu_kernel_ssize,VMALLOC_INDEX )), -"r" (ksp_vsid_data), + ::"r" (ksp_vsid_data),"r" (ksp_esid_data) :"memory" ); }
@@ -186,10 +179,6 @@ void slb_flush_and_rebolt(void)void slb_vmalloc_update (void ) { -unsigned long vflags; - - vflags =SLB_VSID_KERNEL | mmu_psize_defs[mmu_vmalloc_psize].sllp; -slb_shadow_update (VMALLOC_START , mmu_kernel_ssize, vflags,VMALLOC_INDEX );slb_flush_and_rebolt (); }
@@ -324,7 +313,7 @@ void slb_set_size(u16 size)void slb_initialize (void ) {unsigned long linear_llp, vmalloc_llp, io_llp; -unsigned long lflags, vflags; +unsigned long lflags;static int slb_encoding_inited;#ifdef CONFIG_SPARSEMEM_VMEMMAP unsigned long vmemmap_llp;
@@ -360,14 +349,12 @@ void slb_initialize(void)get_paca ()->stab_rr =SLB_NUM_BOLTED -1 ; lflags =SLB_VSID_KERNEL | linear_llp; - vflags =SLB_VSID_KERNEL | vmalloc_llp;/* Invalidate the entire SLB (even entry 0) & all the ERATS */ asmvolatile ("isync" :::"memory" ); asmvolatile ("slbmte %0,%0" ::"r" (0 ) :"memory" ); asmvolatile ("isync; slbia; isync" :::"memory" );create_shadowed_slbe (PAGE_OFFSET , mmu_kernel_ssize, lflags,LINEAR_INDEX ); -create_shadowed_slbe (VMALLOC_START , mmu_kernel_ssize, vflags,VMALLOC_INDEX );/* For the boot cpu, we're running on the stack in init_thread_union, * which is in the first segment of the linear mapping, and also
Signature · 1 line
2.18.0
Nicholas Piggin
reviewerPATCH 5/12 powerpc/64s/hash: Use POWER6 SLBIA IH=1 variant in switch_slb
In reply to Nicholas PigginThe SLBIA IH=1 hint will remove all non-zero SLBEs, but only invalidate ERAT entries associated with a class value of 1, for processors that support the hint (e.g., POWER6 and newer), which Linux assigns to user addresses.
This prevents kernel ERAT entries from being invalidated when context switchig (if the thread faulted in more than 8 user SLBEs).
--- arch/powerpc/mm/slb.c | 38 +++++++++++++++++++++++--------------- 1 file changed, 23 insertions(+), 15 deletions(-)
arch/powerpc/mm/slb.c +23 -15
@@ -128,13 +128,21 @@ void slb_flush_all_realmode(void) asmvolatile ("slbmte %0,%0; slbia" : :"r" (0 )); } -static void __slb_flush_and_rebolt (void ) +void slb_flush_and_rebolt (void ) {/* If you change this make sure you change SLB_NUM_BOLTED * and PR KVM appropriately too. */ unsigned long linear_llp, lflags;unsigned long ksp_esid_data, ksp_vsid_data; +WARN_ON (!irqs_disabled ()); + +/* +* We can't take a PMU exception in the following code, so hard +* disable interrupts. +*/ +hard_irq_disable (); + linear_llp = mmu_psize_defs[mmu_linear_psize].sllp; lflags =SLB_VSID_KERNEL | linear_llp;
@@ -160,20 +168,7 @@ static void __slb_flush_and_rebolt(void) :: "r"(ksp_vsid_data), "r"(ksp_esid_data) : "memory"); -} -void slb_flush_and_rebolt (void ) -{ - -WARN_ON (!irqs_disabled ()); - -/* -* We can't take a PMU exception in the following code, so hard -* disable interrupts. -*/ -hard_irq_disable (); - -__slb_flush_and_rebolt ();get_paca ()->slb_cache_ptr =0 ; }
@@ -248,7 +243,20 @@ void switch_slb(struct task_struct *tsk, struct mm_struct *mm) asmvolatile ("isync" : : :"memory" ); }else { -__slb_flush_and_rebolt (); +struct slb_shadow *p =get_slb_shadow (); +unsigned long ksp_esid_data = +be64_to_cpu (p->save_area[KSTACK_INDEX ].esid); +unsigned long ksp_vsid_data = +be64_to_cpu (p->save_area[KSTACK_INDEX ].vsid); + + asmvolatile ("isync\n" + PPC_SLBIA(1) "\n" + "slbmte %0,%1\n" + "isync" + :: "r"(ksp_vsid_data), + "r"(ksp_esid_data)); + + asmvolatile ("isync" : : :"memory" ); }get_paca ()->slb_cache_ptr =0 ;
Signature · 1 line
2.18.0
Nicholas Piggin
reviewerPATCH 6/12 powerpc/64s/hash: Use POWER9 SLBIA IH=3 variant in switch_slb
In reply to Nicholas PigginPOWER9 introduces SLBIA IH=3, which invalidates all SLB entries and associated lookaside information that have a class value of 1, which Linux assigns to user addresses. This matches what switch_slb wants, and allows a simple fast implementation that avoids the slb_cache complexity.
As a side-effect, the POWER5 < DD2.1 SLB invalidation workaround is also avoided on POWER9.
Process context switching rate is improved about 2.2% for a small process that hits the slb cache which is the best case for the current code.
--- arch/powerpc/mm/slb.c | 86 +++++++++++++++++++++++----------------- arch/powerpc/xmon/xmon.c | 11 +++-- 2 files changed, 57 insertions(+), 40 deletions(-)
arch/powerpc/mm/slb.c +50 -36
@@ -209,7 +209,6 @@ static inline int esids_match(unsigned long addr1, unsigned long addr2)/* Flush all user entries from the segment table of the current processor. */ void switch_slb (struct task_struct *tsk,struct mm_struct *mm) { -unsigned long offset;unsigned long pc =KSTK_EIP (tsk);unsigned long stack =KSTK_ESP (tsk);unsigned long exec_base;
@@ -221,45 +220,57 @@ void switch_slb(struct task_struct *tsk, struct mm_struct *mm)* which would update the slb_cache/slb_cache_ptr fields in the PACA. */ hard_irq_disable (); - offset =get_paca ()->slb_cache_ptr; -if (!mmu_has_feature (MMU_FTR_NO_SLBIE_B ) && - offset <=SLB_CACHE_ENTRIES ) { -unsigned long slbie_data; -int i; - - asmvolatile ("isync" : : :"memory" ); -for (i =0 ; i < offset; i++) { - slbie_data = (unsigned long )get_paca ()->slb_cache[i] - <<SID_SHIFT ;/* EA */ - slbie_data |=user_segment_size (slbie_data) - <<SLBIE_SSIZE_SHIFT ; - slbie_data |=SLBIE_C ;/* C set for user addresses */ - asmvolatile ("slbie %0" : :"r" (slbie_data)); - } - -/* Workaround POWER5 < DD2.1 issue */ -if (!cpu_has_feature (CPU_FTR_ARCH_207S ) && offset ==1 ) - asmvolatile ("slbie %0" : :"r" (slbie_data)); +if (cpu_has_feature (CPU_FTR_ARCH_300 )) { +/* +* SLBIA IH=3 invalidates all Class=1 SLBEs and their +* associated lookaside structures, which matches what +* switch_slb wants. So ARCH_300 does not use the slb +* cache. +*/ + asm volatile("isync ; " PPC_SLBIA(3)" ; isync"); - asmvolatile ("isync" : : :"memory" ); }else { -struct slb_shadow *p =get_slb_shadow (); -unsigned long ksp_esid_data = -be64_to_cpu (p->save_area[KSTACK_INDEX ].esid); -unsigned long ksp_vsid_data = -be64_to_cpu (p->save_area[KSTACK_INDEX ].vsid); - - asmvolatile ("isync\n" - PPC_SLBIA(1) "\n" - "slbmte %0,%1\n" - "isync" - :: "r"(ksp_vsid_data), - "r"(ksp_esid_data)); - - asmvolatile ("isync" : : :"memory" ); +unsigned long offset =get_paca ()->slb_cache_ptr; + +if (!mmu_has_feature (MMU_FTR_NO_SLBIE_B ) && + offset <=SLB_CACHE_ENTRIES ) { +unsigned long slbie_data; +int i; + + asmvolatile ("isync" : : :"memory" ); +for (i =0 ; i < offset; i++) { +/* EA */ + slbie_data = (unsigned long ) +get_paca ()->slb_cache[i] <<SID_SHIFT ; + slbie_data |=user_segment_size (slbie_data) + <<SLBIE_SSIZE_SHIFT ; + slbie_data |=SLBIE_C ;/* user slbs have C=1 */ + asmvolatile ("slbie %0" : :"r" (slbie_data)); + } + +/* Workaround POWER5 < DD2.1 issue */ +if (!cpu_has_feature (CPU_FTR_ARCH_207S ) && offset ==1 ) + asmvolatile ("slbie %0" : :"r" (slbie_data)); + + asmvolatile ("isync" : : :"memory" ); + }else { +struct slb_shadow *p =get_slb_shadow (); +unsigned long ksp_esid_data = +be64_to_cpu (p->save_area[KSTACK_INDEX ].esid); +unsigned long ksp_vsid_data = +be64_to_cpu (p->save_area[KSTACK_INDEX ].vsid); + + asmvolatile ("isync\n" + PPC_SLBIA(1) "\n" + "slbmte %0,%1\n" + "isync" + :: "r"(ksp_vsid_data), + "r"(ksp_esid_data)); + } + +get_paca ()->slb_cache_ptr =0 ; } -get_paca ()->slb_cache_ptr =0 ;copy_mm_to_paca (mm);/*
@@ -385,6 +396,9 @@ static void insert_slb_entry(unsigned long vsid, unsigned long ea,enum slb_index index;int slb_cache_index; +if (cpu_has_feature (CPU_FTR_ARCH_300 )) +return ;/* ISAv3.0B and later does not use slb_cache */ +/* * We are irq disabled, hence should be safe to access PACA. */
arch/powerpc/xmon/xmon.c +7 -4
@@ -2393,10 +2393,13 @@ static void dump_one_paca(int cpu) } }DUMP (p, vmalloc_sllp,"%#-*x" ); -DUMP (p, slb_cache_ptr,"%#-*x" ); -for (i =0 ; i <SLB_CACHE_ENTRIES ; i++) -printf (" %-*s[%d] = 0x%016x\n" , -22 ,"slb_cache" , i, p->slb_cache[i]); + +if (!early_cpu_has_feature (CPU_FTR_ARCH_300 )) { +DUMP (p, slb_cache_ptr,"%#-*x" ); +for (i =0 ; i <SLB_CACHE_ENTRIES ; i++) +printf (" %-*s[%d] = 0x%016x\n" , +22 ,"slb_cache" , i, p->slb_cache[i]); + }DUMP (p, rfi_flush_fallback_area,"%-*px" );#endif
Signature · 1 line
2.18.0
Nicholas Piggin
reviewerPATCH 7/12 powerpc/64s/hash: convert SLB miss handlers to C
In reply to Nicholas PigginThis patch moves SLB miss handlers completely to C, using the standard exception handler macros to set up the stack and branch to C.
This can be done because the segment containing the kernel stack is always bolted, so accessing it with relocation on will not cause an SLB exception.
Arbitrary kernel memory may not be accessed when handling kernel space SLB misses, so care should be taken there. However user SLB misses can access any kernel memory, which can be used to move some fields out of the paca (in later patches).
User SLB misses could quite easily reconcile IRQs and set up a first class kernel environment and exit via ret_from_except, however that doesn't seem to be necessary at the moment, so we only do that if a bad fault is encountered.
[ Credit to Aneesh for bug fixes, error checks, and improvements to bad
address handling, etc ]
Since RFC:
- Added MSR[RI] handling
- Fixed up a register loss bug exposed by irq tracing (Aneesh)
- Reject misses outside the defined kernel regions (Aneesh)
- Added several more sanity checks and error handling (Aneesh), we may look at consolidating these tests and tightenig up the code but for a first pass we decided it's better to check carefully.
Since v1:
- Fixed SLB cache corruption (Aneesh)
- Fixed untidy SLBE allocation "leak" in get_vsid error case
- Now survives some stress testing on real hardware
--- arch/powerpc/include/asm/asm-prototypes.h | 2 + arch/powerpc/include/asm/exception-64s.h | 8 - arch/powerpc/kernel/exceptions-64s.S | 202 +++---------- arch/powerpc/mm/Makefile | 2 +- arch/powerpc/mm/slb.c | 271 +++++++++-------- arch/powerpc/mm/slb_low.S | 335 ---------------------- 6 files changed, 196 insertions(+), 624 deletions(-) delete mode 100644 arch/powerpc/mm/slb_low.S
arch/powerpc/include/asm/asm-prototypes.h +2 -0
@@ -78,6 +78,8 @@ void kernel_bad_stack(struct pt_regs *regs);void system_reset_exception (struct pt_regs *regs);void machine_check_exception (struct pt_regs *regs);void emulation_assist_interrupt (struct pt_regs *regs); +long do_slb_fault (struct pt_regs *regs,unsigned long ea); +void do_bad_slb_fault (struct pt_regs *regs,unsigned long ea,long err);/* signals, syscalls and interrupts */ long sys_swapcontext(struct ucontext __user *old_ctx,
arch/powerpc/include/asm/exception-64s.h +0 -8
@@ -60,14 +60,6 @@*/ #define MAX_MCE_DEPTH 4 -/* -* EX_LR is only used in EXSLB and where it does not overlap with EX_DAR -* EX_CCR similarly with DSISR, but being 4 byte registers there is a hole -* in the save area so it's not necessary to overlap them. Could be used -* for future savings though if another 4 byte register was to be saved. -*/ -#define EX_LR EX_DAR -/* * EX_R3 is only used by the bad_stack handler. bad_stack reloads and * saves DAR from SPRN_DAR, and EX_DAR is not used. So EX_R3 can overlap
arch/powerpc/kernel/exceptions-64s.S +42 -160
230 lines not on this page · raw message
arch/powerpc/mm/Makefile +1 -1
@@ -15,7 +15,7 @@ obj-$(CONFIG_PPC_MMU_NOHASH) += mmu_context_nohash.o tlb_nohash.o \ obj-$(CONFIG_PPC_BOOK3E) += tlb_low_$(BITS)e.o hash64-$(CONFIG_PPC_NATIVE) := hash_native_64.o obj-$(CONFIG_PPC_BOOK3E_64) += pgtable-book3e.o -obj-$(CONFIG_PPC_BOOK3S_64) += pgtable-hash64.o hash_utils_64.o slb_low.o slb.o $(hash64-y) mmu_context_book3s64.o pgtable-book3s64.o +obj-$(CONFIG_PPC_BOOK3S_64) += pgtable-hash64.o hash_utils_64.o slb.o $(hash64-y) mmu_context_book3s64.o pgtable-book3s64.o obj-$(CONFIG_PPC_RADIX_MMU) += pgtable-radix.o tlb-radix.o obj-$(CONFIG_PPC_STD_MMU_32) += ppc_mmu_32.o hash_low_32.o mmu_context_hash32.o obj-$(CONFIG_PPC_STD_MMU) += tlb_hash$(BITS).o
arch/powerpc/mm/slb.c +151 -120
355 lines not on this page · raw message
arch/powerpc/mm/slb_low.S deleted +0 -335
335 lines not on this page · raw message
Signature · 1 line
2.18.0
Nicholas Piggin
reviewerPATCH 8/12 powerpc/64s/hash: remove user SLB data from the paca
In reply to Nicholas PigginUser SLB mappig data is copied into the PACA from the mm->context so it can be accessed by the SLB miss handlers.
After the C conversion, SLB miss handlers now run with relocation on, and user SLB misses are able to take recursive kernel SLB misses, so the user SLB mapping data can be removed from the paca and accessed directly.
--- arch/powerpc/include/asm/book3s/64/mmu-hash.h | 1 + arch/powerpc/include/asm/paca.h | 13 ------ arch/powerpc/kernel/asm-offsets.c | 9 ---- arch/powerpc/kernel/paca.c | 21 --------- arch/powerpc/mm/hash_utils_64.c | 46 +++++-------------- arch/powerpc/mm/mmu_context.c | 3 +- arch/powerpc/mm/slb.c | 20 +++++++- arch/powerpc/mm/slice.c | 29 ++++-------- 8 files changed, 40 insertions(+), 102 deletions(-)
arch/powerpc/include/asm/book3s/64/mmu-hash.h +1 -0
@@ -496,6 +496,7 @@ static inline void hpte_init_pseries(void) { }extern void hpte_init_native (void );extern void slb_initialize (void ); +extern void core_flush_all_slbs (struct mm_struct *mm);extern void slb_flush_and_rebolt (void );void slb_flush_all_realmode (void );void __slb_restore_bolted_realmode (void );
arch/powerpc/include/asm/paca.h +0 -13
@@ -143,18 +143,6 @@ struct paca_struct {struct tlb_core_data tcd;#endif /* CONFIG_PPC_BOOK3E */ -#ifdef CONFIG_PPC_BOOK3S -mm_context_id_t mm_ctx_id; -#ifdef CONFIG_PPC_MM_SLICES -unsigned char mm_ctx_low_slices_psize[BITS_PER_LONG /BITS_PER_BYTE ]; -unsigned char mm_ctx_high_slices_psize[SLICE_ARRAY_SIZE ]; -unsigned long mm_ctx_slb_addr_limit; -#else -u16 mm_ctx_user_psize; -u16 mm_ctx_sllp; -#endif -#endif -/* * then miscellaneous read-write fields */
@@ -256,7 +244,6 @@ struct paca_struct {#endif /* CONFIG_PPC_PSERIES */ }____cacheline_aligned ; -extern void copy_mm_to_paca (struct mm_struct *mm);extern struct paca_struct **paca_ptrs;extern void initialise_paca (struct paca_struct *new_paca,int cpu);extern void setup_paca (struct paca_struct *new_paca);
arch/powerpc/kernel/asm-offsets.c +0 -9
@@ -181,15 +181,6 @@ int main(void)OFFSET (PACAIRQSOFTMASK , paca_struct, irq_soft_mask);OFFSET (PACAIRQHAPPENED , paca_struct, irq_happened);OFFSET (PACA_FTRACE_ENABLED , paca_struct, ftrace_enabled); -#ifdef CONFIG_PPC_BOOK3S -OFFSET (PACACONTEXTID , paca_struct, mm_ctx_id); -#ifdef CONFIG_PPC_MM_SLICES -OFFSET (PACALOWSLICESPSIZE , paca_struct, mm_ctx_low_slices_psize); -OFFSET (PACAHIGHSLICEPSIZE , paca_struct, mm_ctx_high_slices_psize); -OFFSET (PACA_SLB_ADDR_LIMIT , paca_struct, mm_ctx_slb_addr_limit); -DEFINE (MMUPSIZEDEFSIZE ,sizeof (struct mmu_psize_def )); -#endif /* CONFIG_PPC_MM_SLICES */ -#endif #ifdef CONFIG_PPC_BOOK3E OFFSET (PACAPGD , paca_struct, pgd);
arch/powerpc/kernel/paca.c +0 -21
@@ -259,24 +259,3 @@ void __init free_unused_pacas(void) paca_ptrs_size + paca_struct_size, nr_cpu_ids); } -void copy_mm_to_paca (struct mm_struct *mm) -{ -#ifdef CONFIG_PPC_BOOK3S -mm_context_t *context = &mm->context; - -get_paca ()->mm_ctx_id = context->id; -#ifdef CONFIG_PPC_MM_SLICES -VM_BUG_ON (!mm->context.slb_addr_limit); -get_paca ()->mm_ctx_slb_addr_limit = mm->context.slb_addr_limit; -memcpy (&get_paca ()->mm_ctx_low_slices_psize, - &context->low_slices_psize,sizeof (context->low_slices_psize)); -memcpy (&get_paca ()->mm_ctx_high_slices_psize, - &context->high_slices_psize,TASK_SLICE_ARRAY_SZ (mm)); -#else /* CONFIG_PPC_MM_SLICES */ -get_paca ()->mm_ctx_user_psize = context->user_psize; -get_paca ()->mm_ctx_sllp = context->sllp; -#endif -#else /* !CONFIG_PPC_BOOK3S */ -return ; -#endif -}
arch/powerpc/mm/hash_utils_64.c +11 -35
111 lines not on this page · raw message
arch/powerpc/mm/mmu_context.c +1 -2
@@ -54,8 +54,7 @@ void switch_mm_irqs_off(struct mm_struct *prev, struct mm_struct *next,* MMU context id, which is then moved to SPRN_PID. * * For the hash MMU it is either the first load from slb_cache -* in switch_slb(), and/or the store of paca->mm_ctx_id in -* copy_mm_to_paca(). +* in switch_slb(), and/or load of MMU context id. * * On the other side, the barrier is in mm/tlb-radix.c for * radix which orders earlier stores to clear the PTEs vs
arch/powerpc/mm/slb.c +18 -2
@@ -278,8 +278,6 @@ void switch_slb(struct task_struct *tsk, struct mm_struct *mm)get_paca ()->slb_cache_ptr =0 ; } -copy_mm_to_paca (mm); -/* * preload some userspace segments into the SLB. * Almost all 32 and 64bit PowerPC executables are linked at
@@ -306,6 +304,24 @@ void slb_set_size(u16 size) mmu_slb_size = size; } +static void cpu_flush_slb (void *parm) +{ +struct mm_struct *mm = parm; +unsigned long flags; + +if (mm != current->active_mm) +return ; + +local_irq_save (flags); +slb_flush_and_rebolt (); +local_irq_restore (flags); +} + +void core_flush_all_slbs (struct mm_struct *mm) +{ +on_each_cpu (cpu_flush_slb, mm,1 ); +} +void slb_initialize (void ) {unsigned long linear_llp, vmalloc_llp, io_llp;
arch/powerpc/mm/slice.c +9 -20
@@ -207,23 +207,6 @@ static bool slice_check_range_fits(struct mm_struct *mm,return true; } -static void slice_flush_segments (void *parm) -{ -#ifdef CONFIG_PPC64 -struct mm_struct *mm = parm; -unsigned long flags; - -if (mm != current->active_mm) -return ; - -copy_mm_to_paca (current->active_mm); - -local_irq_save (flags); -slb_flush_and_rebolt (); -local_irq_restore (flags); -#endif -} -static void slice_convert (struct mm_struct *mm,const struct slice_mask *mask,int psize) {
@@ -289,6 +272,9 @@ static void slice_convert(struct mm_struct *mm,spin_unlock_irqrestore (&slice_convert_lock, flags);copro_flush_all_slbs (mm); +#ifdef CONFIG_PPC64 +core_flush_all_slbs (mm); +#endif }/*
@@ -502,8 +488,9 @@ unsigned long slice_get_unmapped_area(unsigned long addr, unsigned long len,* be already initialised beyond the old address limit. */ mm->context.slb_addr_limit = high_limit; - -on_each_cpu (slice_flush_segments, mm,1 ); +#ifdef CONFIG_PPC64 +core_flush_all_slbs (mm); +#endif }/* Sanity checks */
@@ -665,8 +652,10 @@ unsigned long slice_get_unmapped_area(unsigned long addr, unsigned long len, (SLICE_NUM_HIGH && !bitmap_empty(potential_mask.high_slices, SLICE_NUM_HIGH))) {slice_convert (mm, &potential_mask, psize); +#ifdef CONFIG_PPC64 if (psize >MMU_PAGE_BASE ) -on_each_cpu (slice_flush_segments, mm,1 ); +core_flush_all_slbs (mm); +#endif }return newaddr;
Signature · 1 line
2.18.0
Nicholas Piggin
reviewerPATCH 9/12 powerpc/64s/hash: SLB allocation status bitmaps
In reply to Nicholas PigginAdd 32-entry bitmaps to track the allocation status of the first 32 SLB entries, and whether they are user or kernel entries. These are used to allocate free SLB entries first, before resorting to the round robin allocator.
--- arch/powerpc/include/asm/paca.h | 6 ++- arch/powerpc/kernel/asm-offsets.c | 2 +- arch/powerpc/mm/slb.c | 62 +++++++++++++++++++++++++------ arch/powerpc/xmon/xmon.c | 4 +- 4 files changed, 58 insertions(+), 16 deletions(-)
arch/powerpc/include/asm/paca.h +4 -2
@@ -113,7 +113,10 @@ struct paca_struct {* on the linear mapping */ /* SLB related definitions */ u16 vmalloc_sllp; -u16 slb_cache_ptr; +u8 slb_cache_ptr; +u8 stab_rr;/* stab/slb round-robin counter */ +u32 slb_used_bitmap;/* Bitmaps for first 32 SLB entries. */ +u32 slb_kern_bitmap;u32 slb_cache[SLB_CACHE_ENTRIES ];#endif /* CONFIG_PPC_BOOK3S_64 */
@@ -148,7 +151,6 @@ struct paca_struct {*/ struct task_struct *__current;/* Pointer to current */ u64 kstack;/* Saved Kernel stack addr */ -u64 stab_rr;/* stab/slb round-robin counter */ u64 saved_r1;/* r1 save for RTAS calls or PM or EE=0 */ u64 saved_msr;/* MSR saved here by enter_rtas */ u16 trap_save;/* Used when bad stack is encountered */
arch/powerpc/kernel/asm-offsets.c +1 -1
@@ -173,7 +173,6 @@ int main(void)OFFSET (PACAKSAVE , paca_struct, kstack);OFFSET (PACACURRENT , paca_struct, __current);OFFSET (PACASAVEDMSR , paca_struct, saved_msr); -OFFSET (PACASTABRR , paca_struct, stab_rr);OFFSET (PACAR1 , paca_struct, saved_r1);OFFSET (PACATOC , paca_struct, kernel_toc);OFFSET (PACAKBASE , paca_struct, kernelbase);
@@ -203,6 +202,7 @@ int main(void)#ifdef CONFIG_PPC_BOOK3S_64 OFFSET (PACASLBCACHE , paca_struct, slb_cache);OFFSET (PACASLBCACHEPTR , paca_struct, slb_cache_ptr); +OFFSET (PACASTABRR , paca_struct, stab_rr);OFFSET (PACAVMALLOCSLLP , paca_struct, vmalloc_sllp);#ifdef CONFIG_PPC_MM_SLICES OFFSET (MMUPSIZESLLP , mmu_psize_def, sllp);
arch/powerpc/mm/slb.c +50 -12
111 lines not on this page · raw message
arch/powerpc/xmon/xmon.c +3 -1
@@ -2393,6 +2393,9 @@ static void dump_one_paca(int cpu) } }DUMP (p, vmalloc_sllp,"%#-*x" ); +DUMP (p, stab_rr,"%#-*x" ); +DUMP (p, slb_used_bitmap,"%#-*x" ); +DUMP (p, slb_kern_bitmap,"%#-*x" );if (!early_cpu_has_feature (CPU_FTR_ARCH_300 )) {DUMP (p, slb_cache_ptr,"%#-*x" );
@@ -2415,7 +2418,6 @@ static void dump_one_paca(int cpu)DUMP (p, __current,"%-*px" );DUMP (p, kstack,"%#-*llx" );printf (" %-*s = 0x%016llx\n" ,25 ,"kstack_base" , p->kstack & ~(THREAD_SIZE -1 )); -DUMP (p, stab_rr,"%#-*llx" );DUMP (p, saved_r1,"%#-*llx" );DUMP (p, trap_save,"%#-*x" );DUMP (p, irq_soft_mask,"%#-*x" );
Signature · 1 line
2.18.0
Nicholas Piggin
reviewerPATCH 10/12 powerpc/64s: xmon do not dump hash fields when using radix mode
In reply to Nicholas Piggin--- arch/powerpc/xmon/xmon.c | 40 +++++++++++++++++++++------------------- 1 file changed, 21 insertions(+), 19 deletions(-)
arch/powerpc/xmon/xmon.c +21 -19
@@ -2378,30 +2378,32 @@ static void dump_one_paca(int cpu)DUMP (p, cpu_start,"%#-*x" );DUMP (p, kexec_state,"%#-*x" );#ifdef CONFIG_PPC_BOOK3S_64 -for (i =0 ; i <SLB_NUM_BOLTED ; i++) { -u64 esid, vsid; +if (!early_radix_enabled ()) { +for (i =0 ; i <SLB_NUM_BOLTED ; i++) { +u64 esid, vsid; -if (!p->slb_shadow_ptr) -continue ; +if (!p->slb_shadow_ptr) +continue ; - esid =be64_to_cpu (p->slb_shadow_ptr->save_area[i].esid); - vsid =be64_to_cpu (p->slb_shadow_ptr->save_area[i].vsid); + esid =be64_to_cpu (p->slb_shadow_ptr->save_area[i].esid); + vsid =be64_to_cpu (p->slb_shadow_ptr->save_area[i].vsid); -if (esid || vsid) { -printf (" %-*s[%d] = 0x%016llx 0x%016llx\n" , -22 ,"slb_shadow" , i, esid, vsid); +if (esid || vsid) { +printf (" %-*s[%d] = 0x%016llx 0x%016llx\n" , +22 ,"slb_shadow" , i, esid, vsid); + } } - } -DUMP (p, vmalloc_sllp,"%#-*x" ); -DUMP (p, stab_rr,"%#-*x" ); -DUMP (p, slb_used_bitmap,"%#-*x" ); -DUMP (p, slb_kern_bitmap,"%#-*x" ); +DUMP (p, vmalloc_sllp,"%#-*x" ); +DUMP (p, stab_rr,"%#-*x" ); +DUMP (p, slb_used_bitmap,"%#-*x" ); +DUMP (p, slb_kern_bitmap,"%#-*x" ); -if (!early_cpu_has_feature (CPU_FTR_ARCH_300 )) { -DUMP (p, slb_cache_ptr,"%#-*x" ); -for (i =0 ; i <SLB_CACHE_ENTRIES ; i++) -printf (" %-*s[%d] = 0x%016x\n" , -22 ,"slb_cache" , i, p->slb_cache[i]); +if (!early_cpu_has_feature (CPU_FTR_ARCH_300 )) { +DUMP (p, slb_cache_ptr,"%#-*x" ); +for (i =0 ; i <SLB_CACHE_ENTRIES ; i++) +printf (" %-*s[%d] = 0x%016x\n" , +22 ,"slb_cache" , i, p->slb_cache[i]); + } }DUMP (p, rfi_flush_fallback_area,"%-*px" );
Signature · 1 line
2.18.0
Nicholas Piggin
reviewerPATCH 11/12 powerpc/64s/hash: provide arch_setup_exec hooks for hash slice setup
In reply to Nicholas PigginThis will be used by the SLB code in the next patch, but for now this sets the slb_addr_limit to the correct size for 32-bit tasks.
--- arch/powerpc/include/asm/book3s/64/mmu-hash.h | 2 ++ arch/powerpc/include/asm/slice.h | 1 + arch/powerpc/include/asm/thread_info.h | 6 ++++++ arch/powerpc/kernel/process.c | 9 +++++++++ arch/powerpc/mm/mmu_context_book3s64.c | 5 +++++ arch/powerpc/mm/slice.c | 14 ++++++++++++++ 6 files changed, 37 insertions(+)
arch/powerpc/include/asm/book3s/64/mmu-hash.h +2 -0
@@ -487,6 +487,8 @@ int htab_remove_mapping(unsigned long vstart, unsigned long vend,extern void pseries_add_gpage (u64 addr,u64 page_size,unsigned long number_of_pages);extern void demote_segment_4k (struct mm_struct *mm,unsigned long addr); +extern void hash__setup_new_exec (void ); +#ifdef CONFIG_PPC_PSERIES void hpte_init_pseries (void );#else
arch/powerpc/include/asm/slice.h +1 -0
@@ -32,6 +32,7 @@ void slice_set_range_psize(struct mm_struct *mm, unsigned long start, unsigned long len, unsigned int psize);void slice_init_new_context_exec (struct mm_struct *mm); +void slice_setup_new_exec (void );#endif /* __ASSEMBLY__ */
arch/powerpc/include/asm/thread_info.h +6 -0
@@ -72,6 +72,12 @@ static inline struct thread_info *current_thread_info(void) }extern int arch_dup_task_struct (struct task_struct *dst,struct task_struct *src); + +#ifdef CONFIG_PPC_BOOK3S_64 +void arch_setup_new_exec (void ); +#define arch_setup_new_exec arch_setup_new_exec +#endif +#endif /* __ASSEMBLY__ */ /*
arch/powerpc/kernel/process.c +9 -0
@@ -1482,6 +1482,15 @@ void flush_thread(void)#endif /* CONFIG_HAVE_HW_BREAKPOINT */ } +#ifdef CONFIG_PPC_BOOK3S_64 +void arch_setup_new_exec (void ) +{ +if (radix_enabled ()) +return ; +hash__setup_new_exec (); +} +#endif +int set_thread_uses_vas (void ) {#ifdef CONFIG_PPC_BOOK3S_64
arch/powerpc/mm/mmu_context_book3s64.c +5 -0
@@ -84,6 +84,11 @@ static int hash__init_new_context(struct mm_struct *mm)return index; } +void hash__setup_new_exec (void ) +{ +slice_setup_new_exec (); +} +static int radix__init_new_context (struct mm_struct *mm) {unsigned long rts_field;
arch/powerpc/mm/slice.c +14 -0
@@ -746,6 +746,20 @@ void slice_init_new_context_exec(struct mm_struct *mm)bitmap_fill (mask->high_slices,SLICE_NUM_HIGH ); } +#ifdef CONFIG_PPC_BOOK3S_64 +void slice_setup_new_exec (void ) +{ +struct mm_struct *mm = current->mm; + +slice_dbg ("slice_setup_new_exec(mm=%p)\n" , mm); + +if (!is_32bit_task ()) +return ; + + mm->context.slb_addr_limit =DEFAULT_MAP_WINDOW ; +} +#endif +void slice_set_range_psize (struct mm_struct *mm,unsigned long start,unsigned long len,unsigned int psize) {
Signature · 1 line
2.18.0
Nicholas Piggin
reviewerPATCH 12/12 powerpc/64s/hash: Add a SLB preload cache
In reply to Nicholas PigginWhen switching processes, currently all user SLBEs are cleared, and a few (exec_base, pc, and stack) are preloaded. In trivial testing with small apps, this tends to miss the heap and low 256MB segments, and it will also miss commonly accessed segments on large memory workloads.
Add a simple round-robin preload cache that just inserts the last SLB miss into the head of the cache and preloads those at context switch time. Every 256 context switches, the oldest entry is removed from the cache to shrink the cache and require fewer slbmte if they are unused.
Much more could go into this, including into the SLB entry reclaim side to track some LRU information etc, which would require a study of large memory workloads. But this is a simple thing we can do now that is an obvious win for common workloads.
With the full series, process switching speed on the context_switch benchmark on POWER9/hash (with kernel speculation security masures disabled) increases from 140K/s to 178K/s (27%).
POWER8 does not change much (within 1%), it's unclear why it does not see a big gain like POWER9.
Booting to busybox init with 256MB segments has SLB misses go down from 945 to 69, and with 1T segments 900 to 21. These could almost all be eliminated by preloading a bit more carefully with ELF binary loading.
--- arch/powerpc/include/asm/processor.h | 1 + arch/powerpc/include/asm/thread_info.h | 5 + arch/powerpc/kernel/process.c | 7 ++ arch/powerpc/mm/mmu_context_book3s64.c | 4 + arch/powerpc/mm/slb.c | 166 +++++++++++++++++++------ 5 files changed, 143 insertions(+), 40 deletions(-)
arch/powerpc/include/asm/processor.h +1 -0
@@ -273,6 +273,7 @@ struct thread_struct {#endif /* CONFIG_HAVE_HW_BREAKPOINT */ struct arch_hw_breakpoint hw_brk;/* info on the hardware breakpoint */ unsigned long trap_nr;/* last trap # on this thread */ +u8 load_slb;/* Ages out SLB preload cache entries */ u8 load_fp;#ifdef CONFIG_ALTIVEC u8 load_vec;
arch/powerpc/include/asm/thread_info.h +5 -0
@@ -29,6 +29,7 @@#include <asm/page.h> #include <asm/accounting.h> +#define SLB_PRELOAD_NR 16U/* * low level task data. */
@@ -44,6 +45,10 @@ struct thread_info {#if defined(CONFIG_VIRT_CPU_ACCOUNTING_NATIVE ) && defined(CONFIG_PPC32 )struct cpu_accounting_data accounting;#endif +unsigned char slb_preload_nr; +unsigned char slb_preload_tail; +u32 slb_preload_esid[SLB_PRELOAD_NR ]; +/* low level flags - has atomic operations done on it */ unsigned long flags____cacheline_aligned_in_smp ; };
arch/powerpc/kernel/process.c +7 -0
@@ -1719,6 +1719,8 @@ int copy_thread(unsigned long clone_flags, unsigned long usp,return 0 ; } +void preload_new_slb_context (unsigned long start,unsigned long sp); +/* * Set up a thread for executing a new program */
@@ -1726,6 +1728,10 @@ void start_thread(struct pt_regs *regs, unsigned long start, unsigned long sp) {#ifdef CONFIG_PPC64 unsigned long load_addr = regs->gpr[2 ];/* saved by ELF_PLAT_INIT */ + +#ifdef CONFIG_PPC_BOOK3S_64 +preload_new_slb_context (start, sp); +#endif #endif /*
@@ -1816,6 +1822,7 @@ void start_thread(struct pt_regs *regs, unsigned long start, unsigned long sp)#ifdef CONFIG_VSX current->thread.used_vsr =0 ;#endif + current->thread.load_slb =0 ; current->thread.load_fp =0 ;memset (¤t->thread.fp_state,0 ,sizeof (current->thread.fp_state)); current->thread.fp_save_area =NULL ;
arch/powerpc/mm/mmu_context_book3s64.c +4 -0
@@ -53,6 +53,8 @@ int hash__alloc_context_id(void) }EXPORT_SYMBOL_GPL (hash__alloc_context_id); +void slb_setup_new_exec (void ); +static int hash__init_new_context (struct mm_struct *mm) {int index;
@@ -87,6 +89,8 @@ static int hash__init_new_context(struct mm_struct *mm)void hash__setup_new_exec (void ) {slice_setup_new_exec (); + +slb_setup_new_exec (); }static int radix__init_new_context (struct mm_struct *mm)
arch/powerpc/mm/slb.c +126 -40
210 lines not on this page · raw message
Signature · 1 line
2.18.0
kbuild test robot
build botRe: [PATCH 03/12] powerpc/64s/hash: move POWER5 < DD2.1 slbie workaround where it is needed
In reply to Nicholas PigginI love your patch! Yet something to improve:
Hi Nicholas,
I love your patch! Yet something to improve:
[auto build test ERROR on powerpc/next]
[also build test ERROR on v4.19-rc3 next-20180913]
[if your patch is applied to the wrong git tree, please drop us a note to help improve the system]
url: https://github.com/0day-ci/linux/commits/Nicholas-Piggin/SLB-miss-conversion-to-C-and-SLB-optimisations/20180917-015458 base: https://git.kernel.org/pub/scm/linux/kernel/git/powerpc/linux.git next config: powerpc-defconfig (attached as .config) compiler: powerpc64-linux-gnu-gcc (Debian 7.2.0-11) 7.2.0
reproduce:
wget https://raw.githubusercontent.com/intel/lkp-tests/master/sbin/make.cross -O ~/bin/make.cross chmod +x ~/bin/make.cross
# save the attached .config to linux build tree
GCC_VERSION=7.2.0 make.cross ARCH=powerpc
Note: it may well be a FALSE warning. FWIW you are at least aware of it now.
http://gcc.gnu.org/wiki/Better_Uninitialized_Warnings
Note: the linux-review/Nicholas-Piggin/SLB-miss-conversion-to-C-and-SLB-optimisations/20180917-015458 HEAD b26bd44c74488169a0fd19eef43ea3db189a207d builds fine.
It only hurts bisectibility.
All errors (new ones prefixed by >>):
arch/powerpc/mm/slb.c: In function 'switch_slb':
>> arch/powerpc/mm/slb.c:258:4: error: 'slbie_data' may be used uninitialized in this function [-Werror=maybe-uninitialized]
asm volatile("slbie %0" : : "r" (slbie_data));
^~~cc1: all warnings being treated as errors
vim +/slbie_data +258 arch/powerpc/mm/slb.c
465ccab9 arch/powerpc/mm/slb.c will schmidt 2007-10-31 224
^1da177e arch/ppc64/mm/slb.c Linus Torvalds 2005-04-16 225 /* Flush all user entries from the segment table of the current processor. */
^1da177e arch/ppc64/mm/slb.c Linus Torvalds 2005-04-16 226 void switch_slb(struct task_struct *tsk, struct mm_struct *mm)
^1da177e arch/ppc64/mm/slb.c Linus Torvalds 2005-04-16 227 {
9c1e1052 arch/powerpc/mm/slb.c Paul Mackerras 2009-08-17 228 unsigned long offset;
^1da177e arch/ppc64/mm/slb.c Linus Torvalds 2005-04-16 229 unsigned long pc = KSTK_EIP(tsk);
^1da177e arch/ppc64/mm/slb.c Linus Torvalds 2005-04-16 230 unsigned long stack = KSTK_ESP(tsk);
de4376c2 arch/powerpc/mm/slb.c Anton Blanchard 2009-07-13 231 unsigned long exec_base;
^1da177e arch/ppc64/mm/slb.c Linus Torvalds 2005-04-16 232
9c1e1052 arch/powerpc/mm/slb.c Paul Mackerras 2009-08-17 233 /*
9c1e1052 arch/powerpc/mm/slb.c Paul Mackerras 2009-08-17 234 * We need interrupts hard-disabled here, not just soft-disabled,
9c1e1052 arch/powerpc/mm/slb.c Paul Mackerras 2009-08-17 235 * so that a PMU interrupt can't occur, which might try to access
9c1e1052 arch/powerpc/mm/slb.c Paul Mackerras 2009-08-17 236 * user memory (to get a stack trace) and possible cause an SLB miss
9c1e1052 arch/powerpc/mm/slb.c Paul Mackerras 2009-08-17 237 * which would update the slb_cache/slb_cache_ptr fields in the PACA.
9c1e1052 arch/powerpc/mm/slb.c Paul Mackerras 2009-08-17 238 */
9c1e1052 arch/powerpc/mm/slb.c Paul Mackerras 2009-08-17 239 hard_irq_disable();
9c1e1052 arch/powerpc/mm/slb.c Paul Mackerras 2009-08-17 240 offset = get_paca()->slb_cache_ptr;
44ae3ab3 arch/powerpc/mm/slb.c Matt Evans 2011-04-06 241 if (!mmu_has_feature(MMU_FTR_NO_SLBIE_B) &&
f66bce5e arch/powerpc/mm/slb.c Olof Johansson 2007-10-16 242 offset <= SLB_CACHE_ENTRIES) {
6697d605 arch/powerpc/mm/slb.c Nicholas Piggin 2018-09-15 243 unsigned long slbie_data;
^1da177e arch/ppc64/mm/slb.c Linus Torvalds 2005-04-16 244 int i;
6697d605 arch/powerpc/mm/slb.c Nicholas Piggin 2018-09-15 245
^1da177e arch/ppc64/mm/slb.c Linus Torvalds 2005-04-16 246 asm volatile("isync" : : : "memory");
^1da177e arch/ppc64/mm/slb.c Linus Torvalds 2005-04-16 247 for (i = 0; i < offset; i++) {
1189be65 arch/powerpc/mm/slb.c Paul Mackerras 2007-10-11 248 slbie_data = (unsigned long)get_paca()->slb_cache[i]
1189be65 arch/powerpc/mm/slb.c Paul Mackerras 2007-10-11 249 << SID_SHIFT; /* EA */
1189be65 arch/powerpc/mm/slb.c Paul Mackerras 2007-10-11 250 slbie_data |= user_segment_size(slbie_data)
1189be65 arch/powerpc/mm/slb.c Paul Mackerras 2007-10-11 251 << SLBIE_SSIZE_SHIFT;
1189be65 arch/powerpc/mm/slb.c Paul Mackerras 2007-10-11 252 slbie_data |= SLBIE_C; /* C set for user addresses */
1189be65 arch/powerpc/mm/slb.c Paul Mackerras 2007-10-11 253 asm volatile("slbie %0" : : "r" (slbie_data));
^1da177e arch/ppc64/mm/slb.c Linus Torvalds 2005-04-16 254 }
^1da177e arch/ppc64/mm/slb.c Linus Torvalds 2005-04-16 255
^1da177e arch/ppc64/mm/slb.c Linus Torvalds 2005-04-16 256 /* Workaround POWER5 < DD2.1 issue */
6697d605 arch/powerpc/mm/slb.c Nicholas Piggin 2018-09-15 257 if (!cpu_has_feature(CPU_FTR_ARCH_207S) && offset == 1)
1189be65 arch/powerpc/mm/slb.c Paul Mackerras 2007-10-11 @258 asm volatile("slbie %0" : : "r" (slbie_data));
6697d605 arch/powerpc/mm/slb.c Nicholas Piggin 2018-09-15 259
6697d605 arch/powerpc/mm/slb.c Nicholas Piggin 2018-09-15 260 asm volatile("isync" : : : "memory");
6697d605 arch/powerpc/mm/slb.c Nicholas Piggin 2018-09-15 261 } else {
6697d605 arch/powerpc/mm/slb.c Nicholas Piggin 2018-09-15 262 __slb_flush_and_rebolt();
c15c9670 arch/powerpc/mm/slb.c Nicholas Piggin 2018-09-15 263 }
^1da177e arch/ppc64/mm/slb.c Linus Torvalds 2005-04-16 264
^1da177e arch/ppc64/mm/slb.c Linus Torvalds 2005-04-16 265 get_paca()->slb_cache_ptr = 0;
52b1e665 arch/powerpc/mm/slb.c Aneesh Kumar K.V 2017-03-22 266 copy_mm_to_paca(mm);
^1da177e arch/ppc64/mm/slb.c Linus Torvalds 2005-04-16 267
^1da177e arch/ppc64/mm/slb.c Linus Torvalds 2005-04-16 268 /*
^1da177e arch/ppc64/mm/slb.c Linus Torvalds 2005-04-16 269 * preload some userspace segments into the SLB.
de4376c2 arch/powerpc/mm/slb.c Anton Blanchard 2009-07-13 270 * Almost all 32 and 64bit PowerPC executables are linked at
de4376c2 arch/powerpc/mm/slb.c Anton Blanchard 2009-07-13 271 * 0x10000000 so it makes sense to preload this segment.
^1da177e arch/ppc64/mm/slb.c Linus Torvalds 2005-04-16 272 */
de4376c2 arch/powerpc/mm/slb.c Anton Blanchard 2009-07-13 273 exec_base = 0x10000000;
^1da177e arch/ppc64/mm/slb.c Linus Torvalds 2005-04-16 274
5eb9bac0 arch/powerpc/mm/slb.c Anton Blanchard 2009-07-13 275 if (is_kernel_addr(pc) || is_kernel_addr(stack) ||
de4376c2 arch/powerpc/mm/slb.c Anton Blanchard 2009-07-13 276 is_kernel_addr(exec_base))
^1da177e arch/ppc64/mm/slb.c Linus Torvalds 2005-04-16 277 return;
^1da177e arch/ppc64/mm/slb.c Linus Torvalds 2005-04-16 278
5eb9bac0 arch/powerpc/mm/slb.c Anton Blanchard 2009-07-13 279 slb_allocate(pc);
^1da177e arch/ppc64/mm/slb.c Linus Torvalds 2005-04-16 280
5eb9bac0 arch/powerpc/mm/slb.c Anton Blanchard 2009-07-13 281 if (!esids_match(pc, stack))
^1da177e arch/ppc64/mm/slb.c Linus Torvalds 2005-04-16 282 slb_allocate(stack);
^1da177e arch/ppc64/mm/slb.c Linus Torvalds 2005-04-16 283
de4376c2 arch/powerpc/mm/slb.c Anton Blanchard 2009-07-13 284 if (!esids_match(pc, exec_base) &&
de4376c2 arch/powerpc/mm/slb.c Anton Blanchard 2009-07-13 285 !esids_match(stack, exec_base))
de4376c2 arch/powerpc/mm/slb.c Anton Blanchard 2009-07-13 286 slb_allocate(exec_base);
^1da177e arch/ppc64/mm/slb.c Linus Torvalds 2005-04-16 287 }
^1da177e arch/ppc64/mm/slb.c Linus Torvalds 2005-04-16 288 :::::: The code at line 258 was first introduced by commit :::::: 1189be6508d45183013ddb82b18f4934193de274 [POWERPC] Use 1TB segments
:::::: TO: Paul Mackerras <[email protected]> :::::: CC: Paul Mackerras <[email protected]>
---
0-DAY kernel test infrastructure Open Source Technology Center
https://lists.01.org/pipermail/kbuild-all Intel Corporation
Re: [PATCH 03/12] powerpc/64s/hash: move POWER5 < DD2.1 slbie workaround where it is needed
In reply to Nicholas PigginQuoted 9 lines · Nicholas Piggin
The POWER5 < DD2.1 issue is that slbie needs to be issued more than once. It came in with this change:
[email protected], 2004-04-29 07:12:31-07:00, [email protected]
[PATCH] POWER5 erratum workaroundEarly POWER5 revisions (<DD2.1) have a problem requiring slbie instructions to be repeated under some circumstances. The patch below adds a workaround (patch made by Anton Blanchard).
Thanks for extracting this. Can we add this to the code? Also I am not sure what is repeated here? Is it that we just need one slb extra(hence only applicable to offset == 1) or is it that we need to make sure there is always one slb extra? The code does the former. Do you a have link for that email patch?
Quoted 63 lines
The extra slbie in
switch_slbis done even for the case where slbia is called (slb_flush_and_rebolt). I don't believe that is required because there are otherslb_flush_and_reboltcallers which do not issue the workaround slbie, which would be broken if it was required.It also seems to be fine inside the isync with the first slbie, as it is in the kernel stack switch code.
So move this workaround to where it is required. This is not much of an optimisation because this is the fast path, but it makes the code more understandable and neater.
Signed-off-by Nicholas Piggin <[email protected]>--- arch/powerpc/mm/slb.c | 14 +++++++------- 1 file changed, 7 insertions(+), 7 deletions(-)arch/powerpc/mm/slb.c +7
-7@@ -226,7 +226,6 @@ static inline int esids_match(unsigned long addr1, unsigned long addr2)void switch_slb (struct task_struct *tsk,struct mm_struct *mm) {unsigned long offset; -unsigned long slbie_data =0 ;unsigned long pc =KSTK_EIP (tsk);unsigned long stack =KSTK_ESP (tsk);unsigned long exec_base;@@ -241,7 +240,9 @@ void switch_slb(struct task_struct *tsk, struct mm_struct *mm) offset =get_paca ()->slb_cache_ptr;if (!mmu_has_feature (MMU_FTR_NO_SLBIE_B ) && offset <=SLB_CACHE_ENTRIES ) { +unsigned long slbie_data;int i; + asmvolatile ("isync" : : :"memory" );for (i =0 ; i < offset; i++) { slbie_data = (unsigned long )get_paca ()->slb_cache[i]@@ -251,15 +252,14 @@ void switch_slb(struct task_struct *tsk, struct mm_struct *mm) slbie_data |=SLBIE_C ;/* C set for user addresses */ asmvolatile ("slbie %0" : :"r" (slbie_data)); } - asmvolatile ("isync" : : :"memory" ); - }else { -__slb_flush_and_rebolt (); - } -if (!cpu_has_feature (CPU_FTR_ARCH_207S )) {/* Workaround POWER5 < DD2.1 issue */ -if (offset ==1 || offset >SLB_CACHE_ENTRIES ) +if (!cpu_has_feature (CPU_FTR_ARCH_207S ) && offset ==1 ) asmvolatile ("slbie %0" : :"r" (slbie_data)); + + asmvolatile ("isync" : : :"memory" ); + }else { +__slb_flush_and_rebolt (); }get_paca ()->slb_cache_ptr =0 ;Signature · 1 line
2.18.0
Re: [PATCH 05/12] powerpc/64s/hash: Use POWER6 SLBIA IH=1 variant in switch_slb
In reply to Nicholas PigginQuoted 7 lines · Nicholas Piggin
The SLBIA IH=1 hint will remove all non-zero SLBEs, but only invalidate ERAT entries associated with a class value of 1, for processors that support the hint (e.g., POWER6 and newer), which Linux assigns to user addresses.
This prevents kernel ERAT entries from being invalidated when context switchig (if the thread faulted in more than 8 user SLBEs).
how about renaming stuff to indicate kernel ERAT entries are kept? something like slb_flush_and_rebolt_user()?
Quoted 78 lines
Signed-off-by Nicholas Piggin <[email protected]>--- arch/powerpc/mm/slb.c | 38 +++++++++++++++++++++++--------------- 1 file changed, 23 insertions(+), 15 deletions(-)arch/powerpc/mm/slb.c +23
-15@@ -128,13 +128,21 @@ void slb_flush_all_realmode(void) asmvolatile ("slbmte %0,%0; slbia" : :"r" (0 )); } -static void __slb_flush_and_rebolt (void ) +void slb_flush_and_rebolt (void ) {/* If you change this make sure you change SLB_NUM_BOLTED * and PR KVM appropriately too. */ unsigned long linear_llp, lflags;unsigned long ksp_esid_data, ksp_vsid_data; +WARN_ON (!irqs_disabled ()); + +/* +* We can't take a PMU exception in the following code, so hard +* disable interrupts. +*/ +hard_irq_disable (); + linear_llp = mmu_psize_defs[mmu_linear_psize].sllp; lflags =SLB_VSID_KERNEL | linear_llp;@@ -160,20 +168,7 @@ static void __slb_flush_and_rebolt(void) :: "r"(ksp_vsid_data), "r"(ksp_esid_data) : "memory"); -} -void slb_flush_and_rebolt (void ) -{ - -WARN_ON (!irqs_disabled ()); - -/* -* We can't take a PMU exception in the following code, so hard -* disable interrupts. -*/ -hard_irq_disable (); - -__slb_flush_and_rebolt ();get_paca ()->slb_cache_ptr =0 ; }@@ -248,7 +243,20 @@ void switch_slb(struct task_struct *tsk, struct mm_struct *mm) asmvolatile ("isync" : : :"memory" ); }else { -__slb_flush_and_rebolt (); +struct slb_shadow *p =get_slb_shadow (); +unsigned long ksp_esid_data = +be64_to_cpu (p->save_area[KSTACK_INDEX ].esid); +unsigned long ksp_vsid_data = +be64_to_cpu (p->save_area[KSTACK_INDEX ].vsid); + + asmvolatile ("isync\n" + PPC_SLBIA(1) "\n" + "slbmte %0,%1\n" + "isync" + :: "r"(ksp_vsid_data), + "r"(ksp_esid_data)); + + asmvolatile ("isync" : : :"memory" ); }get_paca ()->slb_cache_ptr =0 ;Signature · 1 line
2.18.0
Re: [PATCH 01/12] powerpc/64s/hash: Fix stab_rr off by one initialization
In reply to Nicholas PigginQuoted 3 lines · Nicholas Piggin
This causes SLB alloation to start 1 beyond the start of the SLB. There is no real problem because after it wraps it stats behaving
starts?
Quoted 1 line
properly, it's just surprisig to see when looking at SLB traces.
surprising
Quoted 23 lines
Signed-off-by Nicholas Piggin <[email protected]>--- arch/powerpc/mm/slb.c | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-)arch/powerpc/mm/slb.c +1
-2@@ -355,7 +355,7 @@ void slb_initialize(void)#endif } -get_paca ()->stab_rr =SLB_NUM_BOLTED ; +get_paca ()->stab_rr =SLB_NUM_BOLTED -1 ; lflags =SLB_VSID_KERNEL | linear_llp; vflags =SLB_VSID_KERNEL | vmalloc_llp; --2.18.0
Nicholas Piggin
reviewerRe: [PATCH 01/12] powerpc/64s/hash: Fix stab_rr off by one initialization
Quoted 3 lines · On Mon, 17 Sep 2018 16:21:51 +0930 Joel Stanley <[email protected]> wrote:
Quoted 2 lines · Nicholas Piggin
This causes SLB alloation to start 1 beyond the start of the SLB.
allocation
Quoted 7 lines
Quoted 1 line
There is no real problem because after it wraps it stats behaving
starts?
Quoted 1 line
properly, it's just surprisig to see when looking at SLB traces.
surprising
My keyboard is dying :(
Nicholas Piggin
reviewerRe: [PATCH 03/12] powerpc/64s/hash: move POWER5 < DD2.1 slbie workaround where it is needed
In reply to Aneesh Kumar K.VQuoted 5 lines · On Mon, 17 Sep 2018 11:30:16 +0530 "Aneesh Kumar K.V"…
Quoted 3 lines · Nicholas Piggin
The POWER5 < DD2.1 issue is that slbie needs to be issued more than once. It came in with this change:
Quoted 8 lines
Quoted 6 lines
[email protected], 2004-04-29 07:12:31-07:00, [email protected]
[PATCH] POWER5 erratum workaroundEarly POWER5 revisions (<DD2.1) have a problem requiring slbie instructions to be repeated under some circumstances. The patch below adds a workaround (patch made by Anton Blanchard).
Thanks for extracting this. Can we add this to the code?
The comment? Sure.
Quoted 4 lines
Also I am not
sure what is repeated here? Is it that we just need one slb extra(hence only applicable to offset == 1) or is it that we need to make sure there is always one slb extra? The code does the former.
Yeah it has always done the former, so my assumption is that you just need more than one slbie. I don't think we need to bother revisiting that assumption unless someone can pull up something definitive.
What I did change is that slbia no longer has the additional slbie, but I think there are strong reasons not to need that.
Quoted 2 lines
Do you a have link for
that email patch?
I tried looking through the archives around that date but could not find it. That came from a bitkeeper log.
Thanks,
Nick
Nicholas Piggin
reviewerRe: [PATCH 05/12] powerpc/64s/hash: Use POWER6 SLBIA IH=1 variant in switch_slb
In reply to Aneesh Kumar K.VQuoted 7 lines · On Mon, 17 Sep 2018 11:38:35 +0530 "Aneesh Kumar K.V"…
Quoted 5 lines · Nicholas Piggin
The SLBIA IH=1 hint will remove all non-zero SLBEs, but only invalidate ERAT entries associated with a class value of 1, for processors that support the hint (e.g., POWER6 and newer), which Linux assigns to user addresses.
Quoted 6 lines
Quoted 2 lines
This prevents kernel ERAT entries from being invalidated when context switchig (if the thread faulted in more than 8 user SLBEs).
how about renaming stuff to indicate kernel ERAT entries are kept? something like
slb_flush_and_rebolt_user()?
User mappings aren't bolted though. I consider rebolt to mean update the bolted kernel mappings when something has changed (like vmalloc segment update). That doesn't need to be done here, so I think this is okay. I can add a comment though.
Thanks,
Nick
Quoted 80 lines
Quoted 79 lines
Signed-off-by Nicholas Piggin <[email protected]>--- arch/powerpc/mm/slb.c | 38 +++++++++++++++++++++++--------------- 1 file changed, 23 insertions(+), 15 deletions(-)arch/powerpc/mm/slb.c +23
-15@@ -128,13 +128,21 @@ void slb_flush_all_realmode(void) asmvolatile ("slbmte %0,%0; slbia" : :"r" (0 )); } -static void __slb_flush_and_rebolt (void ) +void slb_flush_and_rebolt (void ) {/* If you change this make sure you change SLB_NUM_BOLTED * and PR KVM appropriately too. */ unsigned long linear_llp, lflags;unsigned long ksp_esid_data, ksp_vsid_data; +WARN_ON (!irqs_disabled ()); + +/* +* We can't take a PMU exception in the following code, so hard +* disable interrupts. +*/ +hard_irq_disable (); + linear_llp = mmu_psize_defs[mmu_linear_psize].sllp; lflags =SLB_VSID_KERNEL | linear_llp;@@ -160,20 +168,7 @@ static void __slb_flush_and_rebolt(void) :: "r"(ksp_vsid_data), "r"(ksp_esid_data) : "memory"); -} -void slb_flush_and_rebolt (void ) -{ - -WARN_ON (!irqs_disabled ()); - -/* -* We can't take a PMU exception in the following code, so hard -* disable interrupts. -*/ -hard_irq_disable (); - -__slb_flush_and_rebolt ();get_paca ()->slb_cache_ptr =0 ; }@@ -248,7 +243,20 @@ void switch_slb(struct task_struct *tsk, struct mm_struct *mm) asmvolatile ("isync" : : :"memory" ); }else { -__slb_flush_and_rebolt (); +struct slb_shadow *p =get_slb_shadow (); +unsigned long ksp_esid_data = +be64_to_cpu (p->save_area[KSTACK_INDEX ].esid); +unsigned long ksp_vsid_data = +be64_to_cpu (p->save_area[KSTACK_INDEX ].vsid); + + asmvolatile ("isync\n" + PPC_SLBIA(1) "\n" + "slbmte %0,%1\n" + "isync" + :: "r"(ksp_vsid_data), + "r"(ksp_esid_data)); + + asmvolatile ("isync" : : :"memory" ); }get_paca ()->slb_cache_ptr =0 ;Signature · 1 line
2.18.0
Re: [01/12] powerpc/64s/hash: Fix stab_rr off by one initialization
In reply to Nicholas PigginQuoted 5 lines · Nicholas Piggin
This causes SLB alloation to start 1 beyond the start of the SLB. There is no real problem because after it wraps it stats behaving properly, it's just surprisig to see when looking at SLB traces.
Signed-off-by Nicholas Piggin <[email protected]>
Series applied to powerpc next, thanks.
https://git.kernel.org/powerpc/c/09b4438db13fa83b6219aee5993711
cheers