rewriting SLB miss handler in C

RFC PATCH In review Nicholas Piggin 2 participants 10 messages Aug 20, 2018 – Aug 22, 2018 PowerPC

lore

Share & tools

lore

Nicholas Piggin

author reviewer

2018-08-20 09:42 UTC #0 lore raw

cover rewriting SLB miss handler in C

I'd like to rewrite the SLB miss handlers in C for maintainability and ability to more easily extend the code.

I have not benchmarked it yet but obviously setting up the stack and going to C code rather than carefully hand optimised assembly is likely to slow down SLB misses by a reasonable amount. So I've started looked at a few basic optimisations we can make to justify this change. There is still more that can be done, but SLB misses have been reduced significantly, and with more tuning and optimization I think we could bring it down quite a bit more.

I'm trying to get the first patch solid, and it is the big change so would really appreciate review and commets on that. Other patches are not quite polished but comments would still be welcome on those (keep in mind they are obviously not all polished).

Thanks,
Nick

Nicholas Piggin (5):
powerpc/64s/hash: convert SLB miss handlers to C powerpc/64s/hash: remove user SLB data from the paca powerpc/64s/hash: remove the first vmalloc segment from the bolted SLB powerpc/64s/hash: Add SLB allocation bitmaps powerpc/64s/hash: Add a SLB preload cache

 arch/powerpc/include/asm/asm-prototypes.h     |   2 +
 arch/powerpc/include/asm/book3s/64/mmu-hash.h |   3 +-
 arch/powerpc/include/asm/paca.h               |  19 +-
 arch/powerpc/include/asm/thread_info.h        |   4 +
 arch/powerpc/kernel/asm-offsets.c             |   9 -
 arch/powerpc/kernel/exceptions-64s.S          | 200 ++-------
 arch/powerpc/kernel/paca.c                    |  21 -
 arch/powerpc/kernel/process.c                 |   6 +
 arch/powerpc/mm/Makefile                      |   2 +-
 arch/powerpc/mm/hash_utils_64.c               |  46 +--
 arch/powerpc/mm/mmu_context.c                 |   3 +-
 arch/powerpc/mm/mmu_context_book3s64.c        |  10 +-
 arch/powerpc/mm/slb.c                         | 382 +++++++++++-------
 arch/powerpc/mm/slb_low.S                     | 338 ----------------
 arch/powerpc/mm/slice.c                       |  29 +-
 arch/powerpc/xmon/xmon.c                      |   2 +-
 16 files changed, 328 insertions(+), 748 deletions(-)
 delete mode 100644 arch/powerpc/mm/slb_low.S

Signature · 1 line

2.17.0

Nicholas Piggin

author reviewer

2018-08-20 09:42 UTC #1 lore raw

RFC PATCH 1/5 powerpc/64s/hash: convert SLB miss handlers to C

This patch moves SLB miss handlers completely to C, using the standard exception handler macros to set up the stack and branch to C.

This can be done because the segment containing the kernel stack is always bolted, so accessing it with relocation on will not cause an SLB exception.

Arbitrary kernel memory may not be accessed when handling kernel space SLB misses, so care should be taken there. However user SLB misses can access any kernel memory, which can be used to move some fields out of the paca (in later patches).

User SLB misses could quite easily reconcile IRQs and set up a first class kernel environment and exit via ret_from_except, however that doesn't seem to be necessary at the moment, so we only do that if a bad fault is encountered.

[ Credit to Aneesh for bug fixes and improvements to bad address
handling ]

---
 arch/powerpc/include/asm/asm-prototypes.h |   2 +
 arch/powerpc/kernel/exceptions-64s.S      | 200 +++----------
 arch/powerpc/mm/Makefile                  |   2 +-
 arch/powerpc/mm/slb.c                     | 237 +++++++--------
 arch/powerpc/mm/slb_low.S                 | 338 ----------------------
 5 files changed, 166 insertions(+), 613 deletions(-)
 delete mode 100644 arch/powerpc/mm/slb_low.S

arch/powerpc/include/asm/asm-prototypes.h +2 -0

index 1f4691ce4126..c330ed10074a 100644
@@ -78,6 +78,8 @@ void kernel_bad_stack(struct pt_regs *regs);
 void system_reset_exception(struct pt_regs *regs);
 void machine_check_exception(struct pt_regs *regs);
 void emulation_assist_interrupt(struct pt_regs *regs);
+long do_slb_fault(struct pt_regs *regs, unsigned long ea);
+void do_bad_slb_fault(struct pt_regs *regs, unsigned long ea, unsigned long err);
 
 /* signals, syscalls and interrupts */
 long sys_swapcontext(struct ucontext __user *old_ctx,

arch/powerpc/kernel/exceptions-64s.S +40 ~~-160~~

228 lines not on this page · raw message

arch/powerpc/mm/Makefile +1 -1

index cdf6a9960046..892d4e061d62 100644
@@ -15,7 +15,7 @@ obj-$(CONFIG_PPC_MMU_NOHASH)	+= mmu_context_nohash.o tlb_nohash.o \
 obj-$(CONFIG_PPC_BOOK3E)	+= tlb_low_$(BITS)e.o
 hash64-$(CONFIG_PPC_NATIVE)	:= hash_native_64.o
 obj-$(CONFIG_PPC_BOOK3E_64)   += pgtable-book3e.o
-obj-$(CONFIG_PPC_BOOK3S_64)	+= pgtable-hash64.o hash_utils_64.o slb_low.o slb.o $(hash64-y) mmu_context_book3s64.o pgtable-book3s64.o
+obj-$(CONFIG_PPC_BOOK3S_64)	+= pgtable-hash64.o hash_utils_64.o slb.o $(hash64-y) mmu_context_book3s64.o pgtable-book3s64.o
 obj-$(CONFIG_PPC_RADIX_MMU)	+= pgtable-radix.o tlb-radix.o
 obj-$(CONFIG_PPC_STD_MMU_32)	+= ppc_mmu_32.o hash_low_32.o mmu_context_hash32.o
 obj-$(CONFIG_PPC_STD_MMU)	+= tlb_hash$(BITS).o

arch/powerpc/mm/slb.c +123 ~~-114~~

305 lines not on this page · raw message

arch/powerpc/mm/slb_low.S deleted +0 ~~-338~~

338 lines not on this page · raw message

Signature · 1 line

2.17.0

Nicholas Piggin

author reviewer

2018-08-20 10:08 UTC #2 lore raw

Re: [RFC PATCH 1/5] powerpc/64s/hash: convert SLB miss handlers to C

Quoted 36 lines · On Mon, 20 Aug 2018 19:41:56 +1000 Nicholas Piggin <[email protected]> wrote:

arch/powerpc/mm/slb.c +32 -3

+long do_slb_fault(struct pt_regs *regs, unsigned long ea)
+{
+	unsigned long id = REGION_ID(ea);
+
+	/* IRQs are not reconciled here, so can't check irqs_disabled */
+	VM_WARN_ON(mfmsr() & MSR_EE);
+
+	/*
+	 * SLB kernel faults must be very careful not to touch anything
+	 * that is not bolted. E.g., PACA and global variables are okay,
+	 * mm->context stuff is not.
+	 *
+	 * SLB user faults can access all of kernel memory, but must be
+	 * careful not to touch things like IRQ state because it is not
+	 * "reconciled" here. The difficulty is that we must use
+	 * fast_exception_return to return from kernel SLB faults without
+	 * looking at possible non-bolted memory. We could test user vs
+	 * kernel faults in the interrupt handler asm and do a full fault,
+	 * reconcile, ret_from_except for user faults which would make them
+	 * first class kernel code. But for performance it's probably nicer
+	 * if they go via fast_exception_return too.
+	 */
+	if (id >= KERNEL_REGION_ID) {
+		return slb_allocate_kernel(ea, id);
+	} else {
+		struct mm_struct *mm = current->mm;
+
+		if (unlikely(!mm))
+			return -EFAULT;
 
-	handle_multi_context_slb_miss(context, ea);
-	exception_exit(prev_state);
-	return;
+		return slb_allocate_user(mm, ea);
+	}
+}

arch/powerpc/mm/slb.c static void insert_slb_entry(unsigned long vsid, unsigned long ea, · line 527

Quoted 10 lines

arch/powerpc/mm/slb.c +2 -2

 
-slb_bad_addr:
+void do_bad_slb_fault(struct pt_regs *regs, unsigned long ea, long err)
+{
 	if (user_mode(regs))
 		_exception(SIGSEGV, regs, SEGV_BNDERR, ea);
 	else
 		bad_page_fault(regs, ea, SIGSEGV);
-	exception_exit(prev_state);
 }

I knew I forgot something -- forgot to test MSR[RI] here. That can be done just by returning a different error from do_slb_fault if RI is clear, and do_bad_slb_fault will call unrecoverable_exception() if it sees that code.

Thanks,
Nick

Michael Ellerman

reviewer

2018-08-21 06:46 UTC #3 lore raw

Re: [RFC PATCH 1/5] powerpc/64s/hash: convert SLB miss handlers to C

In reply to Nicholas Piggin

Quoted 9 lines · Nicholas Piggin

This patch moves SLB miss handlers completely to C, using the standard exception handler macros to set up the stack and branch to C.
This can be done because the segment containing the kernel stack is always bolted, so accessing it with relocation on will not cause an SLB exception.
Arbitrary kernel memory may not be accessed when handling kernel space SLB misses, so care should be taken there.

We'll need to mark everything that's used in slb.c as notrace, otherwise

Probably we just need to mark the whole file as not traceable.

cheers

Nicholas Piggin

author reviewer

2018-08-22 03:17 UTC #4 lore raw

Re: [RFC PATCH 1/5] powerpc/64s/hash: convert SLB miss handlers to C

Quoted 9 lines · On Tue, 21 Aug 2018 16:46:02 +1000 Michael Ellerman <[email protected]> wrote:

Quoted 7 lines · Nicholas Piggin
This patch moves SLB miss handlers completely to C, using the standard exception handler macros to set up the stack and branch to C.
This can be done because the segment containing the kernel stack is always bolted, so accessing it with relocation on will not cause an SLB exception.

Quoted 6 lines

Quoted 2 lines
Arbitrary kernel memory may not be accessed when handling kernel space SLB misses, so care should be taken there.
We'll need to mark everything that's used in slb.c as notrace, otherwise
Probably we just need to mark the whole file as not traceable.

Yeah good point there. I'll do that. The whole file including things we allow today? How do we do that, like this?

CFLAGS_REMOVE_slb.o = -mno-sched-epilog $(CC_FLAGS_FTRACE)

Thanks,
Nick

Michael Ellerman

reviewer

2018-08-22 10:11 UTC #5 lore raw

Re: [RFC PATCH 1/5] powerpc/64s/hash: convert SLB miss handlers to C

Quoted 13 lines · Nicholas Piggin

Quoted 11 lines · On Tue, 21 Aug 2018 16:46:02 +1000 Michael Ellerman <[email protected]> wrote:
Quoted 9 lines · Nicholas Piggin
This patch moves SLB miss handlers completely to C, using the standard exception handler macros to set up the stack and branch to C.
This can be done because the segment containing the kernel stack is always bolted, so accessing it with relocation on will not cause an SLB exception.
Arbitrary kernel memory may not be accessed when handling kernel space SLB misses, so care should be taken there.

Quoted 6 lines

Quoted 3 lines
We'll need to mark everything that's used in slb.c as notrace, otherwise
Probably we just need to mark the whole file as not traceable.
Yeah good point there. I'll do that. The whole file including things we allow today? How do we do that, like this?

For now yeah do the whole file, if there's anything in there we're sure is safe then we can move it out later.

Quoted 1 line

CFLAGS_REMOVE_slb.o = -mno-sched-epilog $(CC_FLAGS_FTRACE)

Yeah AFAIK.

And yet another reminder for me to finally work out if we still need the epilog crap.

cheers

Nicholas Piggin

author reviewer

2018-08-20 09:42 UTC #6 lore raw

RFC PATCH 2/5 powerpc/64s/hash: remove user SLB data from the paca

In reply to Nicholas Piggin

User SLB mappig data is copied into the PACA from the mm->context so it can be accessed by the SLB miss handlers.

After the previous patch, SLB miss handlers now run with relocation on, and user SLB misses are able to take recursive kernel SLB misses, so the user SLB mapping data can be removed from the paca and accessed directly.

---
 arch/powerpc/include/asm/book3s/64/mmu-hash.h |  1 +
 arch/powerpc/include/asm/paca.h               | 13 ------
 arch/powerpc/kernel/asm-offsets.c             |  9 ----
 arch/powerpc/kernel/paca.c                    | 21 ---------
 arch/powerpc/mm/hash_utils_64.c               | 46 +++++--------------
 arch/powerpc/mm/mmu_context.c                 |  3 +-
 arch/powerpc/mm/slb.c                         | 20 +++++++-
 arch/powerpc/mm/slice.c                       | 29 ++++--------
 8 files changed, 40 insertions(+), 102 deletions(-)

arch/powerpc/include/asm/book3s/64/mmu-hash.h +1 -0

index b3520b549cba..39764214aef5 100644
@@ -496,6 +496,7 @@ static inline void hpte_init_pseries(void) { }
 extern void hpte_init_native(void);
 
 extern void slb_initialize(void);
+extern void core_flush_all_slbs(struct mm_struct *mm);
 extern void slb_flush_and_rebolt(void);
 void slb_flush_all_realmode(void);
 void __slb_restore_bolted_realmode(void);

arch/powerpc/include/asm/paca.h +0 ~~-13~~

index 4331295db0f7..8c258a057207 100644
@@ -143,18 +143,6 @@ struct paca_struct {
 	struct tlb_core_data tcd;
 #endif /* CONFIG_PPC_BOOK3E */
 
-#ifdef CONFIG_PPC_BOOK3S
-	mm_context_id_t mm_ctx_id;
-#ifdef CONFIG_PPC_MM_SLICES
-	unsigned char mm_ctx_low_slices_psize[BITS_PER_LONG / BITS_PER_BYTE];
-	unsigned char mm_ctx_high_slices_psize[SLICE_ARRAY_SIZE];
-	unsigned long mm_ctx_slb_addr_limit;
-#else
-	u16 mm_ctx_user_psize;
-	u16 mm_ctx_sllp;
-#endif
-#endif
-
 	/*
 	 * then miscellaneous read-write fields
 	 */

@@ -256,7 +244,6 @@ struct paca_struct {
 #endif /* CONFIG_PPC_PSERIES */
 } ____cacheline_aligned;
 
-extern void copy_mm_to_paca(struct mm_struct *mm);
 extern struct paca_struct **paca_ptrs;
 extern void initialise_paca(struct paca_struct *new_paca, int cpu);
 extern void setup_paca(struct paca_struct *new_paca);

arch/powerpc/kernel/asm-offsets.c +0 -9

index 7834256585f1..43b67ead5b97 100644
@@ -181,15 +181,6 @@ int main(void)
 	OFFSET(PACAIRQSOFTMASK, paca_struct, irq_soft_mask);
 	OFFSET(PACAIRQHAPPENED, paca_struct, irq_happened);
 	OFFSET(PACA_FTRACE_ENABLED, paca_struct, ftrace_enabled);
-#ifdef CONFIG_PPC_BOOK3S
-	OFFSET(PACACONTEXTID, paca_struct, mm_ctx_id);
-#ifdef CONFIG_PPC_MM_SLICES
-	OFFSET(PACALOWSLICESPSIZE, paca_struct, mm_ctx_low_slices_psize);
-	OFFSET(PACAHIGHSLICEPSIZE, paca_struct, mm_ctx_high_slices_psize);
-	OFFSET(PACA_SLB_ADDR_LIMIT, paca_struct, mm_ctx_slb_addr_limit);
-	DEFINE(MMUPSIZEDEFSIZE, sizeof(struct mmu_psize_def));
-#endif /* CONFIG_PPC_MM_SLICES */
-#endif
 
 #ifdef CONFIG_PPC_BOOK3E
 	OFFSET(PACAPGD, paca_struct, pgd);

arch/powerpc/kernel/paca.c +0 ~~-21~~

index 0ee3e6d50f28..6752e17f0281 100644
@@ -259,24 +259,3 @@ void __init free_unused_pacas(void)
 			paca_ptrs_size + paca_struct_size, nr_cpu_ids);
 }
 
-void copy_mm_to_paca(struct mm_struct *mm)
-{
-#ifdef CONFIG_PPC_BOOK3S
-	mm_context_t *context = &mm->context;
-
-	get_paca()->mm_ctx_id = context->id;
-#ifdef CONFIG_PPC_MM_SLICES
-	VM_BUG_ON(!mm->context.slb_addr_limit);
-	get_paca()->mm_ctx_slb_addr_limit = mm->context.slb_addr_limit;
-	memcpy(&get_paca()->mm_ctx_low_slices_psize,
-	       &context->low_slices_psize, sizeof(context->low_slices_psize));
-	memcpy(&get_paca()->mm_ctx_high_slices_psize,
-	       &context->high_slices_psize, TASK_SLICE_ARRAY_SZ(mm));
-#else /* CONFIG_PPC_MM_SLICES */
-	get_paca()->mm_ctx_user_psize = context->user_psize;
-	get_paca()->mm_ctx_sllp = context->sllp;
-#endif
-#else /* !CONFIG_PPC_BOOK3S */
-	return;
-#endif
-}

arch/powerpc/mm/hash_utils_64.c +11 ~~-35~~

111 lines not on this page · raw message

arch/powerpc/mm/mmu_context.c +1 -2

index f84e14f23e50..28ae2835db3d 100644
@@ -54,8 +54,7 @@ void switch_mm_irqs_off(struct mm_struct *prev, struct mm_struct *next,
 		 * MMU context id, which is then moved to SPRN_PID.
 		 *
 		 * For the hash MMU it is either the first load from slb_cache
-		 * in switch_slb(), and/or the store of paca->mm_ctx_id in
-		 * copy_mm_to_paca().
+		 * in switch_slb(), and/or load of MMU context id.
 		 *
 		 * On the other side, the barrier is in mm/tlb-radix.c for
 		 * radix which orders earlier stores to clear the PTEs vs

arch/powerpc/mm/slb.c +18 -2

index 5643879fa2b1..221d94b4f9cf 100644
@@ -274,8 +274,6 @@ void switch_slb(struct task_struct *tsk, struct mm_struct *mm)
 		get_paca()->slb_cache_ptr = 0;
 	}
 
-	copy_mm_to_paca(mm);
-
 	/*
 	 * preload some userspace segments into the SLB.
 	 * Almost all 32 and 64bit PowerPC executables are linked at

@@ -302,6 +300,24 @@ void slb_set_size(u16 size)
 	mmu_slb_size = size;
 }
 
+static void cpu_flush_slb(void *parm)
+{
+        struct mm_struct *mm = parm;
+        unsigned long flags;
+
+        if (mm != current->active_mm)
+                return;
+
+        local_irq_save(flags);
+        slb_flush_and_rebolt();
+        local_irq_restore(flags);
+}
+
+void core_flush_all_slbs(struct mm_struct *mm)
+{
+	on_each_cpu(cpu_flush_slb, mm, 1);
+}
+
 void slb_initialize(void)
 {
 	unsigned long linear_llp, vmalloc_llp, io_llp;

arch/powerpc/mm/slice.c +9 ~~-20~~

index 205fe557ca10..606f424aac47 100644
@@ -207,23 +207,6 @@ static bool slice_check_range_fits(struct mm_struct *mm,
 	return true;
 }
 
-static void slice_flush_segments(void *parm)
-{
-#ifdef CONFIG_PPC64
-	struct mm_struct *mm = parm;
-	unsigned long flags;
-
-	if (mm != current->active_mm)
-		return;
-
-	copy_mm_to_paca(current->active_mm);
-
-	local_irq_save(flags);
-	slb_flush_and_rebolt();
-	local_irq_restore(flags);
-#endif
-}
-
 static void slice_convert(struct mm_struct *mm,
 				const struct slice_mask *mask, int psize)
 {

@@ -289,6 +272,9 @@ static void slice_convert(struct mm_struct *mm,
 	spin_unlock_irqrestore(&slice_convert_lock, flags);
 
 	copro_flush_all_slbs(mm);
+#ifdef CONFIG_PPC64
+	core_flush_all_slbs(mm);
+#endif
 }
 
 /*

@@ -502,8 +488,9 @@ unsigned long slice_get_unmapped_area(unsigned long addr, unsigned long len,
 		 * be already initialised beyond the old address limit.
 		 */
 		mm->context.slb_addr_limit = high_limit;
-
-		on_each_cpu(slice_flush_segments, mm, 1);
+#ifdef CONFIG_PPC64
+		core_flush_all_slbs(mm);
+#endif
 	}
 
 	/* Sanity checks */

@@ -665,8 +652,10 @@ unsigned long slice_get_unmapped_area(unsigned long addr, unsigned long len,
 		(SLICE_NUM_HIGH &&
 		 !bitmap_empty(potential_mask.high_slices, SLICE_NUM_HIGH))) {
 		slice_convert(mm, &potential_mask, psize);
+#ifdef CONFIG_PPC64
 		if (psize > MMU_PAGE_BASE)
-			on_each_cpu(slice_flush_segments, mm, 1);
+			core_flush_all_slbs(mm);
+#endif
 	}
 	return newaddr;

Signature · 1 line

2.17.0

Nicholas Piggin

author reviewer

2018-08-20 09:42 UTC #7 lore raw

RFC PATCH 3/5 powerpc/64s/hash: remove the first vmalloc segment from the bolted SLB

In reply to Nicholas Piggin

Remove the first vmalloc segment from bolted SLBEs. This is not required to be bolted, and seems like it was added to help pre-load the SLB on context switch. However there are now other segments like the vmemmap segment that often take misses after a context switch, so it is better to solve this a different way and save a bolted entry.

---
 arch/powerpc/include/asm/book3s/64/mmu-hash.h |  2 +-
 arch/powerpc/mm/slb.c                         | 16 ++++------------
 2 files changed, 5 insertions(+), 13 deletions(-)

arch/powerpc/include/asm/book3s/64/mmu-hash.h +1 -1

index 39764214aef5..4c8d413ce99a 100644
@@ -30,7 +30,7 @@
  * SLB
  */
 
-#define SLB_NUM_BOLTED		3
+#define SLB_NUM_BOLTED		2
 #define SLB_CACHE_ENTRIES	8
 #define SLB_MIN_SIZE		32

arch/powerpc/mm/slb.c +4 ~~-12~~

index 221d94b4f9cf..6e595d75d997 100644
@@ -133,13 +133,11 @@ static void __slb_flush_and_rebolt(void)
 {
 	/* If you change this make sure you change SLB_NUM_BOLTED
 	 * and PR KVM appropriately too. */
-	unsigned long linear_llp, vmalloc_llp, lflags, vflags;
+	unsigned long linear_llp, lflags;
 	unsigned long ksp_esid_data, ksp_vsid_data;
 
 	linear_llp = mmu_psize_defs[mmu_linear_psize].sllp;
-	vmalloc_llp = mmu_psize_defs[mmu_vmalloc_psize].sllp;
 	lflags = SLB_VSID_KERNEL | linear_llp;
-	vflags = SLB_VSID_KERNEL | vmalloc_llp;
 
 	ksp_esid_data = mk_esid_data(get_paca()->kstack, mmu_kernel_ssize, KSTACK_INDEX);
 	if ((ksp_esid_data & ~0xfffffffUL) <= PAGE_OFFSET) {

@@ -157,14 +155,10 @@ static void __slb_flush_and_rebolt(void)
 	 * the stack between the slbia and rebolting it. */
 	asm volatile("isync\n"
 		     "slbia\n"
-		     /* Slot 1 - first VMALLOC segment */
+		     /* Slot 1 - kernel stack */
 		     "slbmte	%0,%1\n"
-		     /* Slot 2 - kernel stack */
-		     "slbmte	%2,%3\n"
 		     "isync"
-		     :: "r"(mk_vsid_data(VMALLOC_START, mmu_kernel_ssize, vflags)),
-		        "r"(mk_esid_data(VMALLOC_START, mmu_kernel_ssize, VMALLOC_INDEX)),
-		        "r"(ksp_vsid_data),
+		     :: "r"(ksp_vsid_data),
 		        "r"(ksp_esid_data)
 		     : "memory");
 }

@@ -321,7 +315,7 @@ void core_flush_all_slbs(struct mm_struct *mm)
 void slb_initialize(void)
 {
 	unsigned long linear_llp, vmalloc_llp, io_llp;
-	unsigned long lflags, vflags;
+	unsigned long lflags;
 	static int slb_encoding_inited;
 #ifdef CONFIG_SPARSEMEM_VMEMMAP
 	unsigned long vmemmap_llp;

@@ -347,14 +341,12 @@ void slb_initialize(void)
 	get_paca()->stab_rr = SLB_NUM_BOLTED - 1;
 
 	lflags = SLB_VSID_KERNEL | linear_llp;
-	vflags = SLB_VSID_KERNEL | vmalloc_llp;
 
 	/* Invalidate the entire SLB (even entry 0) & all the ERATS */
 	asm volatile("isync":::"memory");
 	asm volatile("slbmte  %0,%0"::"r" (0) : "memory");
 	asm volatile("isync; slbia; isync":::"memory");
 	create_shadowed_slbe(PAGE_OFFSET, mmu_kernel_ssize, lflags, LINEAR_INDEX);
-	create_shadowed_slbe(VMALLOC_START, mmu_kernel_ssize, vflags, VMALLOC_INDEX);
 
 	/* For the boot cpu, we're running on the stack in init_thread_union,
 	 * which is in the first segment of the linear mapping, and also

Signature · 1 line

2.17.0

Nicholas Piggin

author reviewer

2018-08-20 09:42 UTC #8 lore raw

RFC PATCH 4/5 powerpc/64s/hash: Add SLB allocation bitmaps

In reply to Nicholas Piggin

Add 32-entry bitmaps to track the allocation status of the first 32 SLB entries, and whether they are user or kernel entries. These are used to prevent context switches rolling the SLB round robin allocator and evicting important kernel SLBEs when there are obvious free entries.

---
 arch/powerpc/include/asm/paca.h |  6 +++--
 arch/powerpc/mm/slb.c           | 42 +++++++++++++++++++++++----------
 arch/powerpc/xmon/xmon.c        |  2 +-
 3 files changed, 35 insertions(+), 15 deletions(-)

arch/powerpc/include/asm/paca.h +4 -2

index 8c258a057207..bf7ab59be3b8 100644
@@ -113,7 +113,10 @@ struct paca_struct {
  				 * on the linear mapping */
 	/* SLB related definitions */
 	u16 vmalloc_sllp;
-	u16 slb_cache_ptr;
+	u8 slb_cache_ptr;
+	u8 stab_rr;			/* stab/slb round-robin counter */
+	u32 slb_used_bitmap;		/* Bitmaps for first 32 SLB entries. */
+	u32 slb_kern_bitmap;
 	u32 slb_cache[SLB_CACHE_ENTRIES];
 #endif /* CONFIG_PPC_BOOK3S_64 */

@@ -148,7 +151,6 @@ struct paca_struct {
 	 */
 	struct task_struct *__current;	/* Pointer to current */
 	u64 kstack;			/* Saved Kernel stack addr */
-	u64 stab_rr;			/* stab/slb round-robin counter */
 	u64 saved_r1;			/* r1 save for RTAS calls or PM or EE=0 */
 	u64 saved_msr;			/* MSR saved here by enter_rtas */
 	u16 trap_save;			/* Used when bad stack is encountered */

arch/powerpc/mm/slb.c +30 ~~-12~~

83 lines not on this page · raw message

arch/powerpc/xmon/xmon.c +1 -1

index 8c1d8b4bf6d5..44e16675c7f9 100644
@@ -2415,7 +2415,7 @@ static void dump_one_paca(int cpu)
 	DUMP(p, __current, "%-*px");
 	DUMP(p, kstack, "%#-*llx");
 	printf(" %-*s = 0x%016llx\n", 25, "kstack_base", p->kstack & ~(THREAD_SIZE - 1));
-	DUMP(p, stab_rr, "%#-*llx");
+	DUMP(p, stab_rr, "%#-*x");
 	DUMP(p, saved_r1, "%#-*llx");
 	DUMP(p, trap_save, "%#-*x");
 	DUMP(p, irq_soft_mask, "%#-*x");

Signature · 1 line

2.17.0

Nicholas Piggin

author reviewer

2018-08-20 09:42 UTC #9 lore raw

RFC PATCH 5/5 powerpc/64s/hash: Add a SLB preload cache

In reply to Nicholas Piggin

When switching processes, currently all user SLBEs are cleared, and a few (exec_base, pc, and stack) are preloaded. In trivial testing with small apps, this tends to miss the heap and low 256MB segments, and it will also miss commonly accessed segments on large memory workloads.

Add a simple round-robin preload cache that just inserts the last SLB miss into the head of the cache and preloads those at context switch time.

Much more could go into this, including into the SLB entry reclaim side to track some LRU information etc, which would require a study of large memory workloads. But this is a simple thing we can do now that is an obvious win for common workloads.

This plus the previous patch reduces SLB misses of a bare bones boot to busybox from 945 to 180 when using 256MB segments, and 900 to 100 when using 1T segments. These could almost all be eliminated by preloading a bit more carefully with ELF binary loading.

---
 arch/powerpc/include/asm/thread_info.h |   4 +
 arch/powerpc/kernel/process.c          |   6 ++
 arch/powerpc/mm/mmu_context_book3s64.c |  10 ++-
 arch/powerpc/mm/slb.c                  | 107 ++++++++++++++++++++-----
 4 files changed, 102 insertions(+), 25 deletions(-)

arch/powerpc/include/asm/thread_info.h +4 -0

index 3c0002044bc9..ee5e49ec12c7 100644
@@ -29,6 +29,7 @@
 #include <asm/page.h>
 #include <asm/accounting.h>
 
+#define SLB_PRELOAD_NR	8U
 /*
  * low level task data.
  */

@@ -44,6 +45,9 @@ struct thread_info {
 #if defined(CONFIG_VIRT_CPU_ACCOUNTING_NATIVE) && defined(CONFIG_PPC32)
 	struct cpu_accounting_data accounting;
 #endif
+	unsigned int slb_preload_nr;
+	unsigned long slb_preload_ea[SLB_PRELOAD_NR];
+
 	/* low level flags - has atomic operations done on it */
 	unsigned long	flags ____cacheline_aligned_in_smp;
 };

arch/powerpc/kernel/process.c +6 -0

index 913c5725cdb2..678a2c668270 100644
@@ -1710,6 +1710,8 @@ int copy_thread(unsigned long clone_flags, unsigned long usp,
 	return 0;
 }
 
+void preload_new_slb_context(unsigned long start, unsigned long sp);
+
 /*
  * Set up a thread for executing a new program
  */

@@ -1717,6 +1719,10 @@ void start_thread(struct pt_regs *regs, unsigned long start, unsigned long sp)
 {
 #ifdef CONFIG_PPC64
 	unsigned long load_addr = regs->gpr[2];	/* saved by ELF_PLAT_INIT */
+
+#ifdef CONFIG_PPC_BOOK3S_64
+	preload_new_slb_context(start, sp);
+#endif
 #endif
 
 	/*

arch/powerpc/mm/mmu_context_book3s64.c +7 -3

index 4a892d894a0f..3671a32141e2 100644
@@ -85,7 +85,9 @@ int hash__alloc_context_id(void)
 }
 EXPORT_SYMBOL_GPL(hash__alloc_context_id);
 
-static int hash__init_new_context(struct mm_struct *mm)
+void init_new_slb_context(struct task_struct *tsk, struct mm_struct *mm);
+
+static int hash__init_new_context(struct task_struct *tsk, struct mm_struct *mm)
 {
 	int index;

@@ -107,8 +109,10 @@ static int hash__init_new_context(struct mm_struct *mm)
 	 * We should not be calling init_new_context() on init_mm. Hence a
 	 * check against 0 is OK.
 	 */
-	if (mm->context.id == 0)
+	if (mm->context.id == 0) {
 		slice_init_new_context_exec(mm);
+		init_new_slb_context(tsk, mm);
+	}
 
 	subpage_prot_init_new_context(mm);

@@ -152,7 +156,7 @@ int init_new_context(struct task_struct *tsk, struct mm_struct *mm)
 	if (radix_enabled())
 		index = radix__init_new_context(mm);
 	else
-		index = hash__init_new_context(mm);
+		index = hash__init_new_context(tsk, mm);
 
 	if (index < 0)
 		return index;

arch/powerpc/mm/slb.c +85 ~~-22~~

135 lines not on this page · raw message

Signature · 1 line

2.17.0