powerpc: Preliminary work to enable SMP BookE (v3)
Subsystems (6)
cover powerpc: Preliminary work to enable SMP BookE (v3)
This series of patches is aimed at supporting SMP on non-hash based processors. It consists of a rework of the MMU context management and TLB management, clearly splitting hash32, hash64 and nohash in both cases, adding SMP safe context handling and some basic SMP TLB management.
There is room for improvements, such as implementing lazy TLB flushing on processors without invalidate-by-PID support HW, some better IPI mechanism, lock less fast path in the MMU context switch, etc... but it should basically work.
This version addresses the issues raised about some of the patches in the previous batch, removes the ones that were already merged in -next and and rebases the remaining ones when needed.
PATCH 1/10 powerpc/4xx: Extended DCR support v2
This adds supports to the "extended" DCR addressing via the indirect mfdcrx/mtdcrx instructions supported by some 4xx cores (440H6 and later)
I enabled the feature for now only on AMCC 460 chips
---
This variant uses "440x6" instead of "440H6". I made no other changes to the code as I think the codegen is the less bad I've had so far and I rely on Josh further work on cleaning up the type of 440core selection at Kconfig time so the feature are properly reflected in the POSSIBLE and ALWAYS masks based on the core selection. That way, if only one core type is selected the feature test should resolve at compile time.
arch/powerpc/include/asm/cputable.h | 7 ++- arch/powerpc/include/asm/dcr-native.h | 63 +++++++++++++++++++++++++++------- arch/powerpc/kernel/cputable.c | 4 +- arch/powerpc/sysdev/dcr-low.S | 8 +++- 4 files changed, 65 insertions(+), 17 deletions(-)
arch/powerpc/include/asm/cputable.h +5 -2
@@ -164,6 +164,7 @@ extern const char *powerpc_base_platform#define CPU_FTR_NEED_PAIRED_STWCX ASM_CONST(0x0000000004000000)#define CPU_FTR_LWSYNC ASM_CONST(0x0000000008000000)#define CPU_FTR_NOEXECUTE ASM_CONST(0x0000000010000000) +#define CPU_FTR_INDEXED_DCR ASM_CONST(0x0000000020000000)/* * Add the 64-bit processor unique features in the top half of the word;
@@ -369,6 +370,8 @@ extern const char *powerpc_base_platform#define CPU_FTRS_8XX (CPU_FTR_USE_TB)#define CPU_FTRS_40X (CPU_FTR_USE_TB | CPU_FTR_NODSISRALIGN | CPU_FTR_NOEXECUTE)#define CPU_FTRS_44X (CPU_FTR_USE_TB | CPU_FTR_NODSISRALIGN | CPU_FTR_NOEXECUTE) +#define CPU_FTRS_440x6 (CPU_FTR_USE_TB | CPU_FTR_NODSISRALIGN | CPU_FTR_NOEXECUTE | \ + CPU_FTR_INDEXED_DCR)#define CPU_FTRS_E200 (CPU_FTR_USE_TB | CPU_FTR_SPE_COMP | \ CPU_FTR_NODSISRALIGN | CPU_FTR_COHERENT_ICACHE | \ CPU_FTR_UNIFIED_ID_CACHE | CPU_FTR_NOEXECUTE)
@@ -455,7 +458,7 @@ enum { CPU_FTRS_40X |#endif #ifdef CONFIG_44x - CPU_FTRS_44X | + CPU_FTRS_44X | CPU_FTRS_440x6 |#endif #ifdef CONFIG_E200 CPU_FTRS_E200 |
@@ -495,7 +498,7 @@ enum { CPU_FTRS_40X &#endif #ifdef CONFIG_44x - CPU_FTRS_44X & + CPU_FTRS_44X & CPU_FTRS_440x6 &#endif #ifdef CONFIG_E200 CPU_FTRS_E200 &
arch/powerpc/include/asm/dcr-native.h +51 -12
105 lines not on this page · raw message
arch/powerpc/sysdev/dcr-low.S +7 -1
@@ -11,14 +11,20 @@ #include <asm/ppc_asm.h> #include <asm/processor.h> +#include <asm/bug.h> #define DCR_ACCESS_PROLOG(table) \ + cmpli cr0,r3,1024; \ rlwinm r3,r3,4,18,27; \ lis r5,table@h; \ ori r5,r5,table@l; \ add r3,r3,r5; \ + bge- 1f; \ mtctr r3; \ - bctr + bctr; \ +1: trap; \ + EMIT_BUG_ENTRY 1b,__FILE__,__LINE__,0; \ + blr _GLOBAL(__mfdcr) DCR_ACCESS_PROLOG(__mfdcr_table)
arch/powerpc/kernel/cputable.c +2 -2
@@ -1509,7 +1509,7 @@ static struct cpu_spec __initdata cpu_sp .pvr_mask = 0xffff0002, .pvr_value = 0x13020002, .cpu_name = "460EX", - .cpu_features = CPU_FTRS_44X, + .cpu_features = CPU_FTRS_440x6, .cpu_user_features = COMMON_USER_BOOKE | PPC_FEATURE_HAS_FPU, .icache_bsize = 32, .dcache_bsize = 32,
@@ -1521,7 +1521,7 @@ static struct cpu_spec __initdata cpu_sp .pvr_mask = 0xffff0002, .pvr_value = 0x13020000, .cpu_name = "460GT", - .cpu_features = CPU_FTRS_44X, + .cpu_features = CPU_FTRS_440x6, .cpu_user_features = COMMON_USER_BOOKE | PPC_FEATURE_HAS_FPU, .icache_bsize = 32, .dcache_bsize = 32,
Re: [PATCH 1/10] powerpc/4xx: Extended DCR support v2
Quoted 7 lines · Benjamin Herrenschmidt
This adds supports to the "extended" DCR addressing via the indirect mfdcrx/mtdcrx instructions supported by some 4xx cores (440H6 and later)
I enabled the feature for now only on AMCC 460 chips
Signed-off-by Benjamin Herrenschmidt <[email protected]>
In general, if you just carry over my previous Acks it'll make life easier ;)
josh
Quoted 216 lines
---
This variant uses "440x6" instead of "440H6". I made no other changes to the code as I think the codegen is the less bad I've had so far and I rely on Josh further work on cleaning up the type of 440core selection at
Kconfigtime so the feature are properly reflected in the POSSIBLE and ALWAYS masks based on the core selection. That way, if only one core type is selected the feature test should resolve at compile time.arch/powerpc/include/asm/cputable.h | 7 ++- arch/powerpc/include/asm/dcr-native.h | 63 +++++++++++++++++++++++++++------- arch/powerpc/kernel/cputable.c | 4 +- arch/powerpc/sysdev/dcr-low.S | 8 +++- 4 files changed, 65 insertions(+), 17 deletions(-)arch/powerpc/include/asm/cputable.h +5
-2@@ -164,6 +164,7 @@ extern const char *powerpc_base_platform#define CPU_FTR_NEED_PAIRED_STWCX ASM_CONST(0x0000000004000000)#define CPU_FTR_LWSYNC ASM_CONST(0x0000000008000000)#define CPU_FTR_NOEXECUTE ASM_CONST(0x0000000010000000) +#define CPU_FTR_INDEXED_DCR ASM_CONST(0x0000000020000000)/* * Add the 64-bit processor unique features in the top half of the word; @@ -369,6 +370,8 @@ extern const char *powerpc_base_platform#define CPU_FTRS_8XX (CPU_FTR_USE_TB)#define CPU_FTRS_40X (CPU_FTR_USE_TB | CPU_FTR_NODSISRALIGN | CPU_FTR_NOEXECUTE)#define CPU_FTRS_44X (CPU_FTR_USE_TB | CPU_FTR_NODSISRALIGN | CPU_FTR_NOEXECUTE) +#define CPU_FTRS_440x6 (CPU_FTR_USE_TB | CPU_FTR_NODSISRALIGN | CPU_FTR_NOEXECUTE | \ + CPU_FTR_INDEXED_DCR)#define CPU_FTRS_E200 (CPU_FTR_USE_TB | CPU_FTR_SPE_COMP | \ CPU_FTR_NODSISRALIGN | CPU_FTR_COHERENT_ICACHE | \ CPU_FTR_UNIFIED_ID_CACHE | CPU_FTR_NOEXECUTE)@@ -455,7 +458,7 @@ enum { CPU_FTRS_40X |#endif #ifdef CONFIG_44x - CPU_FTRS_44X | + CPU_FTRS_44X | CPU_FTRS_440x6 |#endif #ifdef CONFIG_E200 CPU_FTRS_E200 |@@ -495,7 +498,7 @@ enum { CPU_FTRS_40X &#endif #ifdef CONFIG_44x - CPU_FTRS_44X & + CPU_FTRS_44X & CPU_FTRS_440x6 &#endif #ifdef CONFIG_E200 CPU_FTRS_E200 &Index:
linux-work/arch/powerpc/include/asm/dcr-native.h
===================================================================arch/powerpc/include/asm/dcr-native.h +51
-12@@ -23,6 +23,7 @@ #ifndef __ASSEMBLY__ #include <linux/spinlock.h> +#include <asm/cputable.h> typedef struct { unsigned int base;@@ -39,23 +40,45 @@ static inline bool dcr_map_ok_native(dcr#define dcr_read_native (host, dcr_n) mfdcr(dcr_n + host.base)#define dcr_write_native (host, dcr_n, value) mtdcr(dcr_n + host.base, value) -/* Device Control Registers */ -void __mtdcr (int reg,unsigned int val); -unsigned int __mfdcr (int reg); +/* Table based DCR accessors */ +extern void __mtdcr (unsigned int reg,unsigned int val); +extern unsigned int __mfdcr (unsigned int reg); + +/* mfdcrx/mtdcrx instruction based accessors. We hand code +* the opcodes in order not to depend on newer binutils +*/ +static inline unsigned int mfdcrx (unsigned int reg) +{ +unsigned int ret; + asmvolatile (".long 0x7c000206 | (%0 << 21) | (%1 << 16)" + :"=r" (ret) :"r" (reg)); +return ret; +} + +static inline void mtdcrx (unsigned int reg,unsigned int val) +{ + asmvolatile (".long 0x7c000306 | (%0 << 21) | (%1 << 16)" + : :"r" (val),"r" (reg)); +} +#define mfdcr (rn) \ ({unsigned int rval; \ - if (__builtin_constant_p(rn)) \ + if (__builtin_constant_p(rn) && rn < 1024) \ asm volatile("mfdcr %0," __stringify(rn) \ : "=r" (rval)); \ + else if (likely(cpu_has_feature(CPU_FTR_INDEXED_DCR))) \ + rval = mfdcrx(rn); \ else \ rval = __mfdcr(rn); \ rval;})#define mtdcr (rn, v) \ do { \ - if (__builtin_constant_p(rn)) \ + if (__builtin_constant_p(rn) && rn < 1024) \ asm volatile("mtdcr " __stringify(rn) ",%0" \ : : "r" (v)); \ + else if (likely(cpu_has_feature(CPU_FTR_INDEXED_DCR))) \ + mtdcrx(rn, v); \ else \ __mtdcr(rn, v); \ } while (0)@@ -69,8 +92,13 @@ static inline unsigned __mfdcri(int baseunsigned int val;spin_lock_irqsave (&dcr_ind_lock, flags); -__mtdcr (base_addr, reg); - val =__mfdcr (base_data); +if (cpu_has_feature (CPU_FTR_INDEXED_DCR )) { +mtdcrx (base_addr, reg); + val =mfdcrx (base_data); + }else { +__mtdcr (base_addr, reg); + val =__mfdcr (base_data); + }spin_unlock_irqrestore (&dcr_ind_lock, flags);return val; }@@ -81,8 +109,13 @@ static inline void __mtdcri(int base_addunsigned long flags;spin_lock_irqsave (&dcr_ind_lock, flags); -__mtdcr (base_addr, reg); -__mtdcr (base_data, val); +if (cpu_has_feature (CPU_FTR_INDEXED_DCR )) { +mtdcrx (base_addr, reg); +mtdcrx (base_data, val); + }else { +__mtdcr (base_addr, reg); +__mtdcr (base_data, val); + }spin_unlock_irqrestore (&dcr_ind_lock, flags); }@@ -93,9 +126,15 @@ static inline void __dcri_clrset(int basunsigned int val;spin_lock_irqsave (&dcr_ind_lock, flags); -__mtdcr (base_addr, reg); - val = (__mfdcr (base_data) & ~clr) | set; -__mtdcr (base_data, val); +if (cpu_has_feature (CPU_FTR_INDEXED_DCR )) { +mtdcrx (base_addr, reg); + val = (mfdcrx (base_data) & ~clr) | set; +mtdcrx (base_data, val); + }else { +__mtdcr (base_addr, reg); + val = (__mfdcr (base_data) & ~clr) | set; +__mtdcr (base_data, val); + }spin_unlock_irqrestore (&dcr_ind_lock, flags); }Index:
linux-work/arch/powerpc/sysdev/dcr-low.S
===================================================================arch/powerpc/sysdev/dcr-low.S +7
-1@@ -11,14 +11,20 @@ #include <asm/ppc_asm.h> #include <asm/processor.h> +#include <asm/bug.h> #define DCR_ACCESS_PROLOG(table) \ + cmpli cr0,r3,1024; \ rlwinm r3,r3,4,18,27; \ lis r5,table@h; \ ori r5,r5,table@l; \ add r3,r3,r5; \ + bge- 1f; \ mtctr r3; \ - bctr + bctr; \ +1: trap; \ + EMIT_BUG_ENTRY 1b,__FILE__,__LINE__,0; \ + blr _GLOBAL(__mfdcr) DCR_ACCESS_PROLOG(__mfdcr_table)Index:
linux-work/arch/powerpc/kernel/cputable.c
===================================================================arch/powerpc/kernel/cputable.c +2
-2@@ -1509,7 +1509,7 @@ static struct cpu_spec __initdata cpu_sp .pvr_mask = 0xffff0002, .pvr_value = 0x13020002, .cpu_name = "460EX", - .cpu_features = CPU_FTRS_44X, + .cpu_features = CPU_FTRS_440x6, .cpu_user_features = COMMON_USER_BOOKE | PPC_FEATURE_HAS_FPU, .icache_bsize = 32, .dcache_bsize = 32,@@ -1521,7 +1521,7 @@ static struct cpu_spec __initdata cpu_sp .pvr_mask = 0xffff0002, .pvr_value = 0x13020000, .cpu_name = "460GT", - .cpu_features = CPU_FTRS_44X, + .cpu_features = CPU_FTRS_440x6, .cpu_user_features = COMMON_USER_BOOKE | PPC_FEATURE_HAS_FPU, .icache_bsize = 32, .dcache_bsize = 32,
PATCH 2/10 powerpc/mm: Split mmu_context handling v3
In reply to Benjamin HerrenschmidtThis splits the mmu_context handling between 32-bit hash based processors, 64-bit hash based processors and everybody else. This is preliminary work for adding SMP support for BookE processors.
---
v2. address various comments for Josh and Stephen
v3. properly removes the old mmu_context_32.c and mmu_context_64.c
arch/powerpc/include/asm/mmu_context.h | 260 +++------------------------ arch/powerpc/kernel/asm-offsets.c | 1 arch/powerpc/kernel/head_32.S | 12 + arch/powerpc/kernel/ppc_ksyms.c | 3 arch/powerpc/kernel/swsusp.c | 2 arch/powerpc/mm/Makefile | 7 arch/powerpc/mm/mmu_context_32.c | 84 -------- arch/powerpc/mm/mmu_context_64.c | 70 ------- arch/powerpc/mm/mmu_context_hash32.c | 103 ++++++++++ arch/powerpc/mm/mmu_context_hash64.c | 78 ++++++++ arch/powerpc/mm/mmu_context_nohash.c | 162 ++++++++++++++++ arch/powerpc/platforms/Kconfig.cputype | 10 - arch/powerpc/platforms/powermac/cpufreq_32.c | 2 drivers/macintosh/via-pmu.c | 4 14 files changed, 407 insertions(+), 391 deletions(-)
arch/powerpc/include/asm/mmu_context.h +34 -226
@@ -2,240 +2,26 @@#define __ASM_POWERPC_MMU_CONTEXT_H#ifdef __KERNEL__ +#include <linux/kernel.h> +#include <linux/mm.h> +#include <linux/sched.h> +#include <linux/spinlock.h> #include <asm/mmu.h> #include <asm/cputable.h> #include <asm-generic/mm_hooks.h> - -#ifndef CONFIG_PPC64 -#include <asm/atomic.h> -#include <linux/bitops.h> - -/* -* On 32-bit PowerPC 6xx/7xx/7xxx CPUs, we use a set of 16 VSIDs -* (virtual segment identifiers) for each context. Although the -* hardware supports 24-bit VSIDs, and thus >1 million contexts, -* we only use 32,768 of them. That is ample, since there can be -* at most around 30,000 tasks in the system anyway, and it means -* that we can use a bitmap to indicate which contexts are in use. -* Using a bitmap means that we entirely avoid all of the problems -* that we used to have when the context number overflowed, -* particularly on SMP systems. -* -- paulus. -*/ - -/* -* This function defines the mapping from contexts to VSIDs (virtual -* segment IDs). We use a skew on both the context and the high 4 bits -* of the 32-bit virtual address (the "effective segment ID") in order -* to spread out the entries in the MMU hash table. Note, if this -* function is changed then arch/ppc/mm/hashtable.S will have to be -* changed to correspond. -*/ -#define CTX_TO_VSID (ctx, va) (((ctx) * (897 * 16) + ((va) >> 28) * 0x111) \ - & 0xffffff) - -/* -The MPC8xx has only 16 contexts. We rotate through them on each -task switch. A better way would be to keep track of tasks that -own contexts, and implement an LRU usage. That way very active -tasks don't always have to pay the TLB reload overhead. The -kernel pages are mapped shared, so the kernel can run on behalf -of any task that makes a kernel entry. Shared does not mean they -are not protected, just that the ASID comparison is not performed. --- Dan - -The IBM4xx has 256 contexts, so we can just rotate through these -as a way of "switching" contexts. If the TID of the TLB is zero, -the PID/TID comparison is disabled, so we can use a TID of zero -to represent all kernel pages as shared among all contexts. --- Dan -*/ - -static inline void enter_lazy_tlb (struct mm_struct *mm,struct task_struct *tsk) -{ -} - -#ifdef CONFIG_8xx -#define NO_CONTEXT 16 -#define LAST_CONTEXT 15 -#define FIRST_CONTEXT 0 - -#elif defined(CONFIG_4xx) -#define NO_CONTEXT 256 -#define LAST_CONTEXT 255 -#define FIRST_CONTEXT 1 - -#elif defined(CONFIG_E200 ) || defined(CONFIG_E500 ) -#define NO_CONTEXT 256 -#define LAST_CONTEXT 255 -#define FIRST_CONTEXT 1 - -#else - -/* PPC 6xx, 7xx CPUs */ -#define NO_CONTEXT ((unsigned long) -1) -#define LAST_CONTEXT 32767 -#define FIRST_CONTEXT 1 -#endif - -/* -* Set the current MMU context. -* On 32-bit PowerPCs (other than the 8xx embedded chips), this is done by -* loading up the segment registers for the user part of the address space. -* -* Since the PGD is immediately available, it is much faster to simply -* pass this along as a second parameter, which is required for 8xx and -* can be used for debugging on all processors (if you happen to have -* an Abatron). -*/ -extern void set_context (unsigned long contextid,pgd_t *pgd); - -/* -* Bitmap of contexts in use. -* The size of this bitmap is LAST_CONTEXT + 1 bits. -*/ -extern unsigned long context_map[]; - -/* -* This caches the next context number that we expect to be free. -* Its use is an optimization only, we can't rely on this context -* number to be free, but it usually will be. -*/ -extern unsigned long next_mmu_context; - -/* -* If we don't have sufficient contexts to give one to every task -* that could be in the system, we need to be able to steal contexts. -* These variables support that. -*/ -#if LAST_CONTEXT <30000 -#define FEW_CONTEXTS 1 -extern atomic_t nr_free_contexts; -extern struct mm_struct *context_mm[LAST_CONTEXT +1 ]; -extern void steal_context (void ); -#endif - -/* -* Get a new mmu context for the address space described by `mm'. -*/ -static inline void get_mmu_context (struct mm_struct *mm) -{ -unsigned long ctx; - -if (mm->context.id !=NO_CONTEXT ) -return ; -#ifdef FEW_CONTEXTS -while (atomic_dec_if_positive (&nr_free_contexts) <0 ) -steal_context (); -#endif - ctx = next_mmu_context; -while (test_and_set_bit (ctx, context_map)) { - ctx =find_next_zero_bit (context_map,LAST_CONTEXT +1 , ctx); -if (ctx >LAST_CONTEXT ) - ctx =0 ; - } - next_mmu_context = (ctx +1 ) &LAST_CONTEXT ; - mm->context.id = ctx; -#ifdef FEW_CONTEXTS - context_mm[ctx] = mm; -#endif -} - -/* -* Set up the context for a new address space. -*/ -static inline int init_new_context (struct task_struct *t,struct mm_struct *mm) -{ - mm->context.id =NO_CONTEXT ; -return 0 ; -} - -/* -* We're finished using the context for an address space. -*/ -static inline void destroy_context (struct mm_struct *mm) -{ -preempt_disable (); -if (mm->context.id !=NO_CONTEXT ) { -clear_bit (mm->context.id, context_map); - mm->context.id =NO_CONTEXT ; -#ifdef FEW_CONTEXTS -atomic_inc (&nr_free_contexts); -#endif - } -preempt_enable (); -} - -static inline void switch_mm (struct mm_struct *prev,struct mm_struct *next, -struct task_struct *tsk) -{ -#ifdef CONFIG_ALTIVEC -if (cpu_has_feature (CPU_FTR_ALTIVEC )) - asmvolatile ("dssall;\n" -#ifndef CONFIG_POWER4 -"sync;\n" /* G4 needs a sync here, G5 apparently not */ -#endif - : : ); -#endif /* CONFIG_ALTIVEC */ - - tsk->thread.pgdir = next->pgd; - -if (!cpu_isset (smp_processor_id (), next->cpu_vm_mask)) -cpu_set (smp_processor_id (), next->cpu_vm_mask); - -/* No need to flush userspace segments if the mm doesnt change */ -if (prev == next) -return ; - -/* Setup new userspace context */ -get_mmu_context (next); -set_context (next->context.id, next->pgd); -} - -#define deactivate_mm (tsk,mm) do { } while (0) +#include <asm/cputhreads.h> /* -* After we have set current->mm to a new value, this activates -* the context for the new mm so we see the new mappings. +* Most if the context management is out of line */ -#define activate_mm (active_mm, mm) switch_mm(active_mm, mm, current) -extern void mmu_context_init (void ); - - -#else - -#include <linux/kernel.h> -#include <linux/mm.h> -#include <linux/sched.h> - -/* -* Copyright (C) 2001 PPC 64 Team, IBM Corp -* -* This program is free software; you can redistribute it and/or -* modify it under the terms of the GNU General Public License -* as published by the Free Software Foundation; either version -* 2 of the License, or (at your option) any later version. -*/ - -static inline void enter_lazy_tlb (struct mm_struct *mm, -struct task_struct *tsk) -{ -} - -/* -* The proto-VSID space has 2^35 - 1 segments available for user mappings. -* Each segment contains 2^28 bytes. Each context maps 2^44 bytes, -* so we can support 2^19-1 contexts (19 == 35 + 28 - 44). -*/ -#define NO_CONTEXT 0 -#define MAX_CONTEXT ((1UL << 19) - 1) -extern int init_new_context (struct task_struct *tsk,struct mm_struct *mm);extern void destroy_context (struct mm_struct *mm); +extern void switch_mmu_context (struct mm_struct *prev,struct mm_struct *next);extern void switch_stab (struct task_struct *tsk,struct mm_struct *mm);extern void switch_slb (struct task_struct *tsk,struct mm_struct *mm); +extern void set_context (unsigned long id,pgd_t *pgd);/* * switch_mm is the entry point called from the architecture independent
@@ -244,22 +30,39 @@ extern void switch_slb(struct task_strucstatic inline void switch_mm (struct mm_struct *prev,struct mm_struct *next,struct task_struct *tsk) { -if (!cpu_isset (smp_processor_id (), next->cpu_vm_mask)) -cpu_set (smp_processor_id (), next->cpu_vm_mask); +/* Mark this context has been used on the new CPU */ +cpu_set (smp_processor_id (), next->cpu_vm_mask); + +/* 32-bit keeps track of the current PGDIR in the thread struct */ +#ifdef CONFIG_PPC32 + tsk->thread.pgdir = next->pgd; +#endif /* CONFIG_PPC32 */ -/* No need to flush userspace segments if the mm doesnt change */ +/* Nothing else to do if we aren't actually switching */ if (prev == next)return ; +/* We must stop all altivec streams before changing the HW +* context +*/ #ifdef CONFIG_ALTIVEC if (cpu_has_feature (CPU_FTR_ALTIVEC )) asmvolatile ("dssall" );#endif /* CONFIG_ALTIVEC */ +/* The actual HW switching method differs between the various +* sub architectures. +*/ +#ifdef CONFIG_PPC_STD_MMU_64 if (cpu_has_feature (CPU_FTR_SLB ))switch_slb (tsk, next);else switch_stab (tsk, next); +#else +/* Out of line for now */ +switch_mmu_context (prev, next); +#endif + }#define deactivate_mm (tsk,mm) do { } while (0)
@@ -277,6 +80,11 @@ static inline void activate_mm(struct mmlocal_irq_restore (flags); } -#endif /* CONFIG_PPC64 */ +/* We don't currently use enter_lazy_tlb() for anything */ +static inline void enter_lazy_tlb (struct mm_struct *mm, +struct task_struct *tsk) +{ +} +#endif /* __KERNEL__ */ #endif /* __ASM_POWERPC_MMU_CONTEXT_H */
arch/powerpc/kernel/asm-offsets.c +1 -0
@@ -60,6 +60,7 @@ int main(void) {DEFINE (THREAD , offsetof(struct task_struct , thread));DEFINE (MM , offsetof(struct task_struct , mm)); + DEFINE(MMCONTEXTID, offsetof(struct mm_struct, context.id));#ifdef CONFIG_PPC64 DEFINE (AUDITCONTEXT , offsetof(struct task_struct , audit_context));#else
arch/powerpc/kernel/head_32.S +11 -1
@@ -31,6 +31,7 @@ #include <asm/ppc_asm.h> #include <asm/asm-offsets.h> #include <asm/ptrace.h> +#include <asm/bug.h> /* 601 only have IBAT; cr0.eq is set on 601 when using this macro */ #define LOAD_BAT(n, reg, RA, RB) \
@@ -1070,9 +1071,14 @@ start_here: RFI /* + * void switch_mmu_context(struct mm_struct *prev, struct mm_struct *next); + * * Set up the segment registers for a new context. */ -_ENTRY(set_context) +_ENTRY(switch_mmu_context) + lwz r3,MMCONTEXTID(r4) + cmpwi cr0,r3,0 + blt- 4f mulli r3,r3,897 /* multiply context by skew factor */ rlwinm r3,r3,4,8,27 /* VSID = (context & 0xfffff) << 4 */ addis r3,r3,0x6000 /* Set Ks, Ku bits */
@@ -1083,6 +1089,7 @@ _ENTRY(set_context) /* Context switch the PTE pointer for the Abatron BDI2000. * The PGDIR is passed as second argument. */ + lwz r4,MM_PGD(r4) lis r5, KERNELBASE@h lwz r5, 0xf0(r5) stw r4, 0x4(r5)
@@ -1098,6 +1105,9 @@ _ENTRY(set_context) sync isync blr +4: trap + EMIT_BUG_ENTRY 4b,__FILE__,__LINE__,0 + blr /* * An undocumented "feature" of 604e requires that the v bit
arch/powerpc/kernel/ppc_ksyms.c +1 -2
@@ -174,8 +174,7 @@ EXPORT_SYMBOL(cacheable_memcpy);#endif #ifdef CONFIG_PPC32 -EXPORT_SYMBOL (next_mmu_context); -EXPORT_SYMBOL (set_context); +EXPORT_SYMBOL (switch_mmu_context);#endif #ifdef CONFIG_PPC_STD_MMU_32
arch/powerpc/kernel/swsusp.c +1 -1
@@ -34,6 +34,6 @@ void save_processor_state(void)void restore_processor_state (void ) {#ifdef CONFIG_PPC32 -set_context (current->active_mm->context.id, current->active_mm->pgd); +switch_mmu_context (NULL , current->active_mm);#endif }
arch/powerpc/mm/Makefile +4 -3
@@ -8,15 +8,16 @@ endif obj-y := fault.o mem.o pgtable.o \ init_$(CONFIG_WORD_SIZE).o \ - pgtable_$(CONFIG_WORD_SIZE).o \ - mmu_context_$(CONFIG_WORD_SIZE).o + pgtable_$(CONFIG_WORD_SIZE).o +obj-$(CONFIG_PPC_MMU_NOHASH) += mmu_context_nohash.o hash-$(CONFIG_PPC_NATIVE) := hash_native_64.o obj-$(CONFIG_PPC64) += hash_utils_64.o \ slb_low.o slb.o stab.o \ gup.o mmap.o $(hash-y) obj-$(CONFIG_PPC_STD_MMU_32) += ppc_mmu_32.o obj-$(CONFIG_PPC_STD_MMU) += hash_low_$(CONFIG_WORD_SIZE).o \ - tlb_hash$(CONFIG_WORD_SIZE).o + tlb_hash$(CONFIG_WORD_SIZE).o \ + mmu_context_hash$(CONFIG_WORD_SIZE).o obj-$(CONFIG_40x) += 40x_mmu.o obj-$(CONFIG_44x) += 44x_mmu.o obj-$(CONFIG_FSL_BOOKE) += fsl_booke_mmu.o
arch/powerpc/mm/mmu_context_hash32.c new file +103 -0
103 lines not on this page · raw message
arch/powerpc/mm/mmu_context_hash64.c new file +78 -0
78 lines not on this page · raw message
arch/powerpc/mm/mmu_context_nohash.c new file +162 -0
162 lines not on this page · raw message
arch/powerpc/platforms/Kconfig.cputype +9 -1
@@ -195,13 +195,21 @@ config SPE config PPC_STD_MMU bool - depends on 6xx || POWER3 || POWER4 || PPC64 + depends on 6xx || PPC64 default y config PPC_STD_MMU_32 def_bool y depends on PPC_STD_MMU && PPC32 +config PPC_STD_MMU_64 + def_bool y + depends on PPC_STD_MMU && PPC64 + +config PPC_MMU_NOHASH + def_bool y + depends on !PPC_STD_MMU + config PPC_MM_SLICES bool default y if HUGETLB_PAGE || PPC_64K_PAGES
arch/powerpc/platforms/powermac/cpufreq_32.c +1 -1
@@ -310,7 +310,7 @@ static int pmu_set_cpu_speed(int low_spe_set_L3CR (save_l3cr);/* Restore userland MMU context */ -set_context (current->active_mm->context.id, current->active_mm->pgd); +switch_mmu_context (NULL , current->active_mm);#ifdef DEBUG_FREQ printk (KERN_DEBUG "HID1, after: %x\n" ,mfspr (SPRN_HID1 ));
drivers/macintosh/via-pmu.c +2 -2
@@ -1814,7 +1814,7 @@ static int powerbook_sleep_grackle(void)_set_L2CR (save_l2cr);/* Restore userland MMU context */ -set_context (current->active_mm->context.id, current->active_mm->pgd); +switch_mmu_context (NULL , current->active_mm);/* Power things up */ pmu_unlock ();
@@ -1903,7 +1903,7 @@ powerbook_sleep_Core99(void)_set_L3CR (save_l3cr);/* Restore userland MMU context */ -set_context (current->active_mm->context.id, current->active_mm->pgd); +switch_mmu_context (NULL , current->active_mm);/* Tell PMU we are ready */ pmu_unlock ();
arch/powerpc/mm/mmu_context_32.c deleted +0 -84
84 lines not on this page · raw message
arch/powerpc/mm/mmu_context_64.c deleted +0 -70
70 lines not on this page · raw message
PATCH 3/10 powerpc/mm: Rework context management for CPUs with no hash table v2
In reply to Benjamin HerrenschmidtThis reworks the context management code used by 4xx,8xx and freescale BookE. It adds support for SMP by implementing a concept of stale context map to lazily flush the TLB on processors where a context may have been invalidated. This also contains the ground work for generalizing such lazy TLB flushing by just picking up a new PID and marking the old one stale. This will be implemented later.
This is a first implementation that uses a global spinlock.
Ideally, we should try to get at least the fast path (context ID already assigned) lockless or limited to a per context lock, but for now this will do.
I tried to keep the UP case reasonably simple to avoid adding too much overhead to 8xx which does a lot of context stealing since it effectively has only 16 PIDs available.
---
v2. remove some bugs with active tracking on SMP
arch/powerpc/include/asm/mmu-40x.h | 5 arch/powerpc/include/asm/mmu-44x.h | 5 arch/powerpc/include/asm/mmu-8xx.h | 3 arch/powerpc/include/asm/mmu-fsl-booke.h | 5 arch/powerpc/include/asm/tlbflush.h | 2 arch/powerpc/mm/mmu_context_nohash.c | 270 +++++++++++++++++++++++++------ 6 files changed, 235 insertions(+), 55 deletions(-)
arch/powerpc/include/asm/mmu-40x.h +3 -2
@@ -54,8 +54,9 @@#ifndef __ASSEMBLY__typedef struct { -unsigned long id; -unsigned long vdso_base; +unsigned int id; +unsigned int active; +unsigned long vdso_base; }mm_context_t ;#endif /* !__ASSEMBLY__ */
arch/powerpc/include/asm/mmu-44x.h +3 -2
@@ -56,8 +56,9 @@extern unsigned int tlb_44x_hwater;typedef struct { -unsigned long id; -unsigned long vdso_base; +unsigned int id; +unsigned int active; +unsigned long vdso_base; }mm_context_t ;#endif /* !__ASSEMBLY__ */
arch/powerpc/include/asm/mmu-fsl-booke.h +3 -2
@@ -76,8 +76,9 @@#ifndef __ASSEMBLY__typedef struct { -unsigned long id; -unsigned long vdso_base; +unsigned int id; +unsigned int active; +unsigned long vdso_base; }mm_context_t ;#endif /* !__ASSEMBLY__ */
arch/powerpc/mm/mmu_context_nohash.c +222 -48
351 lines not on this page · raw message
arch/powerpc/include/asm/tlbflush.h +2 -0
@@ -29,6 +29,8 @@#include <linux/mm.h> +#define MMU_NO_CONTEXT ((unsigned int)-1) +extern void _tlbie (unsigned long address,unsigned int pid);extern void _tlbil_all (void );extern void _tlbil_pid (unsigned int pid);
arch/powerpc/include/asm/mmu-8xx.h +2 -1
@@ -137,7 +137,8 @@#ifndef __ASSEMBLY__typedef struct { -unsigned long id; +unsigned int id; +unsigned int active;unsigned long vdso_base; }mm_context_t ;#endif /* !__ASSEMBLY__ */
PATCH 4/10 powerpc/mm: Introduce MMU features v3
In reply to Benjamin HerrenschmidtWe're soon running out of CPU features and I need to add some new ones for various MMU related bits, so this patch separates the MMU features from the CPU features. I moved over the 32-bit MMU related ones, added base features for MMU type families, but didn't move over any 64-bit only feature yet.
---
v2. Remove a spurrious character that broke 64-bit build v3. HAS->USE and some comments about individual features
arch/powerpc/include/asm/cputable.h | 85 +++++++++------------- arch/powerpc/include/asm/feature-fixups.h | 30 +++++++ arch/powerpc/include/asm/mmu.h | 41 ++++++++++ arch/powerpc/kernel/cputable.c | 113 ++++++++++++++++++++++++++++++ arch/powerpc/kernel/head_32.S | 8 +- arch/powerpc/kernel/head_fsl_booke.S | 4 - arch/powerpc/kernel/module.c | 6 + arch/powerpc/kernel/setup_32.c | 4 + arch/powerpc/kernel/setup_64.c | 2 arch/powerpc/kernel/swsusp_32.S | 6 - arch/powerpc/kernel/vdso.c | 10 ++ arch/powerpc/kernel/vdso32/vdso32.lds.S | 3 arch/powerpc/kernel/vdso64/vdso64.lds.S | 3 arch/powerpc/kernel/vmlinux.lds.S | 6 + arch/powerpc/mm/ppc_mmu_32.c | 2 arch/powerpc/platforms/powermac/sleep.S | 5 - 16 files changed, 268 insertions(+), 60 deletions(-)
arch/powerpc/include/asm/cputable.h +37 -48
@@ -82,6 +82,7 @@ struct cpu_spec {char *cpu_name;unsigned long cpu_features;/* Kernel features */ unsigned int cpu_user_features;/* Userland features */ +unsigned int mmu_features;/* MMU features */ /* cache line sizes */ unsigned int icache_bsize;
@@ -144,17 +145,14 @@ extern const char *powerpc_base_platform#define CPU_FTR_USE_TB ASM_CONST(0x0000000000000040)#define CPU_FTR_L2CSR ASM_CONST(0x0000000000000080)#define CPU_FTR_601 ASM_CONST(0x0000000000000100) -#define CPU_FTR_HPTE_TABLE ASM_CONST(0x0000000000000200)#define CPU_FTR_CAN_NAP ASM_CONST(0x0000000000000400)#define CPU_FTR_L3CR ASM_CONST(0x0000000000000800)#define CPU_FTR_L3_DISABLE_NAP ASM_CONST(0x0000000000001000)#define CPU_FTR_NAP_DISABLE_L2_PR ASM_CONST(0x0000000000002000)#define CPU_FTR_DUAL_PLL_750FX ASM_CONST(0x0000000000004000)#define CPU_FTR_NO_DPM ASM_CONST(0x0000000000008000) -#define CPU_FTR_HAS_HIGH_BATS ASM_CONST(0x0000000000010000)#define CPU_FTR_NEED_COHERENT ASM_CONST(0x0000000000020000)#define CPU_FTR_NO_BTIC ASM_CONST(0x0000000000040000) -#define CPU_FTR_BIG_PHYS ASM_CONST(0x0000000000080000)#define CPU_FTR_NODSISRALIGN ASM_CONST(0x0000000000100000)#define CPU_FTR_PPC_LE ASM_CONST(0x0000000000200000)#define CPU_FTR_REAL_LE ASM_CONST(0x0000000000400000)
@@ -266,107 +264,99 @@ extern const char *powerpc_base_platform !defined(CONFIG_POWER3) && !defined(CONFIG_POWER4) && \ !defined(CONFIG_BOOKE)) -#define CPU_FTRS_PPC601 (CPU_FTR_COMMON | CPU_FTR_601 | CPU_FTR_HPTE_TABLE | \ +#define CPU_FTRS_PPC601 (CPU_FTR_COMMON | CPU_FTR_601 | \ CPU_FTR_COHERENT_ICACHE | CPU_FTR_UNIFIED_ID_CACHE)#define CPU_FTRS_603 (CPU_FTR_COMMON | \ CPU_FTR_MAYBE_CAN_DOZE | CPU_FTR_USE_TB | \ CPU_FTR_MAYBE_CAN_NAP | CPU_FTR_PPC_LE)#define CPU_FTRS_604 (CPU_FTR_COMMON | \ - CPU_FTR_USE_TB | CPU_FTR_HPTE_TABLE | CPU_FTR_PPC_LE) + CPU_FTR_USE_TB | CPU_FTR_PPC_LE)#define CPU_FTRS_740_NOTAU (CPU_FTR_COMMON | \ CPU_FTR_MAYBE_CAN_DOZE | CPU_FTR_USE_TB | CPU_FTR_L2CR | \ - CPU_FTR_HPTE_TABLE | CPU_FTR_MAYBE_CAN_NAP | CPU_FTR_PPC_LE) + CPU_FTR_MAYBE_CAN_NAP | CPU_FTR_PPC_LE)#define CPU_FTRS_740 (CPU_FTR_COMMON | \ CPU_FTR_MAYBE_CAN_DOZE | CPU_FTR_USE_TB | CPU_FTR_L2CR | \ - CPU_FTR_TAU | CPU_FTR_HPTE_TABLE | CPU_FTR_MAYBE_CAN_NAP | \ + CPU_FTR_TAU | CPU_FTR_MAYBE_CAN_NAP | \ CPU_FTR_PPC_LE)#define CPU_FTRS_750 (CPU_FTR_COMMON | \ CPU_FTR_MAYBE_CAN_DOZE | CPU_FTR_USE_TB | CPU_FTR_L2CR | \ - CPU_FTR_TAU | CPU_FTR_HPTE_TABLE | CPU_FTR_MAYBE_CAN_NAP | \ + CPU_FTR_TAU | CPU_FTR_MAYBE_CAN_NAP | \ CPU_FTR_PPC_LE) -#define CPU_FTRS_750CL (CPU_FTRS_750 | CPU_FTR_HAS_HIGH_BATS) +#define CPU_FTRS_750CL (CPU_FTRS_750)#define CPU_FTRS_750FX1 (CPU_FTRS_750 | CPU_FTR_DUAL_PLL_750FX | CPU_FTR_NO_DPM)#define CPU_FTRS_750FX2 (CPU_FTRS_750 | CPU_FTR_NO_DPM) -#define CPU_FTRS_750FX (CPU_FTRS_750 | CPU_FTR_DUAL_PLL_750FX | \ - CPU_FTR_HAS_HIGH_BATS) +#define CPU_FTRS_750FX (CPU_FTRS_750 | CPU_FTR_DUAL_PLL_750FX)#define CPU_FTRS_750GX (CPU_FTRS_750FX)#define CPU_FTRS_7400_NOTAU (CPU_FTR_COMMON | \ CPU_FTR_MAYBE_CAN_DOZE | CPU_FTR_USE_TB | CPU_FTR_L2CR | \ - CPU_FTR_ALTIVEC_COMP | CPU_FTR_HPTE_TABLE | \ + CPU_FTR_ALTIVEC_COMP | \ CPU_FTR_MAYBE_CAN_NAP | CPU_FTR_PPC_LE)#define CPU_FTRS_7400 (CPU_FTR_COMMON | \ CPU_FTR_MAYBE_CAN_DOZE | CPU_FTR_USE_TB | CPU_FTR_L2CR | \ - CPU_FTR_TAU | CPU_FTR_ALTIVEC_COMP | CPU_FTR_HPTE_TABLE | \ + CPU_FTR_TAU | CPU_FTR_ALTIVEC_COMP | \ CPU_FTR_MAYBE_CAN_NAP | CPU_FTR_PPC_LE)#define CPU_FTRS_7450_20 (CPU_FTR_COMMON | \ CPU_FTR_USE_TB | CPU_FTR_L2CR | CPU_FTR_ALTIVEC_COMP | \ - CPU_FTR_L3CR | CPU_FTR_HPTE_TABLE | CPU_FTR_SPEC7450 | \ + CPU_FTR_L3CR | CPU_FTR_SPEC7450 | \ CPU_FTR_NEED_COHERENT | CPU_FTR_PPC_LE | CPU_FTR_NEED_PAIRED_STWCX)#define CPU_FTRS_7450_21 (CPU_FTR_COMMON | \ CPU_FTR_USE_TB | \ CPU_FTR_MAYBE_CAN_NAP | CPU_FTR_L2CR | CPU_FTR_ALTIVEC_COMP | \ - CPU_FTR_L3CR | CPU_FTR_HPTE_TABLE | CPU_FTR_SPEC7450 | \ + CPU_FTR_L3CR | CPU_FTR_SPEC7450 | \ CPU_FTR_NAP_DISABLE_L2_PR | CPU_FTR_L3_DISABLE_NAP | \ CPU_FTR_NEED_COHERENT | CPU_FTR_PPC_LE | CPU_FTR_NEED_PAIRED_STWCX)#define CPU_FTRS_7450_23 (CPU_FTR_COMMON | \ CPU_FTR_USE_TB | CPU_FTR_NEED_PAIRED_STWCX | \ CPU_FTR_MAYBE_CAN_NAP | CPU_FTR_L2CR | CPU_FTR_ALTIVEC_COMP | \ - CPU_FTR_L3CR | CPU_FTR_HPTE_TABLE | CPU_FTR_SPEC7450 | \ + CPU_FTR_L3CR | CPU_FTR_SPEC7450 | \ CPU_FTR_NAP_DISABLE_L2_PR | CPU_FTR_NEED_COHERENT | CPU_FTR_PPC_LE)#define CPU_FTRS_7455_1 (CPU_FTR_COMMON | \ CPU_FTR_USE_TB | CPU_FTR_NEED_PAIRED_STWCX | \ CPU_FTR_L2CR | CPU_FTR_ALTIVEC_COMP | CPU_FTR_L3CR | \ - CPU_FTR_HPTE_TABLE | CPU_FTR_SPEC7450 | CPU_FTR_HAS_HIGH_BATS | \ - CPU_FTR_NEED_COHERENT | CPU_FTR_PPC_LE) + CPU_FTR_SPEC7450 | CPU_FTR_NEED_COHERENT | CPU_FTR_PPC_LE)#define CPU_FTRS_7455_20 (CPU_FTR_COMMON | \ CPU_FTR_USE_TB | CPU_FTR_NEED_PAIRED_STWCX | \ CPU_FTR_MAYBE_CAN_NAP | CPU_FTR_L2CR | CPU_FTR_ALTIVEC_COMP | \ - CPU_FTR_L3CR | CPU_FTR_HPTE_TABLE | CPU_FTR_SPEC7450 | \ + CPU_FTR_L3CR | CPU_FTR_SPEC7450 | \ CPU_FTR_NAP_DISABLE_L2_PR | CPU_FTR_L3_DISABLE_NAP | \ - CPU_FTR_NEED_COHERENT | CPU_FTR_HAS_HIGH_BATS | CPU_FTR_PPC_LE) + CPU_FTR_NEED_COHERENT | CPU_FTR_PPC_LE)#define CPU_FTRS_7455 (CPU_FTR_COMMON | \ CPU_FTR_USE_TB | \ CPU_FTR_MAYBE_CAN_NAP | CPU_FTR_L2CR | CPU_FTR_ALTIVEC_COMP | \ - CPU_FTR_L3CR | CPU_FTR_HPTE_TABLE | CPU_FTR_SPEC7450 | \ - CPU_FTR_NAP_DISABLE_L2_PR | CPU_FTR_HAS_HIGH_BATS | \ + CPU_FTR_L3CR | CPU_FTR_SPEC7450 | CPU_FTR_NAP_DISABLE_L2_PR | \ CPU_FTR_NEED_COHERENT | CPU_FTR_PPC_LE | CPU_FTR_NEED_PAIRED_STWCX)#define CPU_FTRS_7447_10 (CPU_FTR_COMMON | \ CPU_FTR_USE_TB | \ CPU_FTR_MAYBE_CAN_NAP | CPU_FTR_L2CR | CPU_FTR_ALTIVEC_COMP | \ - CPU_FTR_L3CR | CPU_FTR_HPTE_TABLE | CPU_FTR_SPEC7450 | \ - CPU_FTR_NAP_DISABLE_L2_PR | CPU_FTR_HAS_HIGH_BATS | \ + CPU_FTR_L3CR | CPU_FTR_SPEC7450 | CPU_FTR_NAP_DISABLE_L2_PR | \ CPU_FTR_NEED_COHERENT | CPU_FTR_NO_BTIC | CPU_FTR_PPC_LE | \ CPU_FTR_NEED_PAIRED_STWCX)#define CPU_FTRS_7447 (CPU_FTR_COMMON | \ CPU_FTR_USE_TB | \ CPU_FTR_MAYBE_CAN_NAP | CPU_FTR_L2CR | CPU_FTR_ALTIVEC_COMP | \ - CPU_FTR_L3CR | CPU_FTR_HPTE_TABLE | CPU_FTR_SPEC7450 | \ - CPU_FTR_NAP_DISABLE_L2_PR | CPU_FTR_HAS_HIGH_BATS | \ + CPU_FTR_L3CR | CPU_FTR_SPEC7450 | CPU_FTR_NAP_DISABLE_L2_PR | \ CPU_FTR_NEED_COHERENT | CPU_FTR_PPC_LE | CPU_FTR_NEED_PAIRED_STWCX)#define CPU_FTRS_7447A (CPU_FTR_COMMON | \ CPU_FTR_USE_TB | \ CPU_FTR_MAYBE_CAN_NAP | CPU_FTR_L2CR | CPU_FTR_ALTIVEC_COMP | \ - CPU_FTR_HPTE_TABLE | CPU_FTR_SPEC7450 | \ - CPU_FTR_NAP_DISABLE_L2_PR | CPU_FTR_HAS_HIGH_BATS | \ + CPU_FTR_SPEC7450 | CPU_FTR_NAP_DISABLE_L2_PR | \ CPU_FTR_NEED_COHERENT | CPU_FTR_PPC_LE | CPU_FTR_NEED_PAIRED_STWCX)#define CPU_FTRS_7448 (CPU_FTR_COMMON | \ CPU_FTR_USE_TB | \ CPU_FTR_MAYBE_CAN_NAP | CPU_FTR_L2CR | CPU_FTR_ALTIVEC_COMP | \ - CPU_FTR_HPTE_TABLE | CPU_FTR_SPEC7450 | \ - CPU_FTR_NAP_DISABLE_L2_PR | CPU_FTR_HAS_HIGH_BATS | \ + CPU_FTR_SPEC7450 | CPU_FTR_NAP_DISABLE_L2_PR | \ CPU_FTR_PPC_LE | CPU_FTR_NEED_PAIRED_STWCX)#define CPU_FTRS_82XX (CPU_FTR_COMMON | \ CPU_FTR_MAYBE_CAN_DOZE | CPU_FTR_USE_TB)#define CPU_FTRS_G2_LE (CPU_FTR_COMMON | CPU_FTR_MAYBE_CAN_DOZE | \ - CPU_FTR_USE_TB | CPU_FTR_MAYBE_CAN_NAP | CPU_FTR_HAS_HIGH_BATS) + CPU_FTR_USE_TB | CPU_FTR_MAYBE_CAN_NAP)#define CPU_FTRS_E300 (CPU_FTR_MAYBE_CAN_DOZE | \ - CPU_FTR_USE_TB | CPU_FTR_MAYBE_CAN_NAP | CPU_FTR_HAS_HIGH_BATS | \ + CPU_FTR_USE_TB | CPU_FTR_MAYBE_CAN_NAP | \ CPU_FTR_COMMON)#define CPU_FTRS_E300C2 (CPU_FTR_MAYBE_CAN_DOZE | \ - CPU_FTR_USE_TB | CPU_FTR_MAYBE_CAN_NAP | CPU_FTR_HAS_HIGH_BATS | \ + CPU_FTR_USE_TB | CPU_FTR_MAYBE_CAN_NAP | \ CPU_FTR_COMMON | CPU_FTR_FPU_UNAVAILABLE) -#define CPU_FTRS_CLASSIC32 (CPU_FTR_COMMON | \ - CPU_FTR_USE_TB | CPU_FTR_HPTE_TABLE) +#define CPU_FTRS_CLASSIC32 (CPU_FTR_COMMON | CPU_FTR_USE_TB)#define CPU_FTRS_8XX (CPU_FTR_USE_TB)#define CPU_FTRS_40X (CPU_FTR_USE_TB | CPU_FTR_NODSISRALIGN | CPU_FTR_NOEXECUTE)#define CPU_FTRS_44X (CPU_FTR_USE_TB | CPU_FTR_NODSISRALIGN | CPU_FTR_NOEXECUTE)
@@ -379,55 +369,54 @@ extern const char *powerpc_base_platform CPU_FTR_SPE_COMP | CPU_FTR_MAYBE_CAN_NAP | CPU_FTR_NODSISRALIGN | \ CPU_FTR_NOEXECUTE)#define CPU_FTRS_E500_2 (CPU_FTR_MAYBE_CAN_DOZE | CPU_FTR_USE_TB | \ - CPU_FTR_SPE_COMP | CPU_FTR_MAYBE_CAN_NAP | CPU_FTR_BIG_PHYS | \ + CPU_FTR_SPE_COMP | CPU_FTR_MAYBE_CAN_NAP | \ CPU_FTR_NODSISRALIGN | CPU_FTR_NOEXECUTE)#define CPU_FTRS_E500MC (CPU_FTR_MAYBE_CAN_DOZE | CPU_FTR_USE_TB | \ - CPU_FTR_MAYBE_CAN_NAP | CPU_FTR_BIG_PHYS | CPU_FTR_NODSISRALIGN | \ + CPU_FTR_MAYBE_CAN_NAP | CPU_FTR_NODSISRALIGN | \ CPU_FTR_L2CSR | CPU_FTR_LWSYNC | CPU_FTR_NOEXECUTE)#define CPU_FTRS_GENERIC_32 (CPU_FTR_COMMON | CPU_FTR_NODSISRALIGN)/* 64-bit CPUs */ #define CPU_FTRS_POWER3 (CPU_FTR_USE_TB | CPU_FTR_LWSYNC | \ - CPU_FTR_HPTE_TABLE | CPU_FTR_IABR | CPU_FTR_PPC_LE) + CPU_FTR_IABR | CPU_FTR_PPC_LE)#define CPU_FTRS_RS64 (CPU_FTR_USE_TB | CPU_FTR_LWSYNC | \ - CPU_FTR_HPTE_TABLE | CPU_FTR_IABR | \ + CPU_FTR_IABR | \ CPU_FTR_MMCRA | CPU_FTR_CTRL)#define CPU_FTRS_POWER4 (CPU_FTR_USE_TB | CPU_FTR_LWSYNC | \ - CPU_FTR_HPTE_TABLE | CPU_FTR_PPCAS_ARCH_V2 | CPU_FTR_CTRL | \ + CPU_FTR_PPCAS_ARCH_V2 | CPU_FTR_CTRL | \ CPU_FTR_MMCRA | CPU_FTR_CP_USE_DCBTZ)#define CPU_FTRS_PPC970 (CPU_FTR_USE_TB | CPU_FTR_LWSYNC | \ - CPU_FTR_HPTE_TABLE | CPU_FTR_PPCAS_ARCH_V2 | CPU_FTR_CTRL | \ + CPU_FTR_PPCAS_ARCH_V2 | CPU_FTR_CTRL | \ CPU_FTR_ALTIVEC_COMP | CPU_FTR_CAN_NAP | CPU_FTR_MMCRA | \ CPU_FTR_CP_USE_DCBTZ)#define CPU_FTRS_POWER5 (CPU_FTR_USE_TB | CPU_FTR_LWSYNC | \ - CPU_FTR_HPTE_TABLE | CPU_FTR_PPCAS_ARCH_V2 | CPU_FTR_CTRL | \ + CPU_FTR_PPCAS_ARCH_V2 | CPU_FTR_CTRL | \ CPU_FTR_MMCRA | CPU_FTR_SMT | \ CPU_FTR_COHERENT_ICACHE | CPU_FTR_LOCKLESS_TLBIE | \ CPU_FTR_PURR)#define CPU_FTRS_POWER6 (CPU_FTR_USE_TB | CPU_FTR_LWSYNC | \ - CPU_FTR_HPTE_TABLE | CPU_FTR_PPCAS_ARCH_V2 | CPU_FTR_CTRL | \ + CPU_FTR_PPCAS_ARCH_V2 | CPU_FTR_CTRL | \ CPU_FTR_MMCRA | CPU_FTR_SMT | \ CPU_FTR_COHERENT_ICACHE | CPU_FTR_LOCKLESS_TLBIE | \ CPU_FTR_PURR | CPU_FTR_SPURR | CPU_FTR_REAL_LE | \ CPU_FTR_DSCR | CPU_FTR_UNALIGNED_LD_STD)#define CPU_FTRS_POWER7 (CPU_FTR_USE_TB | CPU_FTR_LWSYNC | \ - CPU_FTR_HPTE_TABLE | CPU_FTR_PPCAS_ARCH_V2 | CPU_FTR_CTRL | \ + CPU_FTR_PPCAS_ARCH_V2 | CPU_FTR_CTRL | \ CPU_FTR_MMCRA | CPU_FTR_SMT | \ CPU_FTR_COHERENT_ICACHE | CPU_FTR_LOCKLESS_TLBIE | \ CPU_FTR_PURR | CPU_FTR_SPURR | CPU_FTR_REAL_LE | \ CPU_FTR_DSCR | CPU_FTR_SAO)#define CPU_FTRS_CELL (CPU_FTR_USE_TB | CPU_FTR_LWSYNC | \ - CPU_FTR_HPTE_TABLE | CPU_FTR_PPCAS_ARCH_V2 | CPU_FTR_CTRL | \ + CPU_FTR_PPCAS_ARCH_V2 | CPU_FTR_CTRL | \ CPU_FTR_ALTIVEC_COMP | CPU_FTR_MMCRA | CPU_FTR_SMT | \ CPU_FTR_PAUSE_ZERO | CPU_FTR_CI_LARGE_PAGE | \ CPU_FTR_CELL_TB_BUG | CPU_FTR_CP_USE_DCBTZ | \ CPU_FTR_UNALIGNED_LD_STD)#define CPU_FTRS_PA6T (CPU_FTR_USE_TB | CPU_FTR_LWSYNC | \ - CPU_FTR_HPTE_TABLE | CPU_FTR_PPCAS_ARCH_V2 | \ + CPU_FTR_PPCAS_ARCH_V2 | \ CPU_FTR_ALTIVEC_COMP | CPU_FTR_CI_LARGE_PAGE | \ CPU_FTR_PURR | CPU_FTR_REAL_LE | CPU_FTR_NO_SLBIE_B) -#define CPU_FTRS_COMPATIBLE (CPU_FTR_USE_TB | \ - CPU_FTR_HPTE_TABLE | CPU_FTR_PPCAS_ARCH_V2) +#define CPU_FTRS_COMPATIBLE (CPU_FTR_USE_TB | CPU_FTR_PPCAS_ARCH_V2)#ifdef __powerpc64__#define CPU_FTRS_POSSIBLE \
arch/powerpc/include/asm/feature-fixups.h +30 -0
@@ -81,6 +81,36 @@ label##5: \#define ALT_FTR_SECTION_END_IFCLR (msk) \ ALT_FTR_SECTION_END_NESTED_IFCLR(msk, 97) +/* MMU feature dependent sections */ +#define BEGIN_MMU_FTR_SECTION_NESTED (label) START_FTR_SECTION(label) +#define BEGIN_MMU_FTR_SECTION START_FTR_SECTION(97) + +#define END_MMU_FTR_SECTION_NESTED (msk, val, label) \ + FTR_SECTION_ELSE_NESTED(label) \ + MAKE_FTR_SECTION_ENTRY(msk, val, label, __mmu_ftr_fixup) + +#define END_MMU_FTR_SECTION (msk, val) \ + END_MMU_FTR_SECTION_NESTED(msk, val, 97) + +#define END_MMU_FTR_SECTION_IFSET (msk) END_MMU_FTR_SECTION((msk), (msk)) +#define END_MMU_FTR_SECTION_IFCLR (msk) END_MMU_FTR_SECTION((msk), 0) + +/* MMU feature sections with alternatives, use BEGIN_FTR_SECTION to start */ +#define MMU_FTR_SECTION_ELSE_NESTED (label) FTR_SECTION_ELSE_NESTED(label) +#define MMU_FTR_SECTION_ELSE MMU_FTR_SECTION_ELSE_NESTED(97) +#define ALT_MMU_FTR_SECTION_END_NESTED (msk, val, label) \ + MAKE_FTR_SECTION_ENTRY(msk, val, label, __mmu_ftr_fixup) +#define ALT_MMU_FTR_SECTION_END_NESTED_IFSET (msk, label) \ + ALT_MMU_FTR_SECTION_END_NESTED(msk, msk, label) +#define ALT_MMU_FTR_SECTION_END_NESTED_IFCLR (msk, label) \ + ALT_MMU_FTR_SECTION_END_NESTED(msk, 0, label) +#define ALT_MMU_FTR_SECTION_END (msk, val) \ + ALT_MMU_FTR_SECTION_END_NESTED(msk, val, 97) +#define ALT_MMU_FTR_SECTION_END_IFSET (msk) \ + ALT_MMU_FTR_SECTION_END_NESTED_IFSET(msk, 97) +#define ALT_MMU_FTR_SECTION_END_IFCLR (msk) \ + ALT_MMU_FTR_SECTION_END_NESTED_IFCLR(msk, 97) +/* Firmware feature dependent sections */ #define BEGIN_FW_FTR_SECTION_NESTED (label) START_FTR_SECTION(label)#define BEGIN_FW_FTR_SECTION START_FTR_SECTION(97)
arch/powerpc/include/asm/mmu.h +41 -0
47 lines not on this page · raw message
arch/powerpc/kernel/cputable.c +113 -0
791 lines not on this page · raw message
arch/powerpc/kernel/module.c +6 -0
@@ -78,6 +78,12 @@ int module_finalize(const Elf_Ehdr *hdr, (void *)sect->sh_addr, (void *)sect->sh_addr + sect->sh_size); + sect =find_section (hdr, sechdrs,"__mmu_ftr_fixup" ); +if (sect !=NULL ) +do_feature_fixups (cur_cpu_spec->mmu_features, + (void *)sect->sh_addr, + (void *)sect->sh_addr + sect->sh_size); +#ifdef CONFIG_PPC64 sect =find_section (hdr, sechdrs,"__fw_ftr_fixup" );if (sect !=NULL )
arch/powerpc/kernel/setup_32.c +4 -0
@@ -97,6 +97,10 @@ notrace unsigned long __init early_init(PTRRELOC (&__start___ftr_fixup), PTRRELOC(&__stop___ftr_fixup)); +do_feature_fixups (spec->mmu_features, +PTRRELOC (&__start___mmu_ftr_fixup), +PTRRELOC (&__stop___mmu_ftr_fixup)); +do_lwsync_fixups (spec->cpu_features,PTRRELOC (&__start___lwsync_fixup),PTRRELOC (&__stop___lwsync_fixup));
arch/powerpc/kernel/setup_64.c +2 -0
@@ -361,6 +361,8 @@ void __init setup_system(void)*/ do_feature_fixups (cur_cpu_spec->cpu_features, &__start___ftr_fixup, &__stop___ftr_fixup); +do_feature_fixups (cur_cpu_spec->mmu_features, + &__start___mmu_ftr_fixup, &__stop___mmu_ftr_fixup);do_feature_fixups (powerpc_firmware_features, &__start___fw_ftr_fixup, &__stop___fw_ftr_fixup); do_lwsync_fixups(cur_cpu_spec->cpu_features,
arch/powerpc/kernel/vdso.c +10 -0
@@ -567,6 +567,11 @@ static __init int vdso_fixup_features(stdo_feature_fixups (cur_cpu_spec->cpu_features, start64, start64 + size64); + start64 =find_section64 (v64->hdr,"__mmu_ftr_fixup" , &size64); +if (start64) +do_feature_fixups (cur_cpu_spec->mmu_features, + start64, start64 + size64); + start64 =find_section64 (v64->hdr,"__fw_ftr_fixup" , &size64);if (start64) do_feature_fixups(powerpc_firmware_features,
@@ -583,6 +588,11 @@ static __init int vdso_fixup_features(stdo_feature_fixups (cur_cpu_spec->cpu_features, start32, start32 + size32); + start32 =find_section32 (v32->hdr,"__mmu_ftr_fixup" , &size32); +if (start32) +do_feature_fixups (cur_cpu_spec->mmu_features, + start32, start32 + size32); +#ifdef CONFIG_PPC64 start32 =find_section32 (v32->hdr,"__fw_ftr_fixup" , &size32);if (start32)
arch/powerpc/kernel/vdso32/vdso32.lds.S +3 -0
@@ -34,6 +34,9 @@ SECTIONS __ftr_fixup : { *(__ftr_fixup) } . = ALIGN(8); + __mmu_ftr_fixup : { *(__mmu_ftr_fixup) } + + . = ALIGN(8); __lwsync_fixup : { *(__lwsync_fixup) } #ifdef CONFIG_PPC64
arch/powerpc/kernel/vdso64/vdso64.lds.S +3 -0
@@ -35,6 +35,9 @@ SECTIONS __ftr_fixup : { *(__ftr_fixup) } . = ALIGN(8); + __mmu_ftr_fixup : { *(__mmu_ftr_fixup) } + + . = ALIGN(8); __lwsync_fixup : { *(__lwsync_fixup) } . = ALIGN(8);
arch/powerpc/kernel/vmlinux.lds.S +6 -0
@@ -152,6 +152,12 @@ SECTIONS __stop___ftr_fixup = .; } . = ALIGN(8); + __mmu_ftr_fixup : AT(ADDR(__mmu_ftr_fixup) - LOAD_OFFSET) { + __start___mmu_ftr_fixup = .; + *(__mmu_ftr_fixup) + __stop___mmu_ftr_fixup = .; + } + . = ALIGN(8); __lwsync_fixup : AT(ADDR(__lwsync_fixup) - LOAD_OFFSET) { __start___lwsync_fixup = .; *(__lwsync_fixup)
arch/powerpc/mm/ppc_mmu_32.c +1 -1
@@ -192,7 +192,7 @@ void __init MMU_init_hw(void)extern unsigned int hash_page[];extern unsigned int flush_hash_patch_A[], flush_hash_patch_B[]; -if (!cpu_has_feature (CPU_FTR_HPTE_TABLE )) { +if (!mmu_has_feature (MMU_FTR_HPTE_TABLE )) {/* * Put a blr (procedure return) instruction at the * start of hash_page, since we can still get DSI
arch/powerpc/kernel/head_32.S +4 -4
@@ -990,12 +990,12 @@ load_up_mmu: LOAD_BAT(1,r3,r4,r5) LOAD_BAT(2,r3,r4,r5) LOAD_BAT(3,r3,r4,r5) -BEGIN_FTR_SECTION +BEGIN_MMU_FTR_SECTION LOAD_BAT(4,r3,r4,r5) LOAD_BAT(5,r3,r4,r5) LOAD_BAT(6,r3,r4,r5) LOAD_BAT(7,r3,r4,r5) -END_FTR_SECTION_IFSET(CPU_FTR_HAS_HIGH_BATS) +END_MMU_FTR_SECTION_IFSET(MMU_FTR_USE_HIGH_BATS) blr /*
@@ -1141,7 +1141,7 @@ clear_bats: mtspr SPRN_IBAT2L,r10 mtspr SPRN_IBAT3U,r10 mtspr SPRN_IBAT3L,r10 -BEGIN_FTR_SECTION +BEGIN_MMU_FTR_SECTION /* Here's a tweak: at this point, CPU setup have * not been called yet, so HIGH_BAT_EN may not be * set in HID0 for the 745x processors. However, it
@@ -1164,7 +1164,7 @@ BEGIN_FTR_SECTION mtspr SPRN_IBAT6L,r10 mtspr SPRN_IBAT7U,r10 mtspr SPRN_IBAT7L,r10 -END_FTR_SECTION_IFSET(CPU_FTR_HAS_HIGH_BATS) +END_MMU_FTR_SECTION_IFSET(MMU_FTR_USE_HIGH_BATS) blr flush_tlbs:
arch/powerpc/kernel/swsusp_32.S +3 -3
@@ -5,7 +5,7 @@ #include <asm/thread_info.h> #include <asm/ppc_asm.h> #include <asm/asm-offsets.h> - +#include <asm/mmu.h> /* * Structure for storing CPU registers on the save area.
@@ -279,7 +279,7 @@ END_FTR_SECTION_IFSET(CPU_FTR_ALTIVEC) mtibatl 3,r4 #endif -BEGIN_FTR_SECTION +BEGIN_MMU_FTR_SECTION li r4,0 mtspr SPRN_DBAT4U,r4 mtspr SPRN_DBAT4L,r4
@@ -297,7 +297,7 @@ BEGIN_FTR_SECTION mtspr SPRN_IBAT6L,r4 mtspr SPRN_IBAT7U,r4 mtspr SPRN_IBAT7L,r4 -END_FTR_SECTION_IFSET(CPU_FTR_HAS_HIGH_BATS) +END_MMU_FTR_SECTION_IFSET(MMU_FTR_USE_HIGH_BATS) /* Flush all TLBs */ lis r4,0x1000
arch/powerpc/platforms/powermac/sleep.S +3 -2
@@ -17,6 +17,7 @@ #include <asm/cache.h> #include <asm/thread_info.h> #include <asm/asm-offsets.h> +#include <asm/mmu.h> #define MAGIC 0x4c617273 /* 'Lars' */
@@ -323,7 +324,7 @@ grackle_wake_up: lwz r4,SL_IBAT3+4(r1) mtibatl 3,r4 -BEGIN_FTR_SECTION +BEGIN_MMU_FTR_SECTION li r4,0 mtspr SPRN_DBAT4U,r4 mtspr SPRN_DBAT4L,r4
@@ -341,7 +342,7 @@ BEGIN_FTR_SECTION mtspr SPRN_IBAT6L,r4 mtspr SPRN_IBAT7U,r4 mtspr SPRN_IBAT7L,r4 -END_FTR_SECTION_IFSET(CPU_FTR_HAS_HIGH_BATS) +END_MMU_FTR_SECTION_IFSET(MMU_FTR_USE_HIGH_BATS) /* Flush all TLBs */ lis r4,0x1000
arch/powerpc/kernel/head_fsl_booke.S +2 -2
@@ -767,10 +767,10 @@ finish_tlb_load: rlwimi r12, r13, 24, 0, 7 /* grab RPN[32:39] */ rlwimi r12, r11, 24, 8, 19 /* grab RPN[40:51] */ mtspr SPRN_MAS3, r12 -BEGIN_FTR_SECTION +BEGIN_MMU_FTR_SECTION srwi r10, r13, 8 /* grab RPN[8:31] */ mtspr SPRN_MAS7, r10 -END_FTR_SECTION_IFSET(CPU_FTR_BIG_PHYS) +END_MMU_FTR_SECTION_IFSET(MMU_FTR_BIG_PHYS) #else rlwimi r11, r12, 0, 20, 31 /* Extract RPN from PTE and merge with perms */ mtspr SPRN_MAS3, r11
PATCH 5/10 powerpc/mm: Add SMP support to no-hash TLB handling v5
In reply to Benjamin HerrenschmidtThis patch moves the whole no-hash TLB handling out of line into a new tlb_nohash.c file, and implements some basic SMP support using IPIs and/or broadcast tlbivax instructions.
Note that I'm using local invalidations for D->I cache coherency.
At worst, if another processor is trying to execute the same and has the old entry in its TLB, it will just take a fault and re-do the TLB flush locally (it won't re-do the cache flush in any case).
---
v2. This variant fixes usage of linux/spinlock.h instead of asm/spinlock.h v3. Invadvertently un-EXPORT_SYMBOL'ed some cache flush calls on ppc64 v4. Fix differences in local_* flush variants between CPU types and
corresponding clash with highmem code. Remove remaining _tlbie calls from nohash code.
v5. HAS->USE and some comments on MMU features
arch/powerpc/include/asm/highmem.h | 4 arch/powerpc/include/asm/mmu.h | 16 ++ arch/powerpc/include/asm/tlbflush.h | 84 ++++++-------- arch/powerpc/kernel/misc_32.S | 9 + arch/powerpc/kernel/ppc_ksyms.c | 6 - arch/powerpc/mm/Makefile | 2 arch/powerpc/mm/fault.c | 2 arch/powerpc/mm/mem.c | 2 arch/powerpc/mm/tlb_hash32.c | 4 arch/powerpc/mm/tlb_nohash.c | 209 ++++++++++++++++++++++++++++++++++++ 10 files changed, 281 insertions(+), 57 deletions(-)
arch/powerpc/include/asm/tlbflush.h +38 -46
@@ -6,7 +6,9 @@* * - flush_tlb_mm(mm) flushes the specified mm context TLB's * - flush_tlb_page(vma, vmaddr) flushes one page -* - local_flush_tlb_page(vmaddr) flushes one page on the local processor +* - local_flush_tlb_mm(mm) flushes the specified mm context on +* the local processor +* - local_flush_tlb_page(vma, vmaddr) flushes one page on the local processor * - flush_tlb_page_nohash(vma, vmaddr) flushes one page if SW loaded TLB * - flush_tlb_range(vma, start, end) flushes a range of pages * - flush_tlb_kernel_range(start, end) flushes a range of kernel pages
@@ -18,7 +20,7 @@*/ #ifdef __KERNEL__ -#if defined(CONFIG_4xx) || defined(CONFIG_8xx) || defined(CONFIG_FSL_BOOKE ) +#ifdef CONFIG_PPC_MMU_NOHASH /* * TLB flushing for software loaded TLB chips *
@@ -31,10 +33,10 @@#define MMU_NO_CONTEXT ((unsigned int)-1) -extern void _tlbie (unsigned long address,unsigned int pid);extern void _tlbil_all (void );extern void _tlbil_pid (unsigned int pid);extern void _tlbil_va (unsigned long address,unsigned int pid); +extern void _tlbivax_bcast (unsigned long address,unsigned int pid);#if defined(CONFIG_40x) || defined(CONFIG_8xx)#define _tlbia () asm volatile ("tlbia; sync" : : : "memory")
@@ -42,48 +44,26 @@ extern void _tlbil_va(unsigned long addrextern void _tlbia (void );#endif -static inline void local_flush_tlb_mm (struct mm_struct *mm) -{ -_tlbil_pid (mm->context.id); -} - -static inline void flush_tlb_mm (struct mm_struct *mm) -{ -_tlbil_pid (mm->context.id); -} - -static inline void local_flush_tlb_page (unsigned long vmaddr) -{ -_tlbil_va (vmaddr,0 ); -} - -static inline void flush_tlb_page (struct vm_area_struct *vma, -unsigned long vmaddr) -{ -_tlbil_va (vmaddr, vma ? vma->vm_mm->context.id :0 ); -} +extern void flush_tlb_range (struct vm_area_struct *vma,unsigned long start, +unsigned long end); +extern void flush_tlb_kernel_range (unsigned long start,unsigned long end); -static inline void flush_tlb_page_nohash (struct vm_area_struct *vma, -unsigned long vmaddr) -{ -flush_tlb_page (vma, vmaddr); -} +extern void local_flush_tlb_mm (struct mm_struct *mm); +extern void local_flush_tlb_page (struct vm_area_struct *vma,unsigned long vmaddr); -static inline void flush_tlb_range (struct vm_area_struct *vma, -unsigned long start,unsigned long end) -{ -_tlbil_pid (vma->vm_mm->context.id); -} +#ifdef CONFIG_SMP +extern void flush_tlb_mm (struct mm_struct *mm); +extern void flush_tlb_page (struct vm_area_struct *vma,unsigned long vmaddr); +#else +#define flush_tlb_mm (mm) local_flush_tlb_mm(mm) +#define flush_tlb_page (vma,addr) local_flush_tlb_page(vma,addr) +#endif +#define flush_tlb_page_nohash (vma,addr) flush_tlb_page(vma,addr) -static inline void flush_tlb_kernel_range (unsigned long start, -unsigned long end) -{ -_tlbil_pid (0 ); -} +#elif defined(CONFIG_PPC_STD_MMU_32) -#elif defined(CONFIG_PPC32)/* -* TLB flushing for "classic" hash-MMMU 32-bit CPUs, 6xx, 7xx, 7xxx +* TLB flushing for "classic" hash-MMU 32-bit CPUs, 6xx, 7xx, 7xxx */ extern void _tlbie (unsigned long address);extern void _tlbia (void );
@@ -94,14 +74,20 @@ extern void flush_tlb_page_nohash(structextern void flush_tlb_range (struct vm_area_struct *vma,unsigned long start,unsigned long end);extern void flush_tlb_kernel_range (unsigned long start,unsigned long end); -static inline void local_flush_tlb_page (unsigned long vmaddr) +static inline void local_flush_tlb_page (struct vm_area_struct *vma, +unsigned long vmaddr) { -flush_tlb_page (NULL , vmaddr); +flush_tlb_page (vma, vmaddr); +} +static inline void local_flush_tlb_mm (struct mm_struct *mm) +{ +flush_tlb_mm (mm); } -#else +#elif defined(CONFIG_PPC_STD_MMU_64 ) +/* -* TLB flushing for 64-bit has-MMU CPUs +* TLB flushing for 64-bit hash-MMU CPUs */ #include <linux/percpu.h>
@@ -151,11 +137,16 @@ extern void flush_hash_page(unsigned lonextern void flush_hash_range (unsigned long number,int local); +static inline void local_flush_tlb_mm (struct mm_struct *mm) +{ +} +static inline void flush_tlb_mm (struct mm_struct *mm) { } -static inline void local_flush_tlb_page (unsigned long vmaddr) +static inline void local_flush_tlb_page (struct vm_area_struct *vma, +unsigned long vmaddr) { }
@@ -183,7 +174,8 @@ static inline void flush_tlb_kernel_rangextern void __flush_hash_table_range (struct mm_struct *mm,unsigned long start,unsigned long end); - +#else +#error Unsupported MMU type#endif #endif /*__KERNEL__ */
arch/powerpc/mm/Makefile +1 -1
@@ -9,7 +9,7 @@ endif obj-y := fault.o mem.o pgtable.o \ init_$(CONFIG_WORD_SIZE).o \ pgtable_$(CONFIG_WORD_SIZE).o -obj-$(CONFIG_PPC_MMU_NOHASH) += mmu_context_nohash.o +obj-$(CONFIG_PPC_MMU_NOHASH) += mmu_context_nohash.o tlb_nohash.o hash-$(CONFIG_PPC_NATIVE) := hash_native_64.o obj-$(CONFIG_PPC64) += hash_utils_64.o \ slb_low.o slb.o stab.o \
arch/powerpc/mm/tlb_nohash.c new file +209 -0
209 lines not on this page · raw message
arch/powerpc/kernel/misc_32.S +9 -0
@@ -29,6 +29,7 @@ #include <asm/asm-offsets.h> #include <asm/processor.h> #include <asm/kexec.h> +#include <asm/bug.h> .text
@@ -496,6 +497,14 @@ _GLOBAL(_tlbil_va) blr #endif /* CONFIG_FSL_BOOKE */ +/* + * Nobody implements this yet + */ +_GLOBAL(_tlbivax_bcast) +1: trap + EMIT_BUG_ENTRY 1b,__FILE__,__LINE__,0; + blr + /* * Flush instruction cache.
arch/powerpc/kernel/ppc_ksyms.c +0 -6
@@ -116,12 +116,6 @@ EXPORT_SYMBOL(giveup_spe);#ifndef CONFIG_PPC64 EXPORT_SYMBOL (flush_instruction_cache); -EXPORT_SYMBOL (flush_tlb_kernel_range); -EXPORT_SYMBOL (flush_tlb_page); -EXPORT_SYMBOL (_tlbie); -#if defined(CONFIG_4xx) || defined(CONFIG_8xx) || defined(CONFIG_FSL_BOOKE ) -EXPORT_SYMBOL (_tlbil_va); -#endif #endif EXPORT_SYMBOL (__flush_icache_range);EXPORT_SYMBOL (flush_dcache_range);
arch/powerpc/mm/tlb_hash32.c +4 -0
@@ -137,6 +137,7 @@ void flush_tlb_kernel_range(unsigned lonflush_range (&init_mm, start, end);FINISH_FLUSH ; } +EXPORT_SYMBOL (flush_tlb_kernel_range);/* * Flush all the (user) entries for the address space described by mm.
@@ -160,6 +161,7 @@ void flush_tlb_mm(struct mm_struct *mm)flush_range (mp->vm_mm, mp->vm_start, mp->vm_end);FINISH_FLUSH ; } +EXPORT_SYMBOL (flush_tlb_mm);void flush_tlb_page (struct vm_area_struct *vma,unsigned long vmaddr) {
@@ -176,6 +178,7 @@ void flush_tlb_page(struct vm_area_strucflush_hash_pages (mm->context.id, vmaddr,pmd_val (*pmd),1 );FINISH_FLUSH ; } +EXPORT_SYMBOL (flush_tlb_page);/* * For each address in the range, find the pte for the address
@@ -188,3 +191,4 @@ void flush_tlb_range(struct vm_area_struflush_range (vma->vm_mm, start, end);FINISH_FLUSH ; } +EXPORT_SYMBOL (flush_tlb_range);
arch/powerpc/include/asm/mmu.h +16 -0
@@ -30,6 +30,22 @@*/ #define MMU_FTR_BIG_PHYS ASM_CONST(0x00020000) +/* Enable use of broadcast TLB invalidations. We don't always set it +* on processors that support it due to other constraints with the +* use of such invalidations +*/ +#define MMU_FTR_USE_TLBIVAX_BCAST ASM_CONST(0x00040000) + +/* Enable use of tlbilx invalidate-by-PID variant. +*/ +#define MMU_FTR_USE_TLBILX_PID ASM_CONST(0x00080000) + +/* This indicates that the processor cannot handle multiple outstanding +* broadcast tlbivax or tlbsync. This makes the code use a spinlock +* around such invalidate forms. +*/ +#define MMU_FTR_LOCK_BCAST_INVAL ASM_CONST(0x00100000) +#ifndef __ASSEMBLY__#include <asm/cputable.h>
arch/powerpc/include/asm/highmem.h +2 -2
@@ -85,7 +85,7 @@ static inline void *kmap_atomic_prot(strBUG_ON (!pte_none (*(kmap_pte-idx)));#endif __set_pte_at (&init_mm, vaddr, kmap_pte-idx,mk_pte (page, prot)); -local_flush_tlb_page (vaddr); +local_flush_tlb_page (NULL , vaddr);return (void *) vaddr; }
@@ -113,7 +113,7 @@ static inline void kunmap_atomic(void *k* this pte without first remap it */ pte_clear (&init_mm, vaddr, kmap_pte-idx); -local_flush_tlb_page (vaddr); +local_flush_tlb_page (NULL , vaddr);#endif pagefault_enable (); }
arch/powerpc/mm/fault.c +1 -1
@@ -284,7 +284,7 @@ good_area: }pte_update (ptep,0 , _PAGE_HWEXEC | _PAGE_ACCESSED); -_tlbie (address, mm->context.id); +local_flush_tlb_page (vma, address);pte_unmap_unlock (ptep, ptl);up_read (&mm->mmap_sem);return 0 ;
arch/powerpc/mm/mem.c +1 -1
@@ -488,7 +488,7 @@ void update_mmu_cache(struct vm_area_str* we invalidate the TLB here, thus avoiding dcbst * misbehaviour. */ -_tlbie (address,0 /* 8xx doesn't care about PID */ ); +_tlbil_va (address,0 /* 8xx doesn't care about PID */ );#endif /* The _PAGE_USER test should really be _PAGE_EXEC, but * older glibc versions execute some code from no-exec
PATCH 6/10 powerpc/mm: Split low level tlb invalidate for nohash processors
In reply to Benjamin HerrenschmidtCurrently, the various forms of low level TLB invalidations are all implemented in misc_32.S for 32-bit processors, in a fairly scary mess of #ifdef's and with interesting duplication such as a whole bunch of code for FSL _tlbie and _tlbia which are no longer used.
This moves things around such that _tlbie is now defined in hash_low_32.S and is only used by the 32-bit hash code, and all nohash CPUs use the various _tlbil_* forms that are now moved to a new file, tlb_nohash_low.S.
I moved all the definitions for that stuff out of include/asm/tlbflush.h as they are really internal mm stuff, into mm/mmu_decl.h
The code should have no functional changes. I kept some variants inline for trivial forms on things like 40x and 8xx.
---
arch/powerpc/include/asm/tlbflush.h | 14 -- arch/powerpc/kernel/misc_32.S | 233 ------------------------------------ arch/powerpc/kvm/powerpc.c | 2 arch/powerpc/mm/Makefile | 3 arch/powerpc/mm/hash_low_32.S | 76 +++++++++++ arch/powerpc/mm/mmu_decl.h | 48 +++++++ arch/powerpc/mm/tlb_nohash_low.S | 165 +++++++++++++++++++++++++ 7 files changed, 292 insertions(+), 249 deletions(-)
arch/powerpc/include/asm/tlbflush.h +0 -14
@@ -33,17 +33,6 @@#define MMU_NO_CONTEXT ((unsigned int)-1) -extern void _tlbil_all (void ); -extern void _tlbil_pid (unsigned int pid); -extern void _tlbil_va (unsigned long address,unsigned int pid); -extern void _tlbivax_bcast (unsigned long address,unsigned int pid); - -#if defined(CONFIG_40x) || defined(CONFIG_8xx) -#define _tlbia () asm volatile ("tlbia; sync" : : : "memory") -#else /* CONFIG_44x || CONFIG_FSL_BOOKE */ -extern void _tlbia (void ); -#endif -extern void flush_tlb_range (struct vm_area_struct *vma,unsigned long start,unsigned long end);extern void flush_tlb_kernel_range (unsigned long start,unsigned long end);
@@ -65,9 +54,6 @@ extern void flush_tlb_page(struct vm_are/* * TLB flushing for "classic" hash-MMU 32-bit CPUs, 6xx, 7xx, 7xxx */ -extern void _tlbie (unsigned long address); -extern void _tlbia (void ); -extern void flush_tlb_mm (struct mm_struct *mm);extern void flush_tlb_page (struct vm_area_struct *vma,unsigned long vmaddr);extern void flush_tlb_page_nohash (struct vm_area_struct *vma,unsigned long addr);
arch/powerpc/mm/mmu_decl.h +48 -0
58 lines not on this page · raw message
arch/powerpc/mm/tlb_nohash_low.S new file +165 -0
165 lines not on this page · raw message
arch/powerpc/kernel/misc_32.S +0 -233
239 lines not on this page · raw message
arch/powerpc/mm/Makefile +2 -1
@@ -9,7 +9,8 @@ endif obj-y := fault.o mem.o pgtable.o \ init_$(CONFIG_WORD_SIZE).o \ pgtable_$(CONFIG_WORD_SIZE).o -obj-$(CONFIG_PPC_MMU_NOHASH) += mmu_context_nohash.o tlb_nohash.o +obj-$(CONFIG_PPC_MMU_NOHASH) += mmu_context_nohash.o tlb_nohash.o \ + tlb_nohash_low.o hash-$(CONFIG_PPC_NATIVE) := hash_native_64.o obj-$(CONFIG_PPC64) += hash_utils_64.o \ slb_low.o slb.o stab.o \
arch/powerpc/mm/hash_low_32.S +76 -0
79 lines not on this page · raw message
arch/powerpc/kvm/powerpc.c +1 -1
@@ -330,7 +330,7 @@ void kvm_arch_vcpu_put(struct kvm_vcpu */* XXX It would be nice to differentiate between heavyweight exit and * sched_out here, since we could avoid the TLB flush for heavyweight * exits. */ -_tlbia (); +_tlbil_all (); } int kvm_arch_vcpu_ioctl_debug_guest(struct kvm_vcpu *vcpu,
PATCH 7/10 powerpc/44x: No need to mask MSR:CE, ME or DE in _tlbil_va on 440
In reply to Benjamin HerrenschmidtThe handlers for Critical, Machine Check or Debug interrupts will save and restore MMUCR nowadays, thus we only need to disable normal interrupts when invalidating TLB entries.
---
arch/powerpc/mm/tlb_nohash_low.S | 19 ++++++++++--------- 1 file changed, 10 insertions(+), 9 deletions(-)
arch/powerpc/mm/tlb_nohash_low.S +10 -9
@@ -75,18 +75,19 @@ _GLOBAL(_tlbil_va) mfspr r5,SPRN_MMUCR rlwimi r5,r4,0,24,31 /* Set TID */ - /* We have to run the search with interrupts disabled, even critical - * and debug interrupts (in fact the only critical exceptions we have - * are debug and machine check). Otherwise an interrupt which causes - * a TLB miss can clobber the MMUCR between the mtspr and the tlbsx. */ + /* We have to run the search with interrupts disabled, otherwise + * an interrupt which causes a TLB miss can clobber the MMUCR + * between the mtspr and the tlbsx. + * + * Critical and Machine Check interrupts take care of saving + * and restoring MMUCR, so only normal interrupts have to be + * taken care of. + */ mfmsr r4 - lis r6,(MSR_EE|MSR_CE|MSR_ME|MSR_DE)@ha - addi r6,r6,(MSR_EE|MSR_CE|MSR_ME|MSR_DE)@l - andc r6,r4,r6 - mtmsr r6 + wrteei 0 mtspr SPRN_MMUCR,r5 tlbsx. r3, 0, r3 - mtmsr r4 + wrtee r4 bne 1f sync /* There are only 64 TLB entries, so r3 < 64,
Re: [PATCH 7/10] powerpc/44x: No need to mask MSR:CE,ME or DE in _tlbil_va on 440
Quoted 5 lines · Benjamin Herrenschmidt
The handlers for Critical, Machine Check or Debug interrupts will save and restore MMUCR nowadays, thus we only need to disable normal interrupts when invalidating TLB entries.
Signed-off-by Benjamin Herrenschmidt <[email protected]>
Quoted 36 lines
---
arch/powerpc/mm/tlb_nohash_low.S| 19 ++++++++++---------1 file changed, 10 insertions(+), 9 deletions(-)arch/powerpc/mm/tlb_nohash_low.S +10
-9@@ -75,18 +75,19 @@ _GLOBAL(_tlbil_va) mfspr r5,SPRN_MMUCR rlwimi r5,r4,0,24,31 /* Set TID */ - /* We have to run the search with interrupts disabled, even critical - * and debug interrupts (in fact the only critical exceptions we have - * are debug and machine check). Otherwise an interrupt which causes - * a TLB miss can clobber the MMUCR between the mtspr and the tlbsx. */ + /* We have to run the search with interrupts disabled, otherwise + * an interrupt which causes a TLB miss can clobber the MMUCR + * between the mtspr and the tlbsx. + * + * Critical and Machine Check interrupts take care of saving + * and restoring MMUCR, so only normal interrupts have to be + * taken care of. + */ mfmsr r4 - lis r6,(MSR_EE|MSR_CE|MSR_ME|MSR_DE)@ha - addi r6,r6,(MSR_EE|MSR_CE|MSR_ME|MSR_DE)@l - andc r6,r4,r6 - mtmsr r6 + wrteei 0 mtspr SPRN_MMUCR,r5 tlbsx. r3, 0, r3 - mtmsr r4 + wrtee r4 bne 1f sync /* There are only 64 TLB entries, so r3 < 64,
PATCH 8/10 powerpc/mm: Runtime allocation of mmu context maps for nohash CPUs v2
In reply to Benjamin HerrenschmidtThis makes the MMU context code used for CPUs with no hash table (except 603) dynamically allocate the various maps used to track the state of contexts.
Only the main free map and CPU 0 stale map are allocated at boot time. Other CPU maps are allocated when those CPUs are brought up and freed if they are unplugged.
This also moves the initialization of the MMU context management slightly later during the boot process, which should be fine as it's really only needed when userland if first started anyways.
---
v2. rebased and add some more debug
arch/powerpc/kernel/setup_32.c | 5 + arch/powerpc/mm/init_32.c | 4 arch/powerpc/mm/mmu_context_nohash.c | 161 ++++++++++++++++++++++++----------- 3 files changed, 116 insertions(+), 54 deletions(-)
arch/powerpc/mm/mmu_context_nohash.c +111 -50
@@ -28,54 +28,30 @@#undef DEBUG#define DEBUG_STEAL_ONLY #undef DEBUG_MAP_CONSISTENCY +/*#define DEBUG_CLAMP_LAST_CONTEXT 15 */ #include <linux/kernel.h> #include <linux/mm.h> #include <linux/init.h> +#include <linux/spinlock.h> +#include <linux/bootmem.h> +#include <linux/notifier.h> +#include <linux/cpu.h> #include <asm/mmu_context.h> #include <asm/tlbflush.h> -#include <linux/spinlock.h> - -/* -* The MPC8xx has only 16 contexts. We rotate through them on each -* task switch. A better way would be to keep track of tasks that -* own contexts, and implement an LRU usage. That way very active -* tasks don't always have to pay the TLB reload overhead. The -* kernel pages are mapped shared, so the kernel can run on behalf -* of any task that makes a kernel entry. Shared does not mean they -* are not protected, just that the ASID comparison is not performed. -* -- Dan -* -* The IBM4xx has 256 contexts, so we can just rotate through these -* as a way of "switching" contexts. If the TID of the TLB is zero, -* the PID/TID comparison is disabled, so we can use a TID of zero -* to represent all kernel pages as shared among all contexts. -* -- Dan -*/ - -#ifdef CONFIG_8xx -#define LAST_CONTEXT 15 -#define FIRST_CONTEXT 0 - -#elif defined(CONFIG_4xx) -#define LAST_CONTEXT 255 -#define FIRST_CONTEXT 1 - -#elif defined(CONFIG_E200 ) || defined(CONFIG_E500 ) -#define LAST_CONTEXT 255 -#define FIRST_CONTEXT 1 - -#else -#error Unsupported processor type -#endif +static unsigned int first_context, last_context;static unsigned int next_context, nr_free_contexts; -static unsigned long context_map[LAST_CONTEXT /BITS_PER_LONG +1 ]; -static unsigned long stale_map[NR_CPUS ][LAST_CONTEXT /BITS_PER_LONG +1 ]; -static struct mm_struct *context_mm[LAST_CONTEXT +1 ]; +static unsigned long *context_map; +static unsigned long *stale_map[NR_CPUS ]; +static struct mm_struct **context_mm;static spinlock_t context_lock =SPIN_LOCK_UNLOCKED ; +#define CTX_MAP_SIZE \ + (sizeof(unsigned long) * (last_context / BITS_PER_LONG + 1)) + +/* Steal a context from a task that has one at the moment. * * This is used when we are running out of available PID numbers
@@ -98,7 +74,7 @@ static unsigned int steal_context_smp(ununsigned int cpu, max; again: - max =LAST_CONTEXT -FIRST_CONTEXT ; + max = last_context - first_context;/* Attempt to free next_context first and then loop until we manage */ while (max--) {
@@ -110,8 +86,8 @@ static unsigned int steal_context_smp(un*/ if (mm->context.active) { id ++; -if (id >LAST_CONTEXT ) - id =FIRST_CONTEXT ; +if (id > last_context) + id = first_context;continue ; } pr_debug("[%d] steal context %d from mm @%p\n",
@@ -169,7 +145,7 @@ static void context_check_map(void)unsigned int id, nrf, nact; nrf = nact =0 ; -for (id =FIRST_CONTEXT ; id <=LAST_CONTEXT ; id++) { +for (id = first_context; id <= last_context; id++) {int used =test_bit (id, context_map);if (!used) nrf++;
@@ -187,6 +163,8 @@ static void context_check_map(void)if (nact >num_online_cpus ())pr_err ("MMU: More active contexts than CPUs ! (%d vs %d)\n" , nact,num_online_cpus ()); +if (first_context >0 && !test_bit (0 , context_map)) + pr_err("MMU: Context 0 has been freed !!!\n"); }#else static void context_check_map (void ) { }
@@ -209,6 +187,10 @@ void switch_mmu_context(struct mm_struct/* Mark us active and the previous one not anymore */ next->context.active++;if (prev) { +#ifndef DEBUG_STEAL_ONLY +pr_debug (" old context %p active was: %d\n" , + prev, prev->context.active); +#endif WARN_ON (prev->context.active <1 ); prev->context.active--; }
@@ -221,8 +203,8 @@ void switch_mmu_context(struct mm_struct/* We really don't have a context, let's try to acquire one */ id = next_context; -if (id >LAST_CONTEXT ) - id =FIRST_CONTEXT ; +if (id > last_context) + id = first_context; map = context_map;/* No more free contexts, let's try to steal one */
@@ -240,9 +222,9 @@ void switch_mmu_context(struct mm_struct/* We know there's at least one free context, try to find it */ while (__test_and_set_bit (id, map)) { - id =find_next_zero_bit (map,LAST_CONTEXT +1 , id); -if (id >LAST_CONTEXT ) - id =FIRST_CONTEXT ; + id =find_next_zero_bit (map, last_context+1 , id); +if (id > last_context) + id = first_context; } stolen: next_context = id +1 ;
@@ -311,6 +293,42 @@ void destroy_context(struct mm_struct *mspin_unlock (&context_lock); } +#ifdef CONFIG_SMP + +static int __cpuinit mmu_context_cpu_notify (struct notifier_block *self, +unsigned long action,void *hcpu) +{ +unsigned int cpu = (unsigned int )(long )hcpu; + +/* We don't touch CPU 0 map, it's allocated at aboot and kept +* around forever +*/ +if (cpu ==0 ) +return NOTIFY_OK ; + +switch (action) { +case CPU_ONLINE : +case CPU_ONLINE_FROZEN : +pr_debug ("MMU: Allocating stale context map for CPU %d\n" , cpu); + stale_map[cpu] =kzalloc (CTX_MAP_SIZE ,GFP_KERNEL ); +break ; +#ifdef CONFIG_HOTPLUG_CPU +case CPU_DEAD : +case CPU_DEAD_FROZEN : +pr_debug ("MMU: Freeing stale context map for CPU %d\n" , cpu); +kfree (stale_map[cpu]); + stale_map[cpu] =NULL ; +break ; +#endif + } +return NOTIFY_OK ; +} + +static struct notifier_block __cpuinitdata mmu_context_cpu_nb = { + .notifier_call = mmu_context_cpu_notify, +}; + +#endif /* CONFIG_SMP */ /* * Initialize the context management stuff.
@@ -324,13 +342,56 @@ void __init mmu_context_init(void) init_mm.context.active =NR_CPUS ;/* +* The MPC8xx has only 16 contexts. We rotate through them on each +* task switch. A better way would be to keep track of tasks that +* own contexts, and implement an LRU usage. That way very active +* tasks don't always have to pay the TLB reload overhead. The +* kernel pages are mapped shared, so the kernel can run on behalf +* of any task that makes a kernel entry. Shared does not mean they +* are not protected, just that the ASID comparison is not performed. +* -- Dan +* +* The IBM4xx has 256 contexts, so we can just rotate through these +* as a way of "switching" contexts. If the TID of the TLB is zero, +* the PID/TID comparison is disabled, so we can use a TID of zero +* to represent all kernel pages as shared among all contexts. +* -- Dan +*/ +if (mmu_has_feature (MMU_FTR_TYPE_8xx)) { + first_context =0 ; + last_context =15 ; + }else { + first_context =1 ; + last_context =255 ; + } + +#ifdef DEBUG_CLAMP_LAST_CONTEXT + last_context =DEBUG_CLAMP_LAST_CONTEXT ; +#endif +/* +* Allocate the maps used by context management +*/ + context_map =alloc_bootmem (CTX_MAP_SIZE ); + context_mm =alloc_bootmem (sizeof (void *) * (last_context +1 )); + stale_map[0 ] =alloc_bootmem (CTX_MAP_SIZE ); + +#ifdef CONFIG_SMP +register_cpu_notifier (&mmu_context_cpu_nb); +#endif + +printk (KERN_INFO +"MMU: Allocated %d bytes of context maps for %d contexts\n" , +2 *CTX_MAP_SIZE + (sizeof (void *) * (last_context +1 )), + last_context - first_context +1 ); + +/* * Some processors have too few contexts to reserve one for * init_mm, and require using context 0 for a normal task. * Other processors reserve the use of context zero for the kernel. -* This code assumes FIRST_CONTEXT < 32. +* This code assumes first_context < 32. */ - context_map[0 ] = (1 <<FIRST_CONTEXT ) -1 ; - next_context =FIRST_CONTEXT ; - nr_free_contexts =LAST_CONTEXT -FIRST_CONTEXT +1 ; + context_map[0 ] = (1 << first_context) -1 ; + next_context = first_context; + nr_free_contexts = last_context - first_context +1 ; }
arch/powerpc/kernel/setup_32.c +5 -0
@@ -38,6 +38,7 @@#include <asm/time.h> #include <asm/serial.h> #include <asm/udbg.h> +#include <asm/mmu_context.h> #include "setup.h"
@@ -330,4 +331,8 @@ void __init setup_arch(char **cmdline_p)if ( ppc_md.progress ) ppc_md.progress ("arch: exit" ,0x3eab );paging_init (); + +/* Initialize the MMU context management stuff */ +mmu_context_init (); + }
arch/powerpc/mm/init_32.c +0 -4
@@ -35,7 +35,6 @@#include <asm/pgalloc.h> #include <asm/prom.h> #include <asm/io.h> -#include <asm/mmu_context.h> #include <asm/pgtable.h> #include <asm/mmu.h> #include <asm/smp.h>
@@ -180,9 +179,6 @@ void __init MMU_init(void)if (ppc_md.progress) ppc_md.progress ("MMU:setio" ,0x302 ); -/* Initialize the context management stuff */ -mmu_context_init (); -if (ppc_md.progress) ppc_md.progress ("MMU:exit" ,0x211 );
PATCH 9/10 powerpc/mm: Rework usage of _PAGE_COHERENT/NO_CACHE/GUARDED v2
In reply to Benjamin HerrenschmidtCurrently, we never set _PAGE_COHERENT in the PTEs, we just OR it in in the hash code based on some CPU feature bit. We also manipulate _PAGE_NO_CACHE and _PAGE_GUARDED by hand in all sorts of places.
This changes the logic so that instead, the PTE now contains _PAGE_COHERENT for all normal RAM pages thay have I = 0 on platforms that need it. The hash code clears it if the feature bit is not set.
It also adds some clean accessors to setup various valid combinations of access flags and change various bits of code to use them instead.
This should help having the PTE actually containing the bit combinations that we really want.
I also removed _PAGE_GUARDED from _PAGE_BASE on 44x and instead set it explicitely from the TLB miss. I will ultimately remove it completely as it appears that it might not be needed after all but in the meantime, having it in the TLB miss makes things a lot easier.
---
v2. Make _PAGE_NO_CACHE part of _PAGE_BASE_NC and fixup typo in
the definition of _PAGE_CACHE_CTL
arch/powerpc/include/asm/pgtable-ppc32.h | 42 ++++++++++++------------------- arch/powerpc/include/asm/pgtable-ppc64.h | 13 --------- arch/powerpc/include/asm/pgtable.h | 26 +++++++++++++++++++ arch/powerpc/kernel/head_44x.S | 1 arch/powerpc/kernel/pci-common.c | 24 ++++++----------- arch/powerpc/mm/hash_low_32.S | 4 +- arch/powerpc/mm/mem.c | 4 +- arch/powerpc/platforms/cell/spufs/file.c | 27 ++++++------------- drivers/video/controlfb.c | 4 +- 9 files changed, 68 insertions(+), 77 deletions(-)
arch/powerpc/include/asm/pgtable-ppc32.h +17 -25
@@ -228,9 +228,10 @@ extern int icache_44x_need_flush;* - FILE *must* be in the bottom three bits because swap cache * entries use the top 29 bits for TLB2. * -* - CACHE COHERENT bit (M) has no effect on PPC440 core, because it -* doesn't support SMP. So we can use this as software bit, like -* DIRTY. +* - CACHE COHERENT bit (M) has no effect on original PPC440 cores, +* because it doesn't support SMP. However, some later 460 variants +* have -some- form of SMP support and so I keep the bit there for +* future use * * With the PPC 44x Linux implementation, the 0-11th LSBs of the PTE are used * for memory protection related functions (see PTE structure in
@@ -436,20 +437,23 @@ extern int icache_44x_need_flush; _PAGE_USER | _PAGE_ACCESSED | \ _PAGE_RW | _PAGE_HWWRITE | _PAGE_DIRTY | \ _PAGE_EXEC | _PAGE_HWEXEC) +/* -* Note: the _PAGE_COHERENT bit automatically gets set in the hardware -* PTE if CONFIG_SMP is defined (hash_page does this); there is no need -* to have it in the Linux PTE, and in fact the bit could be reused for -* another purpose. -- paulus. +* We define 2 sets of base prot bits, one for basic pages (ie, +* cacheable kernel and user pages) and one for non cacheable +* pages. We always set _PAGE_COHERENT when SMP is enabled or +* the processor might need it for DMA coherency. */ - -#ifdef CONFIG_44x -#define _PAGE_BASE (_PAGE_PRESENT | _PAGE_ACCESSED | _PAGE_GUARDED) +#if defined(CONFIG_SMP ) || defined(CONFIG_PPC_STD_MMU ) +#define _PAGE_BASE (_PAGE_PRESENT | _PAGE_ACCESSED | _PAGE_COHERENT)#else #define _PAGE_BASE (_PAGE_PRESENT | _PAGE_ACCESSED)#endif +#define _PAGE_BASE_NC (_PAGE_PRESENT | _PAGE_ACCESSED | _PAGE_NO_CACHE) +#define _PAGE_WRENABLE (_PAGE_RW | _PAGE_DIRTY | _PAGE_HWWRITE)#define _PAGE_KERNEL (_PAGE_BASE | _PAGE_SHARED | _PAGE_WRENABLE) +#define _PAGE_KERNEL_NC (_PAGE_BASE_NC | _PAGE_SHARED | _PAGE_WRENABLE)#ifdef CONFIG_PPC_STD_MMU /* On standard PPC MMU, no user access implies kernel read/write access,
@@ -459,7 +463,7 @@ extern int icache_44x_need_flush;#define _PAGE_KERNEL_RO (_PAGE_BASE | _PAGE_SHARED)#endif -#define _PAGE_IO (_PAGE_KERNEL | _PAGE_NO_CACHE | _PAGE_GUARDED) +#define _PAGE_IO (_PAGE_KERNEL_NC | _PAGE_GUARDED)#define _PAGE_RAM (_PAGE_KERNEL | _PAGE_HWEXEC) #if defined(CONFIG_KGDB) || defined(CONFIG_XMON) || defined(CONFIG_BDI_SWITCH) ||\
@@ -552,9 +556,6 @@ static inline int pte_young(pte_t pte)static inline int pte_file (pte_t pte) {return pte_val (pte) & _PAGE_FILE; }static inline int pte_special (pte_t pte) {return pte_val (pte) & _PAGE_SPECIAL; } -static inline void pte_uncache (pte_t pte) {pte_val (pte) |= _PAGE_NO_CACHE; } -static inline void pte_cache (pte_t pte) {pte_val (pte) &= ~_PAGE_NO_CACHE; } -static inline pte_t pte_wrprotect (pte_t pte) {pte_val (pte) &= ~(_PAGE_RW | _PAGE_HWWRITE);return pte; }static inline pte_t pte_mkclean (pte_t pte) {
@@ -693,10 +694,11 @@ static inline void __set_pte_at(struct m#endif } +static inline void set_pte_at (struct mm_struct *mm,unsigned long addr,pte_t *ptep,pte_t pte) { -#if defined(CONFIG_PTE_64BIT ) && defined(CONFIG_SMP ) +#if defined(CONFIG_PTE_64BIT ) && defined(CONFIG_SMP ) && defined(CONFIG_DEBUG_VM )WARN_ON (pte_present (*ptep));#endif __set_pte_at (mm, addr, ptep, pte);
@@ -760,16 +762,6 @@ static inline void __ptep_set_access_fla __changed; \ }) -/* -* Macro to mark a page protection value as "uncacheable". -*/ -#define pgprot_noncached (prot) (__pgprot(pgprot_val(prot) | _PAGE_NO_CACHE | _PAGE_GUARDED)) - -struct file ; -extern pgprot_t phys_mem_access_prot (struct file *file,unsigned long pfn, -unsigned long size,pgprot_t vma_prot); -#define __HAVE_PHYS_MEM_ACCESS_PROT -#define __HAVE_ARCH_PTE_SAME#define pte_same (A ,B ) (((pte_val(A) ^ pte_val(B)) & ~_PAGE_HASHPTE) == 0)
arch/powerpc/kernel/head_44x.S +1 -0
@@ -570,6 +570,7 @@ finish_tlb_load: rlwimi r10,r12,29,30,30 /* DIRTY -> SW position */ and r11,r12,r10 /* Mask PTE bits to keep */ andi. r10,r12,_PAGE_USER /* User page ? */ + ori r11,r11,_PAGE_GUARDED /* 440 errata, needs G set */ beq 1f /* nope, leave U bits empty */ rlwimi r11,r11,3,26,28 /* yes, copy S bits to U */ 1: tlbwe r11,r13,PPC44x_TLB_ATTRIB /* Write ATTRIB */
arch/powerpc/mm/hash_low_32.S +2 -2
@@ -323,8 +323,8 @@ _GLOBAL(create_hpte) ori r8,r8,0xe14 /* clear out reserved bits and M */ andc r8,r5,r8 /* PP = user? (rw&dirty? 2: 3): 0 */ BEGIN_FTR_SECTION - ori r8,r8,_PAGE_COHERENT /* set M (coherence required) */ -END_FTR_SECTION_IFSET(CPU_FTR_NEED_COHERENT) + rlwinm r8,r8,0,~_PAGE_COHERENT /* clear M (coherence not required) */ +END_FTR_SECTION_IFCLR(CPU_FTR_NEED_COHERENT) #ifdef CONFIG_PTE_64BIT /* Put the XPN bits into the PTE */ rlwimi r8,r10,8,20,22
arch/powerpc/include/asm/pgtable-ppc64.h +0 -13
@@ -245,9 +245,6 @@ static inline int pte_young(pte_t pte) {static inline int pte_file (pte_t pte) {return pte_val (pte) & _PAGE_FILE;}static inline int pte_special (pte_t pte) {return pte_val (pte) & _PAGE_SPECIAL; } -static inline void pte_uncache (pte_t pte) {pte_val (pte) |= _PAGE_NO_CACHE; } -static inline void pte_cache (pte_t pte) {pte_val (pte) &= ~_PAGE_NO_CACHE; } -static inline pte_t pte_wrprotect (pte_t pte) {pte_val (pte) &= ~(_PAGE_RW);return pte; }static inline pte_t pte_mkclean (pte_t pte) {
@@ -405,16 +402,6 @@ static inline void __ptep_set_access_fla __changed; \ }) -/* -* Macro to mark a page protection value as "uncacheable". -*/ -#define pgprot_noncached (prot) (__pgprot(pgprot_val(prot) | _PAGE_NO_CACHE | _PAGE_GUARDED)) - -struct file ; -extern pgprot_t phys_mem_access_prot (struct file *file,unsigned long pfn, -unsigned long size,pgprot_t vma_prot); -#define __HAVE_PHYS_MEM_ACCESS_PROT -#define __HAVE_ARCH_PTE_SAME#define pte_same (A ,B ) (((pte_val(A) ^ pte_val(B)) & ~_PAGE_HPTEFLAGS) == 0)
arch/powerpc/include/asm/pgtable.h +26 -0
@@ -16,6 +16,32 @@ struct mm_struct;#endif #ifndef __ASSEMBLY__ + +/* +* Macro to mark a page protection value as "uncacheable". +*/ + +#define _PAGE_CACHE_CTL (_PAGE_COHERENT | _PAGE_GUARDED | _PAGE_NO_CACHE | \ + _PAGE_WRITETHRU) + +#define pgprot_noncached (prot) (__pgprot((pgprot_val(prot) & ~_PAGE_CACHE_CTL) | \ + _PAGE_NO_CACHE | _PAGE_GUARDED)) + +#define pgprot_noncached_wc (prot) (__pgprot((pgprot_val(prot) & ~_PAGE_CACHE_CTL) | \ + _PAGE_NO_CACHE)) + +#define pgprot_cached (prot) (__pgprot((pgprot_val(prot) & ~_PAGE_CACHE_CTL) | \ + _PAGE_COHERENT)) + +#define pgprot_cached_wthru (prot) (__pgprot((pgprot_val(prot) & ~_PAGE_CACHE_CTL) | \ + _PAGE_COHERENT | _PAGE_WRITETHRU)) + + +struct file ; +extern pgprot_t phys_mem_access_prot (struct file *file,unsigned long pfn, +unsigned long size,pgprot_t vma_prot); +#define __HAVE_PHYS_MEM_ACCESS_PROT +/* * ZERO_PAGE is a global shared page that is always zero: used * for zero-mapped memory areas etc..
arch/powerpc/kernel/pci-common.c +9 -15
@@ -370,13 +370,10 @@ static pgprot_t __pci_mmap_set_pgprot(st }/* XXX would be nice to have a way to ask for write-through */ - prot |= _PAGE_NO_CACHE;if (write_combine) - prot &= ~_PAGE_GUARDED; +return pgprot_noncached_wc (prot);else - prot |= _PAGE_GUARDED; - -return __pgprot (prot); +return pgprot_noncached (prot); }/*
@@ -387,19 +384,17 @@ static pgprot_t __pci_mmap_set_pgprot(stpgprot_t pci_phys_mem_access_prot (struct file *file,unsigned long pfn,unsigned long size, -pgprot_t protection) +pgprot_t prot) {struct pci_dev *pdev =NULL ;struct resource *found =NULL ; -unsigned long prot =pgprot_val (protection);resource_size_t offset = ((resource_size_t )pfn) <<PAGE_SHIFT ;int i;if (page_is_ram (pfn)) -return __pgprot (prot); - - prot |= _PAGE_NO_CACHE | _PAGE_GUARDED; +return prot; + prot =pgprot_noncached (prot);for_each_pci_dev (pdev) {for (i =0 ; i <=PCI_ROM_RESOURCE ; i++) {struct resource *rp = &pdev->resource[i];
@@ -420,14 +415,14 @@ pgprot_t pci_phys_mem_access_prot(struct }if (found) {if (found->flags &IORESOURCE_PREFETCH ) - prot &= ~_PAGE_GUARDED; + prot =pgprot_noncached_wc (prot);pci_dev_put (pdev); }pr_debug ("PCI: Non-PCI map for %llx, prot: %lx\n" , - (unsigned long long )offset, prot); + (unsigned long long )offset,pgprot_val (prot)); -return __pgprot (prot); +return prot; }
@@ -583,8 +578,7 @@ int pci_mmap_legacy_page_range(struct pcpr_debug (" -> mapping phys %llx\n" , (unsigned long long )offset); vma->vm_pgoff = offset >>PAGE_SHIFT ; - vma->vm_page_prot =__pgprot (pgprot_val (vma->vm_page_prot) - | _PAGE_NO_CACHE | _PAGE_GUARDED); + vma->vm_page_prot =pgprot_noncached (vma->vm_page_prot);return remap_pfn_range (vma, vma->vm_start, vma->vm_pgoff, vma->vm_end - vma->vm_start, vma->vm_page_prot);
arch/powerpc/mm/mem.c +2 -2
@@ -102,8 +102,8 @@ pgprot_t phys_mem_access_prot(struct filreturn ppc_md.phys_mem_access_prot (file, pfn, size, vma_prot);if (!page_is_ram (pfn)) - vma_prot =__pgprot (pgprot_val (vma_prot) - | _PAGE_GUARDED | _PAGE_NO_CACHE); + vma_prot =pgprot_noncached (vma_prot); +return vma_prot; }EXPORT_SYMBOL (phys_mem_access_prot);
arch/powerpc/platforms/cell/spufs/file.c +9 -18
@@ -273,12 +273,10 @@ spufs_mem_mmap_fault(struct vm_area_strureturn VM_FAULT_NOPAGE ;if (ctx->state ==SPU_STATE_SAVED ) { - vma->vm_page_prot =__pgprot (pgprot_val (vma->vm_page_prot) - & ~_PAGE_NO_CACHE); + vma->vm_page_prot =pgprot_cached (vma->vm_page_prot); pfn =vmalloc_to_pfn (ctx->csa.lscsa->ls + offset); }else { - vma->vm_page_prot =__pgprot (pgprot_val (vma->vm_page_prot) - | _PAGE_NO_CACHE); + vma->vm_page_prot =pgprot_noncached_wc (vma->vm_page_prot); pfn = (ctx->spu->local_store_phys + offset) >>PAGE_SHIFT ; }vm_insert_pfn (vma, address, pfn);
@@ -338,8 +336,7 @@ static int spufs_mem_mmap(struct file *freturn -EINVAL ; vma->vm_flags |=VM_IO |VM_PFNMAP ; - vma->vm_page_prot =__pgprot (pgprot_val (vma->vm_page_prot) - | _PAGE_NO_CACHE); + vma->vm_page_prot =pgprot_noncached_wc (vma->vm_page_prot); vma->vm_ops = &spufs_mem_mmap_vmops;return 0 ;
@@ -452,8 +449,7 @@ static int spufs_cntl_mmap(struct file *return -EINVAL ; vma->vm_flags |=VM_IO |VM_PFNMAP ; - vma->vm_page_prot =__pgprot (pgprot_val (vma->vm_page_prot) - | _PAGE_NO_CACHE | _PAGE_GUARDED); + vma->vm_page_prot =pgprot_noncached (vma->vm_page_prot); vma->vm_ops = &spufs_cntl_mmap_vmops;return 0 ;
@@ -1155,8 +1151,7 @@ static int spufs_signal1_mmap(struct filreturn -EINVAL ; vma->vm_flags |=VM_IO |VM_PFNMAP ; - vma->vm_page_prot =__pgprot (pgprot_val (vma->vm_page_prot) - | _PAGE_NO_CACHE | _PAGE_GUARDED); + vma->vm_page_prot =pgprot_noncached (vma->vm_page_prot); vma->vm_ops = &spufs_signal1_mmap_vmops;return 0 ;
@@ -1292,8 +1287,7 @@ static int spufs_signal2_mmap(struct filreturn -EINVAL ; vma->vm_flags |=VM_IO |VM_PFNMAP ; - vma->vm_page_prot =__pgprot (pgprot_val (vma->vm_page_prot) - | _PAGE_NO_CACHE | _PAGE_GUARDED); + vma->vm_page_prot =pgprot_noncached (vma->vm_page_prot); vma->vm_ops = &spufs_signal2_mmap_vmops;return 0 ;
@@ -1414,8 +1408,7 @@ static int spufs_mss_mmap(struct file *freturn -EINVAL ; vma->vm_flags |=VM_IO |VM_PFNMAP ; - vma->vm_page_prot =__pgprot (pgprot_val (vma->vm_page_prot) - | _PAGE_NO_CACHE | _PAGE_GUARDED); + vma->vm_page_prot =pgprot_noncached (vma->vm_page_prot); vma->vm_ops = &spufs_mss_mmap_vmops;return 0 ;
@@ -1476,8 +1469,7 @@ static int spufs_psmap_mmap(struct filereturn -EINVAL ; vma->vm_flags |=VM_IO |VM_PFNMAP ; - vma->vm_page_prot =__pgprot (pgprot_val (vma->vm_page_prot) - | _PAGE_NO_CACHE | _PAGE_GUARDED); + vma->vm_page_prot =pgprot_noncached (vma->vm_page_prot); vma->vm_ops = &spufs_psmap_mmap_vmops;return 0 ;
@@ -1536,8 +1528,7 @@ static int spufs_mfc_mmap(struct file *freturn -EINVAL ; vma->vm_flags |=VM_IO |VM_PFNMAP ; - vma->vm_page_prot =__pgprot (pgprot_val (vma->vm_page_prot) - | _PAGE_NO_CACHE | _PAGE_GUARDED); + vma->vm_page_prot =pgprot_noncached (vma->vm_page_prot); vma->vm_ops = &spufs_mfc_mmap_vmops;return 0 ;
drivers/video/controlfb.c +2 -2
@@ -298,10 +298,10 @@ static int controlfb_mmap(struct fb_inforeturn -EINVAL ; start = info->fix.mmio_start; len =PAGE_ALIGN ((start & ~PAGE_MASK )+info->fix.mmio_len); -pgprot_val (vma->vm_page_prot) |= _PAGE_NO_CACHE|_PAGE_GUARDED; + vma->vm_page_prot =pgprot_noncached (vma->vm_page_prot); }else {/* framebuffer */ -pgprot_val (vma->vm_page_prot) |= _PAGE_WRITETHRU; + vma->vm_page_prot =pgprot_cached_wthru (vma->vm_page_prot); } start &=PAGE_MASK ;if ((vma->vm_end - vma->vm_start + off) > len)
Re: [PATCH 9/10] powerpc/mm: Rework usage of _PAGE_COHERENT/NO_CACHE/GUARDED v2
-------- Original-Nachricht --------
Quoted 6 lines
Datum: Fri, 19 Dec 2008 16:13:51 +1100
Von: Benjamin Herrenschmidt <[email protected]>
An: Paul Mackerras <[email protected]>
CC: [email protected], Kumar Gala <[email protected]>
Betreff: [PATCH 9/10] powerpc/mm: Rework usage of _PAGE_COHERENT/NO_CACHE/GUARDED v2
Quoted 7 lines
Currently, we never set
_PAGE_COHERENTin the PTEs, we just OR it in in the hash code based on some CPU feature bit. We also manipulate_PAGE_NO_CACHEand_PAGE_GUARDEDby hand in all sorts of places.This changes the logic so that instead, the PTE now contains
_PAGE_COHERENTfor all normal RAM pages thay have I = 0 on platforms that need it. The hash code clears it if the feature bit is not set.
This somehow breaks Linux on my AmigaOne, as its northbridge doesn't like pages marked as coherent. I removed the CPU_FTR_NEED_COHERENT feature bit in setup_32.c before the fixups are done, but the system still locks up (doing the fixup in the platform code is too late). It only works stable, if I remove the _PAGE_COHERENT flag in pgtable-ppc32.h below.
arch/powerpc/mm/hash_low_32.S _GLOBAL(create_hpte) · line 327
Quoted 8 lines
arch/powerpc/mm/hash_low_32.S +4
-2+#if defined(CONFIG_SMP) || defined(CONFIG_PPC_STD_MMU) +#define _PAGE_BASE (_PAGE_PRESENT | _PAGE_ACCESSED | _PAGE_COHERENT) BEGIN_FTR_SECTION - ori r8,r8,_PAGE_COHERENT /* set M (coherence required) */ -END_FTR_SECTION_IFSET(CPU_FTR_NEED_COHERENT) + rlwinm r8,r8,0,~_PAGE_COHERENT /* clear M (coherence not required) */ +END_FTR_SECTION_IFCLR(CPU_FTR_NEED_COHERENT)
I'm pretty sure that the hack for setup_32.c above protects this code in hash_low_32.S/ create_hpte() from being overwritten with nops. But I wonder which other paths exists in the memory management code to set the coherent flag in PTEs? Any ideas how I can get it to work without removing _PAGE_COHERENT from _PAGE_BASE?
Thanks!
Gerhard
Signature · 2 lines
Sensationsangebot verlängert: GMX FreeDSL - Telefonanschluss + DSL für nur 16,37 Euro/mtl.!* http://dsl.gmx.de/?ac=OM.AD.PD003K1308T4569a
Re: [PATCH 9/10] powerpc/mm: Rework usage of _PAGE_COHERENT/NO_CACHE/GUARDED v2
Quoted 4 lines · Gerhard Pircher
I'm pretty sure that the hack for
setup_32.cabove protects this code in hash_low_32.S/create_hpte()from being overwritten with nops. But I wonder which other paths exists in the memory management code to set the coherent flag in PTEs? Any ideas how I can get it to work without removing_PAGE_COHERENTfrom_PAGE_BASE?
I'll have a look, could be something with the BAT setup too, and I haven't modified the 603 code, are you using a 603 ?
Ben.
Re: [PATCH 9/10] powerpc/mm: Rework usage of _PAGE_COHERENT/NO_CACHE/GUARDED v2
-------- Original-Nachricht --------
Quoted 6 lines
Datum: Mon, 19 Jan 2009 06:55:06 +1100
Von: Benjamin Herrenschmidt <[email protected]>
An: Gerhard Pircher <[email protected]>
CC: [email protected], [email protected], [email protected]
Betreff: Re: [PATCH 9/10] powerpc/mm: Rework usage of _PAGE_COHERENT/NO_CACHE/GUARDED v2
Quoted 8 lines
Quoted 4 lines · Gerhard Pircher
I'm pretty sure that the hack for
setup_32.cabove protects this code in hash_low_32.S/create_hpte()from being overwritten with nops. But I wonder which other paths exists in the memory management code to set the coherent flag in PTEs? Any ideas how I can get it to work without removing_PAGE_COHERENTfrom_PAGE_BASE?I'll have a look, could be something with the BAT setup too, and I haven't modified the 603 code, are you using a 603 ?
No, it's a 7455 (which has CPU_FTR_NEED_COHERENT set in its CPU features).
Thanks!
Gerhard
Signature · 2 lines
Sensationsangebot verlängert: GMX FreeDSL - Telefonanschluss + DSL für nur 16,37 Euro/mtl.!* http://dsl.gmx.de/?ac=OM.AD.PD003K1308T4569a
PATCH 1/1 powerpc/mm: Fix handling of _PAGE_COHERENT in BAT setup code
_PAGE_COHERENT is now always set in _PAGE_RAM resp. PAGE_KERNEL. Thus it has to be masked out, if the BAT mapping should be non cacheable or CPU_FTR_NEED_COHERENT is not set.
--- arch/powerpc/mm/ppc_mmu_32.c | 6 +++--- 1 files changed, 3 insertions(+), 3 deletions(-)
arch/powerpc/mm/ppc_mmu_32.c +3 -3
@@ -123,9 +123,9 @@ void __init setbat(int index, unsigned long virt, phys_addr_t phys,int wimgxpp;struct ppc_bat *bat =BATS [index]; -if (((flags & _PAGE_NO_CACHE) ==0 ) && -cpu_has_feature (CPU_FTR_NEED_COHERENT )) - flags |= _PAGE_COHERENT; +if ((flags & _PAGE_NO_CACHE) || + (cpu_has_feature (CPU_FTR_NEED_COHERENT ) ==0 )) + flags &= ~_PAGE_COHERENT; bl = (size >>17 ) -1 ;if (PVR_VER (mfspr (SPRN_PVR )) !=1 ) {
Signature · 6 lines
1.5.3.4 -- NUR NOCH BIS 31.01.! GMX FreeDSL - Telefonanschluss + DSL für nur 16,37 EURO/mtl.!* http://dsl.gmx.de/?ac=OM.AD.PD003K11308T4569a
PATCH 10/10 powerpc/44x: 44x TLB doesn't need "Guarded" set for all pages
In reply to Benjamin HerrenschmidtAfter discussing with chip designers, it appears that it's not necessary to set G everywhere on 440 cores. The various core errata related to prefetch should be sorted out by firmware by disabling icache prefetching in CCR0. We add the workaround to the kernel however just in case oooold firmwares don't do it.
This is valid for -all- 4xx core variants. Later ones hard wire the absence of prefetch but it doesn't harm to clear the bits in CCR0 (they should already be cleared anyway).
We still leave G=1 on the linear mapping for now, we need to stop over-mapping RAM to be able to remove it.
---
arch/powerpc/kernel/head_44x.S | 12 +++++++++++- 1 file changed, 11 insertions(+), 1 deletion(-)
arch/powerpc/kernel/head_44x.S +11 -1
@@ -69,6 +69,17 @@ _ENTRY(_start); li r24,0 /* CPU number */ /* + * In case the firmware didn't do it, we apply some workarounds + * that are good for all 440 core variants here + */ + mfspr r3,SPRN_CCR0 + rlwinm r3,r3,0,0,27 /* disable icache prefetch */ + isync + mtspr SPRN_CCR0,r3 + isync + sync + +/* * Set up the initial MMU state * * We are still executing code at the virtual address
@@ -570,7 +581,6 @@ finish_tlb_load: rlwimi r10,r12,29,30,30 /* DIRTY -> SW position */ and r11,r12,r10 /* Mask PTE bits to keep */ andi. r10,r12,_PAGE_USER /* User page ? */ - ori r11,r11,_PAGE_GUARDED /* 440 errata, needs G set */ beq 1f /* nope, leave U bits empty */ rlwimi r11,r11,3,26,28 /* yes, copy S bits to U */ 1: tlbwe r11,r13,PPC44x_TLB_ATTRIB /* Write ATTRIB */
Re: [PATCH 10/10] powerpc/44x: 44x TLB doesn't need "Guarded" set for all pages
Quoted 6 lines · Benjamin Herrenschmidt
After discussing with chip designers, it appears that it's not necessary to set G everywhere on 440 cores. The various core errata related to prefetch should be sorted out by firmware by disabling icache prefetching in CCR0. We add the workaround to the kernel however just in case oooold firmwares don't do it.
Quoted 8 lines
This is valid for -all- 4xx core variants. Later ones hard wire the absence of prefetch but it doesn't harm to clear the bits in CCR0 (they should already be cleared anyway).
We still leave G=1 on the linear mapping for now, we need to stop over-mapping RAM to be able to remove it.
Signed-off-by Benjamin Herrenschmidt <[email protected]>
I've been testing this for the past couple of days and it has been working generally well on large process and heavy threaded workloads.
I'd like to get better testcases, but that's an on-going need so:
Quoted 33 lines
---
arch/powerpc/kernel/head_44x.S| 12 +++++++++++-1 file changed, 11 insertions(+), 1 deletion(-)arch/powerpc/kernel/head_44x.S +11
-1@@ -69,6 +69,17 @@ _ENTRY(_start); li r24,0 /* CPU number */ /* + * In case the firmware didn't do it, we apply some workarounds + * that are good for all 440 core variants here + */ + mfspr r3,SPRN_CCR0 + rlwinm r3,r3,0,0,27 /* disable icache prefetch */ + isync + mtspr SPRN_CCR0,r3 + isync + sync + +/* * Set up the initial MMU state * * We are still executing code at the virtual address@@ -570,7 +581,6 @@ finish_tlb_load: rlwimi r10,r12,29,30,30 /* DIRTY -> SW position */ and r11,r12,r10 /* Mask PTE bits to keep */ andi. r10,r12,_PAGE_USER /* User page ? */ - ori r11,r11,_PAGE_GUARDED /* 440 errata, needs G set */ beq 1f /* nope, leave U bits empty */ rlwimi r11,r11,3,26,28 /* yes, copy S bits to U */1: tlbwe r11,r13,PPC44x_TLB_ATTRIB /* Write ATTRIB */
Quoted 6 lines · Benjamin Herrenschmidt
This series of patches is aimed at supporting SMP on non-hash based processors. It consists of a rework of the MMU context management and TLB management, clearly splitting hash32, hash64 and nohash in both cases, adding SMP safe context handling and some basic SMP TLB management.
Quoted 9 lines
There is room for improvements, such as implementing lazy TLB flushing on processors without invalidate-by-PID support HW, some better IPI mechanism, lock less fast path in the MMU context switch, etc... but it should basically work.
This version addresses the issues raised about some of the patches in the previous batch, removes the ones that were already merged in -next and and rebases the remaining ones when needed.
All patches
Please put these in powerpc/next and I can work up some 85xx SMP support on top of them.
- k