powerpc/64s: Fix local TLB flush (for init and MCE) on POWER9
Subsystems (2)
Nicholas Piggin
reviewerPATCH 1/1 powerpc/64s: Fix local TLB flush (for init and MCE) on POWER9
There are two cases outside the normal address space management where a CPU's local TLB is to be flushed:
1. Host boot; in case something has left stale entries in the
TLB (e.g., kexec).
2. Host machine check; to clean corrupted TLB entries.
CPU state restore from an idle state incurring full state loss also flushes the TLB, which seems to be a side effect of using the same code as #1 rather than a requirement itself.
The problem with these are they are coded inflexibly, several times for each CPU type, and they also don't take into account the radix mode of the MMU in POWER9 (tlbiel instructions with R parameter not matching the radix mode cause undefined behaviour according to ISA v3.0B).
- tlbiel is undefined if R does not match the current radix mode. This could happen in machine check and cpu_restore from non-boot code (idle restore and OPAL call bug workaround).
- ISAv3.0B hash mode should also be flushing the partition and process table caches.
- Likewise for radix mode, and radix should also be flushing page walk cache.
Move all that flushing code to C, move it into the mm/ directory, and add ISAv3 cases for radix and hash. Take it out from the early cputable detection hooks and go by ISA level checks instead.
The boot process calls this flush on each CPU after it has set up all its MMU registers, and right before switching on relocation.
TLB flush is no longer called when restoring from idle states.
This could use some review.
--- arch/powerpc/include/asm/book3s/64/tlbflush-hash.h | 1 + .../powerpc/include/asm/book3s/64/tlbflush-radix.h | 3 + arch/powerpc/include/asm/book3s/64/tlbflush.h | 22 ++++++ arch/powerpc/include/asm/cputable.h | 12 ---- arch/powerpc/kernel/cpu_setup_power.S | 43 ------------ arch/powerpc/kernel/cputable.c | 14 ---- arch/powerpc/kernel/mce_power.c | 61 +--------------- arch/powerpc/kvm/book3s_hv_ras.c | 6 +- arch/powerpc/mm/hash_native_64.c | 82 ++++++++++++++++++++++ arch/powerpc/mm/hash_utils_64.c | 4 ++ arch/powerpc/mm/pgtable-radix.c | 4 ++ arch/powerpc/mm/tlb-radix.c | 57 +++++++++++++++ 12 files changed, 177 insertions(+), 132 deletions(-)
arch/powerpc/include/asm/book3s/64/tlbflush-hash.h +1 -0
@@ -50,6 +50,7 @@ static inline void arch_leave_lazy_mmu_mode(void)#define arch_flush_lazy_mmu_mode () do {} while (0) +extern void hash__tlbiel_all (unsigned int action);extern void flush_hash_page (unsigned long vpn,real_pte_t pte,int psize,int ssize,unsigned long flags);
arch/powerpc/include/asm/book3s/64/tlbflush-radix.h +3 -0
@@ -10,6 +10,8 @@ static inline int mmu_get_ap(int psize)return mmu_psize_defs[psize].ap; } +extern void radix__tlbiel_all (unsigned int action); +extern void radix__flush_hugetlb_tlb_range (struct vm_area_struct *vma,unsigned long start,unsigned long end); extern void radix__flush_tlb_range_psize(struct mm_struct *mm, unsigned long start,
@@ -44,4 +46,5 @@ extern void radix__flush_tlb_lpid(unsigned long lpid);extern void radix__flush_tlb_all (void );extern void radix__flush_tlb_pte_p9_dd1 (unsigned long old_pte,struct mm_struct *mm,unsigned long address); +#endif
arch/powerpc/include/asm/book3s/64/tlbflush.h +22 -0
@@ -7,6 +7,28 @@#include <asm/book3s/64/tlbflush-hash.h> #include <asm/book3s/64/tlbflush-radix.h> +/* TLB flush actions. Used as argument to tlbiel_all() */ +enum { +TLB_INVAL_SCOPE_GLOBAL =0 ,/* invalidate all TLBs */ +TLB_INVAL_SCOPE_LPID =1 ,/* invalidate TLBs for current LPID */ +}; + +static inline void tlbiel_all (unsigned int action) +{ +/* +* This is used for host machine check and bootup. +* +* This could be reimplemented more robustly without using the +* radix_is_enabled(), cpu_feature(), etc. calls. However these +* should be set up before relocation starts to be used at boot, +* so we shouldn't see TLB machine checks before then. +*/ +if (radix_enabled ()) +radix__tlbiel_all (action); +else +hash__tlbiel_all (action); +} +#define __HAVE_ARCH_FLUSH_PMD_TLB_RANGEstatic inline void flush_pmd_tlb_range (struct vm_area_struct *vma,unsigned long start,unsigned long end)
arch/powerpc/include/asm/cputable.h +0 -12
@@ -106,12 +106,6 @@ struct cpu_spec {* called in real mode to handle SLB and TLB errors. */ long (*machine_check_early)(struct pt_regs *regs); - -/* -* Processor specific routine to flush tlbs. -*/ -void (*flush_tlb)(unsigned int action); - };extern struct cpu_spec *cur_cpu_spec;
@@ -130,12 +124,6 @@ extern void cpu_feature_keys_init(void);static inline void cpu_feature_keys_init (void ) { }#endif -/* TLB flush actions. Used as argument to cpu_spec.flush_tlb() hook */ -enum { -TLB_INVAL_SCOPE_GLOBAL =0 ,/* invalidate all TLBs */ -TLB_INVAL_SCOPE_LPID =1 ,/* invalidate TLBs for current LPID */ -}; -#endif /* __ASSEMBLY__ */ /* CPU kernel features */
arch/powerpc/kernel/cpu_setup_power.S +0 -43
85 lines not on this page · raw message
arch/powerpc/kernel/cputable.c +0 -14
@@ -72,9 +72,6 @@ extern void __setup_cpu_power8(unsigned long offset, struct cpu_spec* spec);extern void __restore_cpu_power8 (void );extern void __setup_cpu_power9 (unsigned long offset,struct cpu_spec * spec);extern void __restore_cpu_power9 (void ); -extern void __flush_tlb_power7 (unsigned int action); -extern void __flush_tlb_power8 (unsigned int action); -extern void __flush_tlb_power9 (unsigned int action);extern long __machine_check_early_realmode_p7 (struct pt_regs *regs);extern long __machine_check_early_realmode_p8 (struct pt_regs *regs);extern long __machine_check_early_realmode_p9 (struct pt_regs *regs);
@@ -365,7 +362,6 @@ static struct cpu_spec __initdata cpu_specs[] = { .oprofile_cpu_type = "ppc64/ibm-compat-v1", .cpu_setup = __setup_cpu_power7, .cpu_restore = __restore_cpu_power7, - .flush_tlb = __flush_tlb_power7, .machine_check_early = __machine_check_early_realmode_p7, .platform = "power7", },
@@ -383,7 +379,6 @@ static struct cpu_spec __initdata cpu_specs[] = { .oprofile_cpu_type = "ppc64/ibm-compat-v1", .cpu_setup = __setup_cpu_power8, .cpu_restore = __restore_cpu_power8, - .flush_tlb = __flush_tlb_power8, .machine_check_early = __machine_check_early_realmode_p8, .platform = "power8", },
@@ -401,7 +396,6 @@ static struct cpu_spec __initdata cpu_specs[] = { .oprofile_cpu_type = "ppc64/ibm-compat-v1", .cpu_setup = __setup_cpu_power9, .cpu_restore = __restore_cpu_power9, - .flush_tlb = __flush_tlb_power9, .platform = "power9", }, {/* Power7 */
@@ -420,7 +414,6 @@ static struct cpu_spec __initdata cpu_specs[] = { .oprofile_type = PPC_OPROFILE_POWER4, .cpu_setup = __setup_cpu_power7, .cpu_restore = __restore_cpu_power7, - .flush_tlb = __flush_tlb_power7, .machine_check_early = __machine_check_early_realmode_p7, .platform = "power7", },
@@ -440,7 +433,6 @@ static struct cpu_spec __initdata cpu_specs[] = { .oprofile_type = PPC_OPROFILE_POWER4, .cpu_setup = __setup_cpu_power7, .cpu_restore = __restore_cpu_power7, - .flush_tlb = __flush_tlb_power7, .machine_check_early = __machine_check_early_realmode_p7, .platform = "power7+", },
@@ -460,7 +452,6 @@ static struct cpu_spec __initdata cpu_specs[] = { .oprofile_type = PPC_OPROFILE_INVALID, .cpu_setup = __setup_cpu_power8, .cpu_restore = __restore_cpu_power8, - .flush_tlb = __flush_tlb_power8, .machine_check_early = __machine_check_early_realmode_p8, .platform = "power8", },
@@ -480,7 +471,6 @@ static struct cpu_spec __initdata cpu_specs[] = { .oprofile_type = PPC_OPROFILE_INVALID, .cpu_setup = __setup_cpu_power8, .cpu_restore = __restore_cpu_power8, - .flush_tlb = __flush_tlb_power8, .machine_check_early = __machine_check_early_realmode_p8, .platform = "power8", },
@@ -500,7 +490,6 @@ static struct cpu_spec __initdata cpu_specs[] = { .oprofile_type = PPC_OPROFILE_INVALID, .cpu_setup = __setup_cpu_power8, .cpu_restore = __restore_cpu_power8, - .flush_tlb = __flush_tlb_power8, .machine_check_early = __machine_check_early_realmode_p8, .platform = "power8", },
@@ -520,7 +509,6 @@ static struct cpu_spec __initdata cpu_specs[] = { .oprofile_type = PPC_OPROFILE_INVALID, .cpu_setup = __setup_cpu_power8, .cpu_restore = __restore_cpu_power8, - .flush_tlb = __flush_tlb_power8, .machine_check_early = __machine_check_early_realmode_p8, .platform = "power8", },
@@ -540,7 +528,6 @@ static struct cpu_spec __initdata cpu_specs[] = { .oprofile_type = PPC_OPROFILE_INVALID, .cpu_setup = __setup_cpu_power9, .cpu_restore = __restore_cpu_power9, - .flush_tlb = __flush_tlb_power9, .machine_check_early = __machine_check_early_realmode_p9, .platform = "power9", },
@@ -560,7 +547,6 @@ static struct cpu_spec __initdata cpu_specs[] = { .oprofile_type = PPC_OPROFILE_INVALID, .cpu_setup = __setup_cpu_power9, .cpu_restore = __restore_cpu_power9, - .flush_tlb = __flush_tlb_power9, .machine_check_early = __machine_check_early_realmode_p9, .platform = "power9", },
arch/powerpc/kernel/mce_power.c +2 -59
73 lines not on this page · raw message
arch/powerpc/kvm/book3s_hv_ras.c +2 -4
@@ -87,8 +87,7 @@ static long kvmppc_realmode_mc_power7(struct kvm_vcpu *vcpu) DSISR_MC_SLB_PARITY | DSISR_MC_DERAT_MULTI); }if (dsisr &DSISR_MC_TLB_MULTI ) { -if (cur_cpu_spec && cur_cpu_spec->flush_tlb) - cur_cpu_spec->flush_tlb (TLB_INVAL_SCOPE_LPID ); +tlbiel_all (TLB_INVAL_SCOPE_LPID ); dsisr &= ~DSISR_MC_TLB_MULTI ; }/* Any other errors we don't understand? */
@@ -105,8 +104,7 @@ static long kvmppc_realmode_mc_power7(struct kvm_vcpu *vcpu)reload_slb (vcpu);break ;case SRR1_MC_IFETCH_TLBMULTI : -if (cur_cpu_spec && cur_cpu_spec->flush_tlb) - cur_cpu_spec->flush_tlb (TLB_INVAL_SCOPE_LPID ); +tlbiel_all (TLB_INVAL_SCOPE_LPID );break ;default : handled =0 ;
arch/powerpc/mm/hash_native_64.c +82 -0
88 lines not on this page · raw message
arch/powerpc/mm/hash_utils_64.c +4 -0
@@ -1044,6 +1044,8 @@ void __init hash__early_init_mmu(void)pr_info ("Initializing hash mmu with SLB\n" );/* Initialize SLB management */ slb_initialize (); + +tlbiel_all (TLB_INVAL_SCOPE_GLOBAL ); }#ifdef CONFIG_SMP
@@ -1063,6 +1065,8 @@ void hash__early_init_mmu_secondary(void) }/* Initialize SLB */ slb_initialize (); + +tlbiel_all (TLB_INVAL_SCOPE_GLOBAL ); }#endif /* CONFIG_SMP */
arch/powerpc/mm/pgtable-radix.c +4 -0
@@ -426,6 +426,8 @@ void __init radix__early_init_mmu(void)radix_init_iamr ();radix_init_pgtable (); + +tlbiel_all (TLB_INVAL_SCOPE_GLOBAL ); }void radix__early_init_mmu_secondary (void )
@@ -447,6 +449,8 @@ void radix__early_init_mmu_secondary(void)radix_init_amor (); }radix_init_iamr (); + +tlbiel_all (TLB_INVAL_SCOPE_GLOBAL ); }void radix__mmu_cleanup_all (void )
arch/powerpc/mm/tlb-radix.c +57 -0
63 lines not on this page · raw message
Signature · 1 line
2.11.0