powerpc: untangle cputable mce include
Nicholas Piggin
reviewerPATCH v2 1/2 powerpc: untangle cputable mce include
Having cputable.h include mce.h means it pulls in a bunch of low level headers (e.g., synch.h) which then can't use CPU_FTR_ definitions.
--- arch/powerpc/include/asm/cputable.h | 5 ----- arch/powerpc/kernel/cputable.c | 1 + arch/powerpc/kernel/dt_cpu_ftrs.c | 1 + 3 files changed, 2 insertions(+), 5 deletions(-)
arch/powerpc/include/asm/cputable.h +0 -5
@@ -9,11 +9,6 @@#ifndef __ASSEMBLY__ -/* -* Added to include __machine_check_early_realmode_* functions -*/ -#include <asm/mce.h> -/* This structure can grow, it's real size is used by head.S code * via the mkdefs mechanism. */
arch/powerpc/kernel/cputable.c +1 -0
@@ -16,6 +16,7 @@#include <asm/oprofile_impl.h> #include <asm/cputable.h> #include <asm/prom.h> /* for PTRRELOC on ARCH=ppc */ +#include <asm/mce.h> #include <asm/mmu.h> #include <asm/setup.h>
arch/powerpc/kernel/dt_cpu_ftrs.c +1 -0
@@ -17,6 +17,7 @@#include <asm/cputable.h> #include <asm/dt_cpu_ftrs.h> +#include <asm/mce.h> #include <asm/mmu.h> #include <asm/oprofile_impl.h> #include <asm/prom.h>
Signature · 1 line
2.23.0
Nicholas Piggin
reviewerPATCH v2 2/2 powerpc/64s: Add cp_abort after tlbiel to invalidate copy-buffer address
The copy buffer is implemented as a real address in the nest which is translated from EA by copy, and used for memory access by paste. This requires that it be invalidated by TLB invalidation.
TLBIE does invalidate the copy buffer, but TLBIEL does not. Add cp_abort to the tlbiel sequence.
---
v2:
- Untangle headers that were causing build failures.
- Improve the comment a bit.
- Exempt POWER9 from the workaround, as described by the comment (we worked this out already but I forgot about it when doing v1!)
arch/powerpc/include/asm/synch.h | 19 ++++++++++++++++++- arch/powerpc/mm/book3s64/hash_native.c | 8 ++++---- arch/powerpc/mm/book3s64/radix_tlb.c | 12 ++++++------ 3 files changed, 28 insertions(+), 11 deletions(-)
arch/powerpc/include/asm/synch.h +18 -1
@@ -3,8 +3,9 @@#define _ASM_POWERPC_SYNCH_H #ifdef __KERNEL__ +#include <asm/cputable.h> #include <asm/feature-fixups.h> -#include <asm/asm-const.h> +#include <asm/ppc-opcode.h> #ifndef __ASSEMBLY__extern unsigned int __start___lwsync_fixup, __stop___lwsync_fixup;
@@ -20,6 +21,22 @@ static inline void isync(void) { __asm__ __volatile__ ("isync" : : : "memory"); } + +static inline void ppc_after_tlbiel_barrier(void) +{ + asm volatile("ptesync": : :"memory"); + /* + * POWER9, POWER10 need a cp_abort after tlbiel to ensure the copy + * is invalidated correctly. If this is not done, the paste can take + * data from the physical address that was translated at copy time. + * + * POWER9 in practice does not need this, because address spaces + * with accelerators mapped will use tlbie (which does invalidate + * the copy) to invalidate translations. It's not possible to limit + * POWER10 this way due to local copy-paste. + */ + asm volatile(ASM_FTR_IFSET(PPC_CP_ABORT, "", %0) : : "i" (CPU_FTR_ARCH_31) : "memory"); +} #endif /* __ASSEMBLY__ */ #if defined(__powerpc64__)
arch/powerpc/mm/book3s64/hash_native.c +4 -4
@@ -82,7 +82,7 @@ static void tlbiel_all_isa206(unsigned int num_sets, unsigned int is)for (set =0 ; set < num_sets; set++)tlbiel_hash_set_isa206 (set, is); - asmvolatile ("ptesync" : : :"memory" ); +ppc_after_tlbiel_barrier (); }static void tlbiel_all_isa300 (unsigned int num_sets,unsigned int is)
@@ -110,7 +110,7 @@ static void tlbiel_all_isa300(unsigned int num_sets, unsigned int is)*/ tlbiel_hash_set_isa300 (0 , is,0 ,2 ,1 ); - asmvolatile ("ptesync" : : :"memory" ); +ppc_after_tlbiel_barrier (); asmvolatile (PPC_ISA_3_0_INVALIDATE_ERAT "; isync" : : :"memory" ); }
@@ -303,7 +303,7 @@ static inline void tlbie(unsigned long vpn, int psize, int apsize, asmvolatile ("ptesync" : : :"memory" );if (use_local) {__tlbiel (vpn, psize, apsize, ssize); - asmvolatile ("ptesync" : : :"memory" ); +ppc_after_tlbiel_barrier (); }else {__tlbie (vpn, psize, apsize, ssize);fixup_tlbie_vpn (vpn, psize, apsize, ssize);
@@ -879,7 +879,7 @@ static void native_flush_hash_range(unsigned long number, int local)__tlbiel (vpn, psize, psize, ssize); }pte_iterate_hashed_end (); } - asmvolatile ("ptesync" :::"memory" ); +ppc_after_tlbiel_barrier (); }else {int lock_tlbie = !mmu_has_feature (MMU_FTR_LOCKLESS_TLBIE );
arch/powerpc/mm/book3s64/radix_tlb.c +6 -6
@@ -65,7 +65,7 @@ static void tlbiel_all_isa300(unsigned int num_sets, unsigned int is)for (set =1 ; set < num_sets; set++)tlbiel_radix_set_isa300 (set, is,0 ,RIC_FLUSH_TLB ,1 ); - asmvolatile ("ptesync" : : :"memory" ); +ppc_after_tlbiel_barrier (); }void radix__tlbiel_all (unsigned int action)
@@ -296,7 +296,7 @@ static __always_inline void _tlbiel_pid(unsigned long pid, unsigned long ric)/* For PWC, only one flush is needed */ if (ric ==RIC_FLUSH_PWC ) { - asmvolatile ("ptesync" : : :"memory" ); +ppc_after_tlbiel_barrier ();return ; }
@@ -304,7 +304,7 @@ static __always_inline void _tlbiel_pid(unsigned long pid, unsigned long ric)for (set =1 ; set <POWER9_TLB_SETS_RADIX ; set++)__tlbiel_pid (pid, set,RIC_FLUSH_TLB ); - asmvolatile ("ptesync" : : :"memory" ); +ppc_after_tlbiel_barrier (); asmvolatile (PPC_RADIX_INVALIDATE_ERAT_USER "; isync" : : :"memory" ); }
@@ -431,7 +431,7 @@ static __always_inline void _tlbiel_va(unsigned long va, unsigned long pid, asmvolatile ("ptesync" : : :"memory" );__tlbiel_va (va, pid, ap, ric); - asmvolatile ("ptesync" : : :"memory" ); +ppc_after_tlbiel_barrier (); } static inline void _tlbiel_va_range(unsigned long start, unsigned long end,
@@ -442,7 +442,7 @@ static inline void _tlbiel_va_range(unsigned long start, unsigned long end,if (also_pwc)__tlbiel_pid (pid,0 ,RIC_FLUSH_PWC );__tlbiel_va_range (start, end, pid, page_size, psize); - asmvolatile ("ptesync" : : :"memory" ); +ppc_after_tlbiel_barrier (); } static inline void __tlbie_va_range(unsigned long start, unsigned long end,
@@ -940,7 +940,7 @@ static inline void __radix__flush_tlb_range(struct mm_struct *mm,if (hflush)__tlbiel_va_range (hstart, hend, pid,PMD_SIZE ,MMU_PAGE_2M ); - asmvolatile ("ptesync" : : :"memory" ); +ppc_after_tlbiel_barrier (); }else if (cputlb_use_tlbie ()) { asmvolatile ("ptesync" : : :"memory" );__tlbie_va_range (start, end, pid, page_size, mmu_virtual_psize);
Signature · 1 line
2.23.0
Michael Ellerman
AppliedQuoted 2 lines · Nicholas Piggin
Having
cputable.hincludemce.hmeans it pulls in a bunch of low level headers (e.g.,synch.h) which then can't useCPU_FTR_definitions.
Applied to powerpc/next.
[1/2] powerpc: untangle cputable mce include
https://git.kernel.org/powerpc/c/9983efa83b0a98da33807ccf5c047e204fdcac4c [2/2] powerpc/64s: Add cp_abort after tlbiel to invalidate copy-buffer address https://git.kernel.org/powerpc/c/05504b42562066ae27ce3e7dcec37f81dea476cb
cheers