powerpc/book3s64/mm: Don't do tlbie fixup for some hardware revisions
Subsystems (3)
PATCH 1/4 powerpc/book3s64/mm: Don't do tlbie fixup for some hardware revisions
The store ordering vs tlbie issue mentioned in
commit a5d4b5891c2f ("powerpc/mm: Fixup tlbie vs store ordering issue on POWER9") is fixed for Nimbus 2.3 and Cumulus 1.3 revisions. We don't need to apply the fixup if we are running on them
We can only do this on PowerNV. On pseries guest with kvm we still don't support redoing the feature fixup after migration. So we should be enabling all the workarounds needed, because whe can possibly migrate between DD 2.3 and DD 2.2
--- arch/powerpc/kernel/dt_cpu_ftrs.c | 30 ++++++++++++++++++++++++++++-- 1 file changed, 28 insertions(+), 2 deletions(-)
arch/powerpc/kernel/dt_cpu_ftrs.c +28 -2
@@ -691,9 +691,35 @@ static bool __init cpufeatures_process_feature(struct dt_cpu_feature *f)return true; } +/* +* Handle POWER9 broadcast tlbie invalidation issue using +* cpu feature flag. +*/ +static __init void update_tlbie_feature_flag (unsigned long pvr) +{ +if (PVR_VER (pvr) ==PVR_POWER9 ) { +/* +* Set the tlbie feature flag for anything below +* Nimbus DD 2.3 and Cumulus DD 1.3 +*/ +if ((pvr &0xe000 ) ==0 ) { +/* Nimbus */ +if ((pvr &0xfff ) <0x203 ) + cur_cpu_spec->cpu_features |=CPU_FTR_P9_TLBIE_BUG ; + }else if ((pvr &0xc000 ) ==0 ) { +/* Cumulus */ +if ((pvr &0xfff ) <0x103 ) + cur_cpu_spec->cpu_features |=CPU_FTR_P9_TLBIE_BUG ; + }else { +WARN_ONCE (1 ,"Unknown PVR" ); + cur_cpu_spec->cpu_features |=CPU_FTR_P9_TLBIE_BUG ; + } + } +} +static __init void cpufeatures_cpu_quirks (void ) { -int version =mfspr (SPRN_PVR ); +unsigned long version =mfspr (SPRN_PVR );/* * Not all quirks can be derived from the cpufeatures device tree.
@@ -712,10 +738,10 @@ static __init void cpufeatures_cpu_quirks(void)if ((version &0xffff0000 ) ==0x004e0000 ) { cur_cpu_spec->cpu_features &= ~(CPU_FTR_DAWR ); - cur_cpu_spec->cpu_features |=CPU_FTR_P9_TLBIE_BUG ; cur_cpu_spec->cpu_features |=CPU_FTR_P9_TIDR ; } +update_tlbie_feature_flag (version);/* * PKEY was not in the initial base or feature node * specification, but it should become optional in the next
Signature · 1 line
2.21.0
PATCH 2/4 powerpc/book3s64/radix: Rename CPU_FTR_P9_TLBIE_BUG feature flag
Rename the #define to indicate this is related to store vs tlbie ordering issue. In the next patch, we will be adding another feature flag that is used to handles ERAT flush vs tlbie ordering issue.
--- arch/powerpc/include/asm/cputable.h | 4 ++-- arch/powerpc/kernel/dt_cpu_ftrs.c | 6 +++--- arch/powerpc/kvm/book3s_hv_rm_mmu.c | 2 +- arch/powerpc/mm/book3s64/hash_native.c | 2 +- arch/powerpc/mm/book3s64/radix_tlb.c | 4 ++-- 5 files changed, 9 insertions(+), 9 deletions(-)
arch/powerpc/include/asm/cputable.h +2 -2
@@ -209,7 +209,7 @@ static inline void cpu_feature_keys_init(void) { }#define CPU_FTR_POWER9_DD2_1 LONG_ASM_CONST(0x0000080000000000)#define CPU_FTR_P9_TM_HV_ASSIST LONG_ASM_CONST(0x0000100000000000)#define CPU_FTR_P9_TM_XER_SO_BUG LONG_ASM_CONST(0x0000200000000000) -#define CPU_FTR_P9_TLBIE_BUG LONG_ASM_CONST(0x0000400000000000) +#define CPU_FTR_P9_TLBIE_STQ_BUG LONG_ASM_CONST(0x0000400000000000)#define CPU_FTR_P9_TIDR LONG_ASM_CONST(0x0000800000000000)#ifndef __ASSEMBLY__
@@ -457,7 +457,7 @@ static inline void cpu_feature_keys_init(void) { } CPU_FTR_CFAR | CPU_FTR_HVMODE | CPU_FTR_VMX_COPY | \ CPU_FTR_DBELL | CPU_FTR_HAS_PPR | CPU_FTR_ARCH_207S | \ CPU_FTR_TM_COMP | CPU_FTR_ARCH_300 | CPU_FTR_PKEY | \ - CPU_FTR_P9_TLBIE_BUG | CPU_FTR_P9_TIDR) + CPU_FTR_P9_TLBIE_STQ_BUG | CPU_FTR_P9_TIDR)#define CPU_FTRS_POWER9_DD2_0 CPU_FTRS_POWER9#define CPU_FTRS_POWER9_DD2_1 (CPU_FTRS_POWER9 | CPU_FTR_POWER9_DD2_1)#define CPU_FTRS_POWER9_DD2_2 (CPU_FTRS_POWER9 | CPU_FTR_POWER9_DD2_1 | \
arch/powerpc/kernel/dt_cpu_ftrs.c +3 -3
@@ -705,14 +705,14 @@ static __init void update_tlbie_feature_flag(unsigned long pvr)if ((pvr &0xe000 ) ==0 ) {/* Nimbus */ if ((pvr &0xfff ) <0x203 ) - cur_cpu_spec->cpu_features |=CPU_FTR_P9_TLBIE_BUG ; + cur_cpu_spec->cpu_features |=CPU_FTR_P9_TLBIE_STQ_BUG ; }else if ((pvr &0xc000 ) ==0 ) {/* Cumulus */ if ((pvr &0xfff ) <0x103 ) - cur_cpu_spec->cpu_features |=CPU_FTR_P9_TLBIE_BUG ; + cur_cpu_spec->cpu_features |=CPU_FTR_P9_TLBIE_STQ_BUG ; }else {WARN_ONCE (1 ,"Unknown PVR" ); - cur_cpu_spec->cpu_features |=CPU_FTR_P9_TLBIE_BUG ; + cur_cpu_spec->cpu_features |=CPU_FTR_P9_TLBIE_STQ_BUG ; } } }
arch/powerpc/kvm/book3s_hv_rm_mmu.c +1 -1
@@ -451,7 +451,7 @@ static void do_tlbies(struct kvm *kvm, unsigned long *rbvalues, "r" (rbvalues[i]), "r" (kvm->arch.lpid)); } -if (cpu_has_feature (CPU_FTR_P9_TLBIE_BUG )) { +if (cpu_has_feature (CPU_FTR_P9_TLBIE_STQ_BUG )) {/* * Need the extra ptesync to make sure we don't * re-order the tlbie
arch/powerpc/mm/book3s64/hash_native.c +1 -1
@@ -199,7 +199,7 @@ static inline unsigned long ___tlbie(unsigned long vpn, int psize,static inline void fixup_tlbie (unsigned long vpn,int psize,int apsize,int ssize) { -if (cpu_has_feature (CPU_FTR_P9_TLBIE_BUG )) { +if (cpu_has_feature (CPU_FTR_P9_TLBIE_STQ_BUG )) {/* Need the extra ptesync to ensure we don't reorder tlbie*/ asmvolatile ("ptesync" : : :"memory" );___tlbie (vpn, psize, apsize, ssize);
arch/powerpc/mm/book3s64/radix_tlb.c +2 -2
@@ -201,7 +201,7 @@ static inline void fixup_tlbie(void)unsigned long pid =0 ;unsigned long va = ((1UL <<52 ) -1 ); -if (cpu_has_feature (CPU_FTR_P9_TLBIE_BUG )) { +if (cpu_has_feature (CPU_FTR_P9_TLBIE_STQ_BUG )) { asmvolatile ("ptesync" : : :"memory" );__tlbie_va (va, pid,mmu_get_ap (MMU_PAGE_64K ),RIC_FLUSH_TLB ); }
@@ -211,7 +211,7 @@ static inline void fixup_tlbie_lpid(unsigned long lpid) {unsigned long va = ((1UL <<52 ) -1 ); -if (cpu_has_feature (CPU_FTR_P9_TLBIE_BUG )) { +if (cpu_has_feature (CPU_FTR_P9_TLBIE_STQ_BUG )) { asmvolatile ("ptesync" : : :"memory" );__tlbie_lpid_va (va, lpid,mmu_get_ap (MMU_PAGE_64K ),RIC_FLUSH_TLB ); }
Signature · 1 line
2.21.0
PATCH 4/4 powerpc/mm/selftest: Add test case for tlbie vs mtpidr ordering issue
In reply to Aneesh Kumar K.V--- tools/testing/selftests/powerpc/mm/Makefile | 3 + .../testing/selftests/powerpc/mm/tlbie_test.c | 739 ++++++++++++++++++ 2 files changed, 742 insertions(+) create mode 100644 tools/testing/selftests/powerpc/mm/tlbie_test.c
tools/testing/selftests/powerpc/mm/Makefile +3 -0
@@ -4,6 +4,7 @@ noarg: TEST_GEN_PROGS := hugetlb_vs_thp_test subpage_prot prot_sao segv_errors wild_bctr \ large_vm_fork_separation +TEST_GEN_PROGS_EXTENDED=tlbie_test TEST_GEN_FILES := tempfile top_srcdir = ../../../../..
@@ -19,3 +20,5 @@ $(OUTPUT)/large_vm_fork_separation: CFLAGS += -m64 $(OUTPUT)/tempfile: dd if=/dev/zero of=$@ bs=64k count=1 +$(OUTPUT)/tlbie_test: tlbie_test.c + cc -o $@ $^ -lpthread
tools/testing/selftests/powerpc/mm/tlbie_test.c new file +739 -0
739 lines not on this page · raw message
Signature · 1 line
2.21.0
PATCH 3/4 powerpc/mm: Fixup tlbie vs mtpidr/mtlpidr ordering issue on POWER9
In reply to Aneesh Kumar K.VOn POWER9, under some circumstances, a broadcast TLB invalidation will fail to invalidate the ERAT cache on some threads when there are parallel mtpidr/mtlpidr happening on other threads of the same core. This can cause stores to continue to go to a page after it's unmapped.
The workaround is to force an ERAT flush using PID=0 or LPID=0 tlbie flush. This additional TLB flush will cause the ERAT cache invalidation. Since we are using PID=0 or LPID=0, we don't get filtered out by the TLB snoop filtering logic.
We need to still follow this up with another tlbie to take care of store vs tlbie ordering issue explained in commit: a5d4b5891c2f ("powerpc/mm: Fixup tlbie vs store ordering issue on POWER9"). The presence of ERAT cache implies we can still get new stores and they may miss store queue marking flush.
--- arch/powerpc/include/asm/cputable.h | 3 +- arch/powerpc/kernel/dt_cpu_ftrs.c | 2 + arch/powerpc/kvm/book3s_hv_rm_mmu.c | 42 +++++++++++---- arch/powerpc/mm/book3s64/hash_native.c | 29 ++++++++-- arch/powerpc/mm/book3s64/radix_tlb.c | 74 +++++++++++++++++++++++--- 5 files changed, 128 insertions(+), 22 deletions(-)
arch/powerpc/include/asm/cputable.h +2 -1
@@ -211,6 +211,7 @@ static inline void cpu_feature_keys_init(void) { }#define CPU_FTR_P9_TM_XER_SO_BUG LONG_ASM_CONST(0x0000200000000000)#define CPU_FTR_P9_TLBIE_STQ_BUG LONG_ASM_CONST(0x0000400000000000)#define CPU_FTR_P9_TIDR LONG_ASM_CONST(0x0000800000000000) +#define CPU_FTR_P9_TLBIE_ERAT_BUG LONG_ASM_CONST(0x0001000000000000)#ifndef __ASSEMBLY__
@@ -457,7 +458,7 @@ static inline void cpu_feature_keys_init(void) { } CPU_FTR_CFAR | CPU_FTR_HVMODE | CPU_FTR_VMX_COPY | \ CPU_FTR_DBELL | CPU_FTR_HAS_PPR | CPU_FTR_ARCH_207S | \ CPU_FTR_TM_COMP | CPU_FTR_ARCH_300 | CPU_FTR_PKEY | \ - CPU_FTR_P9_TLBIE_STQ_BUG | CPU_FTR_P9_TIDR) + CPU_FTR_P9_TLBIE_STQ_BUG | CPU_FTR_P9_TLBIE_ERAT_BUG | CPU_FTR_P9_TIDR)#define CPU_FTRS_POWER9_DD2_0 CPU_FTRS_POWER9#define CPU_FTRS_POWER9_DD2_1 (CPU_FTRS_POWER9 | CPU_FTR_POWER9_DD2_1)#define CPU_FTRS_POWER9_DD2_2 (CPU_FTRS_POWER9 | CPU_FTR_POWER9_DD2_1 | \
arch/powerpc/kernel/dt_cpu_ftrs.c +2 -0
@@ -714,6 +714,8 @@ static __init void update_tlbie_feature_flag(unsigned long pvr)WARN_ONCE (1 ,"Unknown PVR" ); cur_cpu_spec->cpu_features |=CPU_FTR_P9_TLBIE_STQ_BUG ; } + + cur_cpu_spec->cpu_features |=CPU_FTR_P9_TLBIE_ERAT_BUG ; } }
arch/powerpc/kvm/book3s_hv_rm_mmu.c +32 -10
54 lines not on this page · raw message
arch/powerpc/mm/book3s64/hash_native.c +26 -3
@@ -197,8 +197,31 @@ static inline unsigned long ___tlbie(unsigned long vpn, int psize,return va; } -static inline void fixup_tlbie (unsigned long vpn,int psize,int apsize,int ssize) +static inline void fixup_tlbie_vpn (unsigned long vpn,int psize, +int apsize,int ssize) { +if (cpu_has_feature (CPU_FTR_P9_TLBIE_ERAT_BUG )) { +/* Radix flush for a hash guest */ + +unsigned long rb,rs,prs,r,ric; + + rb =PPC_BIT (52 );/* IS = 2 */ + rs =0 ;/* lpid = 0 */ + prs =0 ;/* partition scoped */ + r =1 ;/* radix format */ + ric =0 ;/* RIC_FLSUH_TLB */ + +/* +* Need the extra ptesync to make sure we don't +* re-order the tlbie +*/ + asmvolatile ("ptesync" : : :"memory" ); + asm volatile(PPC_TLBIE_5(%0, %4, %3, %2, %1) + : : "r"(rb), "i"(r), "i"(prs), + "i"(ric), "r"(rs) : "memory"); + } + +if (cpu_has_feature (CPU_FTR_P9_TLBIE_STQ_BUG )) {/* Need the extra ptesync to ensure we don't reorder tlbie*/ asmvolatile ("ptesync" : : :"memory" );
@@ -283,7 +306,7 @@ static inline void tlbie(unsigned long vpn, int psize, int apsize, asmvolatile ("ptesync" : : :"memory" ); }else {__tlbie (vpn, psize, apsize, ssize); -fixup_tlbie (vpn, psize, apsize, ssize); +fixup_tlbie_vpn (vpn, psize, apsize, ssize); asmvolatile ("eieio; tlbsync; ptesync" : : :"memory" ); }if (lock_tlbie && !use_local)
@@ -856,7 +879,7 @@ static void native_flush_hash_range(unsigned long number, int local)/* * Just do one more with the last used values. */ -fixup_tlbie (vpn, psize, psize, ssize); +fixup_tlbie_vpn (vpn, psize, psize, ssize); asmvolatile ("eieio; tlbsync; ptesync" :::"memory" );if (lock_tlbie)
arch/powerpc/mm/book3s64/radix_tlb.c +66 -8
149 lines not on this page · raw message
Signature · 1 line
2.21.0
Quoted 11 lines · Aneesh Kumar K.V
The store ordering vs tlbie issue mentioned in
commita5d4b5891c2f("powerpc/mm: Fixup tlbie vs store ordering issue on POWER9") is fixed for Nimbus 2.3 and Cumulus 1.3 revisions. We don't need to apply the fixup if we are running on themWe can only do this on PowerNV. On pseries guest with kvm we still don't support redoing the feature fixup after migration. So we should be enabling all the workarounds needed, because whe can possibly migrate between DD 2.3 and DD 2.2
Signed-off-by Aneesh Kumar K.V <[email protected]>
Series applied to powerpc fixes, thanks.
https://git.kernel.org/powerpc/c/677733e296b5c7a37c47da391fc70a43dc40bd67
cheers