Making tlbie optional for radix
Subsystems (5)
Nicholas Piggin
reviewerPATCH 2/6 powerpc/64s/radix: tidy up TLB flushing code
In reply to Nicholas PigginThere should be no functional changes.
- Use calls to existing radix_tlb.c functions in flush_partition.
- Rename radix__flush_tlb_lpid to radix__flush_all_lpid and similar, because they flush everything, matching flush_all_mm rather than flush_tlb_mm for the lpid.
- Remove some unused radix_tlb.c flush primitives.
Signed-off: Nicholas Piggin <[email protected]>
--- .../include/asm/book3s/64/tlbflush-radix.h | 12 +- arch/powerpc/kvm/book3s_hv_nested.c | 2 +- arch/powerpc/mm/book3s64/pgtable.c | 13 +- arch/powerpc/mm/book3s64/radix_tlb.c | 117 ++++-------------- 4 files changed, 34 insertions(+), 110 deletions(-)
arch/powerpc/include/asm/book3s/64/tlbflush-radix.h +3 -9
@@ -17,8 +17,8 @@ extern void radix__flush_tlb_lpid_page(unsigned int lpid,unsigned long addr, unsigned long page_size);extern void radix__flush_pwc_lpid (unsigned int lpid); -extern void radix__flush_tlb_lpid (unsigned int lpid); -extern void radix__local_flush_tlb_lpid_guest (unsigned int lpid); +extern void radix__flush_all_lpid (unsigned int lpid); +extern void radix__flush_all_lpid_guest (unsigned int lpid);#else static inline void radix__tlbiel_all (unsigned int action) {WARN_ON (1 ); }; static inline void radix__flush_tlb_lpid_page(unsigned int lpid,
@@ -31,11 +31,7 @@ static inline void radix__flush_pwc_lpid(unsigned int lpid) {WARN_ON (1 ); } -static inline void radix__flush_tlb_lpid (unsigned int lpid) -{ -WARN_ON (1 ); -} -static inline void radix__local_flush_tlb_lpid_guest (unsigned int lpid) +static inline void radix__flush_all_lpid (unsigned int lpid) {WARN_ON (1 ); }
@@ -73,6 +69,4 @@ extern void radix__flush_tlb_pwc(struct mmu_gather *tlb, unsigned long addr);extern void radix__flush_tlb_collapsed_pmd (struct mm_struct *mm,unsigned long addr);extern void radix__flush_tlb_all (void ); -extern void radix__local_flush_tlb_lpid (unsigned int lpid); -#endif
arch/powerpc/kvm/book3s_hv_nested.c +1 -1
@@ -398,7 +398,7 @@ static void kvmhv_flush_lpid(unsigned int lpid)long rc;if (!kvmhv_on_pseries ()) { -radix__flush_tlb_lpid (lpid); +radix__flush_all_lpid (lpid);return ; }
arch/powerpc/mm/book3s64/pgtable.c +5 -8
@@ -210,20 +210,17 @@ void __init mmu_partition_table_init(void)static void flush_partition (unsigned int lpid,bool radix) { - asmvolatile ("ptesync" : : :"memory" );if (radix) { - asm volatile(PPC_TLBIE_5(%0,%1,2,0,1) : : - "r" (TLBIEL_INVAL_SET_LPID), "r" (lpid)); - asm volatile(PPC_TLBIE_5(%0,%1,2,1,1) : : - "r" (TLBIEL_INVAL_SET_LPID), "r" (lpid)); -trace_tlbie (lpid,0 ,TLBIEL_INVAL_SET_LPID , lpid,2 ,0 ,1 ); +radix__flush_all_lpid (lpid); +radix__flush_all_lpid_guest (lpid); }else { + asmvolatile ("ptesync" : : :"memory" ); asm volatile(PPC_TLBIE_5(%0,%1,2,0,0) : : "r" (TLBIEL_INVAL_SET_LPID), "r" (lpid)); +/* do we need fixup here ?*/ + asmvolatile ("eieio; tlbsync; ptesync" : : :"memory" );trace_tlbie (lpid,0 ,TLBIEL_INVAL_SET_LPID , lpid,2 ,0 ,0 ); } -/* do we need fixup here ?*/ - asmvolatile ("eieio; tlbsync; ptesync" : : :"memory" ); } void mmu_partition_table_set_entry(unsigned int lpid, unsigned long dw0,
arch/powerpc/mm/book3s64/radix_tlb.c +25 -92
171 lines not on this page · raw message
Signature · 1 line
2.22.0
Nicholas Piggin
reviewerPATCH 1/6 powerpc/64s: remove register_process_table callback
In reply to Nicholas PigginThis callback is only required because the partition table init comes before process table allocation on powernv (aka bare metal aka native).
Change the order to allocate the process table first, and remove the callback.
--- arch/powerpc/include/asm/book3s/64/mmu.h | 4 --- arch/powerpc/mm/book3s64/hash_utils.c | 6 ---- arch/powerpc/mm/book3s64/pgtable.c | 3 -- arch/powerpc/mm/book3s64/radix_pgtable.c | 45 +++++++----------------- arch/powerpc/platforms/pseries/lpar.c | 17 +++++++-- 5 files changed, 27 insertions(+), 48 deletions(-)
arch/powerpc/include/asm/book3s/64/mmu.h +0 -4
@@ -206,7 +206,6 @@ extern int mmu_io_psize;void mmu_early_init_devtree (void );void hash__early_init_devtree (void );void radix__early_init_devtree (void ); -extern void radix_init_native (void );extern void hash__early_init_mmu (void );extern void radix__early_init_mmu (void );static inline void early_init_mmu (void )
@@ -238,9 +237,6 @@ static inline void setup_initial_memory_limit(phys_addr_t first_memblock_base, first_memblock_size); } -extern int (*register_process_table)(unsigned long base,unsigned long page_size, -unsigned long tbl_size); -#ifdef CONFIG_PPC_PSERIES extern void radix_init_pseries (void );#else
arch/powerpc/mm/book3s64/hash_utils.c +0 -6
@@ -859,12 +859,6 @@ static void __init htab_initialize(void)/* Using a hypervisor which owns the htab */ htab_address =NULL ; _SDR1 =0 ; -/* -* On POWER9, we need to do a H_REGISTER_PROC_TBL hcall -* to inform the hypervisor that we wish to use the HPT. -*/ -if (cpu_has_feature (CPU_FTR_ARCH_300 )) -register_process_table (0 ,0 ,0 );#ifdef CONFIG_FA_DUMP /* * If firmware assisted dump is active firmware preserves
arch/powerpc/mm/book3s64/pgtable.c +0 -3
@@ -23,9 +23,6 @@ EXPORT_SYMBOL(__pmd_frag_nr);unsigned long __pmd_frag_size_shift;EXPORT_SYMBOL (__pmd_frag_size_shift); -int (*register_process_table)(unsigned long base,unsigned long page_size, -unsigned long tbl_size); -#ifdef CONFIG_TRANSPARENT_HUGEPAGE /* * This is called when relaxing access to a hugepage. It's also called in the page
arch/powerpc/mm/book3s64/radix_pgtable.c +12 -33
86 lines not on this page · raw message
arch/powerpc/platforms/pseries/lpar.c +15 -2
@@ -1531,16 +1531,29 @@ void __init hpte_init_pseries(void) mmu_hash_ops.flush_hash_range = pSeries_lpar_flush_hash_range; mmu_hash_ops.hpte_clear_all = pseries_hpte_clear_all; mmu_hash_ops.hugepage_invalidate = pSeries_lpar_hugepage_invalidate; - register_process_table = pseries_lpar_register_process_table;if (firmware_has_feature (FW_FEATURE_HPT_RESIZE )) mmu_hash_ops.resize_hpt = pseries_lpar_resize_hpt; + +/* +* On POWER9, we need to do a H_REGISTER_PROC_TBL hcall +* to inform the hypervisor that we wish to use the HPT. +*/ +if (cpu_has_feature (CPU_FTR_ARCH_300 )) +pseries_lpar_register_process_table (0 ,0 ,0 ); }void radix_init_pseries (void ) {pr_info ("Using radix MMU under hypervisor\n" ); - register_process_table = pseries_lpar_register_process_table; + +pseries_lpar_register_process_table (__pa (process_tb), +0 ,PRTB_SIZE_SHIFT -12 ); + asmvolatile ("ptesync" : : :"memory" ); + asm volatile(PPC_TLBIE_5(%0,%1,2,1,1) : : + "r" (TLBIEL_INVAL_SET_LPID), "r" (0)); + asmvolatile ("eieio; tlbsync; ptesync" : : :"memory" ); +trace_tlbie (0 ,0 ,TLBIEL_INVAL_SET_LPID ,0 ,2 ,1 ,1 ); }#ifdef CONFIG_PPC_SMLPAR
Signature · 1 line
2.22.0
Nicholas Piggin
reviewerPATCH 3/6 powerpc/64s: make mmu_partition_table_set_entry TLB flush optional
In reply to Nicholas PigginNo functional change.
--- arch/powerpc/include/asm/mmu.h | 2 +- arch/powerpc/kvm/book3s_hv_nested.c | 2 +- arch/powerpc/mm/book3s64/hash_utils.c | 2 +- arch/powerpc/mm/book3s64/pgtable.c | 4 ++-- arch/powerpc/mm/book3s64/radix_pgtable.c | 2 +- 5 files changed, 6 insertions(+), 6 deletions(-)
arch/powerpc/include/asm/mmu.h +1 -1
@@ -257,7 +257,7 @@ extern void radix__mmu_cleanup_all(void);/* Functions for creating and updating partition table on POWER9 */ extern void mmu_partition_table_init (void );extern void mmu_partition_table_set_entry (unsigned int lpid,unsigned long dw0, -unsigned long dw1); +unsigned long dw1,bool flush);#endif /* CONFIG_PPC64 */ struct mm_struct ;
arch/powerpc/kvm/book3s_hv_nested.c +1 -1
@@ -411,7 +411,7 @@ static void kvmhv_flush_lpid(unsigned int lpid)void kvmhv_set_ptbl_entry (unsigned int lpid,u64 dw0,u64 dw1) {if (!kvmhv_on_pseries ()) { -mmu_partition_table_set_entry (lpid, dw0, dw1); +mmu_partition_table_set_entry (lpid, dw0, dw1, true);return ; }
arch/powerpc/mm/book3s64/hash_utils.c +1 -1
@@ -825,7 +825,7 @@ static void __init hash_init_partition_table(phys_addr_t hash_table,* For now, UPRT is 0 and we have no segment table. */ htab_size =__ilog2 (htab_size) -18 ; -mmu_partition_table_set_entry (0 , hash_table | htab_size,0 ); +mmu_partition_table_set_entry (0 , hash_table | htab_size,0 , true);pr_info ("Partition table %p\n" , partition_tb); }
arch/powerpc/mm/book3s64/pgtable.c +2 -2
@@ -224,7 +224,7 @@ static void flush_partition(unsigned int lpid, bool radix) }void mmu_partition_table_set_entry (unsigned int lpid,unsigned long dw0, -unsigned long dw1) +unsigned long dw1,bool flush) {unsigned long old =be64_to_cpu (partition_tb[lpid].patb0);
@@ -251,7 +251,7 @@ void mmu_partition_table_set_entry(unsigned int lpid, unsigned long dw0,uv_register_pate (lpid, dw0, dw1);pr_info ("PATE registered by ultravisor: dw0 = 0x%lx, dw1 = 0x%lx\n" , dw0, dw1); - }else { + }else if (flush) {flush_partition (lpid, (old &PATB_HR )); } }
arch/powerpc/mm/book3s64/radix_pgtable.c +1 -1
@@ -396,7 +396,7 @@ static void __init radix_init_partition_table(void) rts_field =radix__get_tree_size (); dw0 = rts_field |__pa (init_mm.pgd) |RADIX_PGD_INDEX_SIZE |PATB_HR ; dw1 =__pa (process_tb) | (PRTB_SIZE_SHIFT -12 ) |PATB_GR ; -mmu_partition_table_set_entry (0 , dw0, dw1); +mmu_partition_table_set_entry (0 , dw0, dw1, true); asmvolatile ("ptesync" : : :"memory" ); asm volatile(PPC_TLBIE_5(%0,%1,2,1,1) : :
Signature · 1 line
2.22.0
Nicholas Piggin
reviewerPATCH 4/6 powerpc/64s/pseries: radix flush translations before MMU is enabled at boot
In reply to Nicholas PigginRadix guests are responsible for managing their own translation caches, so make them match bare metal radix and hash, and make each CPU flush all its translations right before enabling its MMU.
Radix guests may not flush partition scope translations, so in tlbiel_all, make these flushes conditional on CPU_FTR_HVMODE. Process scope translations are the only type visible to the guest.
--- arch/powerpc/mm/book3s64/radix_pgtable.c | 6 ++---- arch/powerpc/mm/book3s64/radix_tlb.c | 12 ++++++++---- 2 files changed, 10 insertions(+), 8 deletions(-)
arch/powerpc/mm/book3s64/radix_pgtable.c +2 -4
@@ -616,8 +616,7 @@ void __init radix__early_init_mmu(void)/* Switch to the guard PID before turning on MMU */ radix__switch_mmu_context (NULL , &init_mm); -if (cpu_has_feature (CPU_FTR_HVMODE )) -tlbiel_all (); +tlbiel_all (); }void radix__early_init_mmu_secondary (void )
@@ -637,8 +636,7 @@ void radix__early_init_mmu_secondary(void) }radix__switch_mmu_context (NULL , &init_mm); -if (cpu_has_feature (CPU_FTR_HVMODE )) -tlbiel_all (); +tlbiel_all (); }void radix__mmu_cleanup_all (void )
arch/powerpc/mm/book3s64/radix_tlb.c +8 -4
@@ -51,11 +51,15 @@ static void tlbiel_all_isa300(unsigned int num_sets, unsigned int is)* and partition table entries. Then flush the remaining sets of the * TLB. */ -tlbiel_radix_set_isa300 (0 , is,0 ,RIC_FLUSH_ALL ,0 ); -for (set =1 ; set < num_sets; set++) -tlbiel_radix_set_isa300 (set, is,0 ,RIC_FLUSH_TLB ,0 ); -/* Do the same for process scoped entries. */ +if (early_cpu_has_feature (CPU_FTR_HVMODE )) { +/* MSR[HV] should flush partition scope translations first. */ +tlbiel_radix_set_isa300 (0 , is,0 ,RIC_FLUSH_ALL ,0 ); +for (set =1 ; set < num_sets; set++) +tlbiel_radix_set_isa300 (set, is,0 ,RIC_FLUSH_TLB ,0 ); + } + +/* Flush process scoped entries. */ tlbiel_radix_set_isa300 (0 , is,0 ,RIC_FLUSH_ALL ,1 );for (set =1 ; set < num_sets; set++)tlbiel_radix_set_isa300 (set, is,0 ,RIC_FLUSH_TLB ,1 );
Signature · 1 line
2.22.0
Nicholas Piggin
reviewerPATCH 5/6 powerpc/64s: remove unnecessary translation cache flushes at boot
In reply to Nicholas PigginThe various translation structure invalidations performed in early boot when the MMU is off are not required, because everything is invalidated immediately before a CPU first enables its MMU (see early_init_mmu and early_init_mmu_secondary).
--- arch/powerpc/mm/book3s64/hash_utils.c | 2 +- arch/powerpc/mm/book3s64/pgtable.c | 5 +++++ arch/powerpc/mm/book3s64/radix_pgtable.c | 8 +------- arch/powerpc/platforms/pseries/lpar.c | 5 ----- 4 files changed, 7 insertions(+), 13 deletions(-)
arch/powerpc/mm/book3s64/hash_utils.c +1 -1
@@ -825,7 +825,7 @@ static void __init hash_init_partition_table(phys_addr_t hash_table,* For now, UPRT is 0 and we have no segment table. */ htab_size =__ilog2 (htab_size) -18 ; -mmu_partition_table_set_entry (0 , hash_table | htab_size,0 , true); +mmu_partition_table_set_entry (0 , hash_table | htab_size,0 , false);pr_info ("Partition table %p\n" , partition_tb); }
arch/powerpc/mm/book3s64/pgtable.c +5 -0
@@ -252,6 +252,11 @@ void mmu_partition_table_set_entry(unsigned int lpid, unsigned long dw0,pr_info ("PATE registered by ultravisor: dw0 = 0x%lx, dw1 = 0x%lx\n" , dw0, dw1); }else if (flush) { +/* +* Boot does not need to flush, because MMU is off and each +* CPU does a tlbiel_all() before switching them on, which +* flushes everything. +*/ flush_partition (lpid, (old &PATB_HR )); } }
arch/powerpc/mm/book3s64/radix_pgtable.c +1 -7
@@ -396,13 +396,7 @@ static void __init radix_init_partition_table(void) rts_field =radix__get_tree_size (); dw0 = rts_field |__pa (init_mm.pgd) |RADIX_PGD_INDEX_SIZE |PATB_HR ; dw1 =__pa (process_tb) | (PRTB_SIZE_SHIFT -12 ) |PATB_GR ; -mmu_partition_table_set_entry (0 , dw0, dw1, true); - - asmvolatile ("ptesync" : : :"memory" ); - asm volatile(PPC_TLBIE_5(%0,%1,2,1,1) : : - "r" (TLBIEL_INVAL_SET_LPID), "r" (0)); - asmvolatile ("eieio; tlbsync; ptesync" : : :"memory" ); -trace_tlbie (0 ,0 ,TLBIEL_INVAL_SET_LPID ,0 ,2 ,1 ,1 ); +mmu_partition_table_set_entry (0 , dw0, dw1, false);pr_info ("Initializing Radix MMU\n" );pr_info ("Partition table %p\n" , partition_tb);
arch/powerpc/platforms/pseries/lpar.c +0 -5
@@ -1549,11 +1549,6 @@ void radix_init_pseries(void)pseries_lpar_register_process_table (__pa (process_tb),0 ,PRTB_SIZE_SHIFT -12 ); - asmvolatile ("ptesync" : : :"memory" ); - asm volatile(PPC_TLBIE_5(%0,%1,2,1,1) : : - "r" (TLBIEL_INVAL_SET_LPID), "r" (0)); - asmvolatile ("eieio; tlbsync; ptesync" : : :"memory" ); -trace_tlbie (0 ,0 ,TLBIEL_INVAL_SET_LPID ,0 ,2 ,1 ,1 ); }#ifdef CONFIG_PPC_SMLPAR
Signature · 1 line
2.22.0
Nicholas Piggin
reviewercover Making tlbie optional for radix
This is a rebase of the series against the the powerpc next branch with ultravisor changes. Main improvements are implementing and splitting out the precursor patches better.
KVM still requires tlbie to run radix guests. A naive implementation of tlbiel + IPI for LPID flushes was crashing so requires more investigation.
Thanks,
Nick
Nicholas Piggin (6):
powerpc/64s: remove register_process_table callback powerpc/64s/radix: tidy up TLB flushing code
powerpc/64s: make mmu_partition_table_set_entry TLB flush optional powerpc/64s/pseries: radix flush translations before MMU is enabled at
boot
powerpc/64s: remove unnecessary translation cache flushes at boot powerpc/64s/radix: introduce options to disable use of the tlbie
instruction
.../admin-guide/kernel-parameters.txt | 4 + arch/powerpc/include/asm/book3s/64/mmu.h | 4 - .../include/asm/book3s/64/tlbflush-radix.h | 12 +- arch/powerpc/include/asm/book3s/64/tlbflush.h | 9 + arch/powerpc/include/asm/mmu.h | 2 +- arch/powerpc/kvm/book3s_hv.c | 6 + arch/powerpc/kvm/book3s_hv_nested.c | 4 +- arch/powerpc/mm/book3s64/hash_utils.c | 8 +- arch/powerpc/mm/book3s64/pgtable.c | 72 ++++- arch/powerpc/mm/book3s64/radix_pgtable.c | 45 +-- arch/powerpc/mm/book3s64/radix_tlb.c | 303 ++++++++++++------ arch/powerpc/platforms/pseries/lpar.c | 12 +- drivers/misc/cxl/main.c | 4 + drivers/misc/ocxl/main.c | 4 + 14 files changed, 308 insertions(+), 181 deletions(-)
Signature · 1 line
2.22.0
Nicholas Piggin
reviewerPATCH 6/6 powerpc/64s/radix: introduce options to disable use of the tlbie instruction
Introduce two options to control the use of the tlbie instruction. A boot time option which completely disables the kernel using the instruction, this is currently incompatible with HASH MMU, KVM, and coherent accelerators.
And a debugfs option can be switched at runtime and avoids using tlbie for invalidating CPU TLBs for normal process and kernel address mappings. Coherent accelerators are still managed with tlbie, as will KVM partition scope translations.
Cross-CPU TLB flushing is implemented with IPIs and tlbiel. This is a basic implementation which does not attempt to make any optimisation beyond the tlbie implementation.
This is useful for performance testing among other things. For example in certain situations on large systems, using IPIs may be faster than tlbie as they can be directed rather than broadcast. Later we may also take advantage of the IPIs to do more interesting things such as trim the mm cpumask more aggressively.
--- .../admin-guide/kernel-parameters.txt | 4 + arch/powerpc/include/asm/book3s/64/tlbflush.h | 9 + arch/powerpc/kvm/book3s_hv.c | 6 + arch/powerpc/mm/book3s64/pgtable.c | 47 +++++ arch/powerpc/mm/book3s64/radix_tlb.c | 190 ++++++++++++++++-- drivers/misc/cxl/main.c | 4 + drivers/misc/ocxl/main.c | 4 + 7 files changed, 246 insertions(+), 18 deletions(-)
Documentation/admin-guide/kernel-parameters.txt +4 -0
@@ -860,6 +860,10 @@ disable_radix [PPC] Disable RADIX MMU mode on POWER9 + disable_tlbie [PPC] + Disable TLBIE instruction. Currently does not work + with KVM, with HASH MMU, or with coherent accelerators. + disable_cpu_apicid= [X86,APIC,SMP] Format: <int> The number of initial APIC ID for the
arch/powerpc/include/asm/book3s/64/tlbflush.h +9 -0
@@ -162,4 +162,13 @@ static inline void flush_tlb_pgtable(struct mmu_gather *tlb, unsigned long addre radix__flush_tlb_pwc(tlb, address); } + +extern bool tlbie_capable; +extern bool tlbie_enabled; + +static inline bool cputlb_use_tlbie (void ) +{ +return tlbie_enabled; +} +#endif /* _ASM_POWERPC_BOOK3S_64_TLBFLUSH_H */
arch/powerpc/kvm/book3s_hv.c +6 -0
@@ -5462,6 +5462,12 @@ static int kvmppc_radix_possible(void)static int kvmppc_book3s_init_hv (void ) {int r; + +if (!tlbie_capable) { +pr_err ("KVM-HV: Host does not support TLBIE\n" ); +return -ENODEV ; + } +/* * FIXME!! Do we need to check on all cpus ? */
arch/powerpc/mm/book3s64/pgtable.c +47 -0
56 lines not on this page · raw message
arch/powerpc/mm/book3s64/radix_tlb.c +172 -18
281 lines not on this page · raw message
drivers/misc/cxl/main.c +4 -0
@@ -18,6 +18,7 @@#include <linux/sched/task.h> #include <asm/cputable.h> +#include <asm/mmu.h> #include <misc/cxl-base.h> #include "cxl.h"
@@ -315,6 +316,9 @@ static int __init init_cxl(void) {int rc =0 ; +if (!tlbie_capable) +return -EINVAL ; +if ((rc =cxl_file_init ()))return rc;
drivers/misc/ocxl/main.c +4 -0
@@ -2,12 +2,16 @@// Copyright 2017 IBM Corp. #include <linux/module.h> #include <linux/pci.h> +#include <asm/mmu.h> #include "ocxl_internal.h" static int __init init_ocxl (void ) {int rc =0 ; +if (!tlbie_capable) +return -EINVAL ; + rc =ocxl_file_init ();if (rc)return rc;
Signature · 1 line
2.22.0
Re: [PATCH 6/6] powerpc/64s/radix: introduce options to disable use of the tlbie instruction
Nick,
Quoted 4 lines · AEST Nicholas Piggin
Introduce two options to control the use of the tlbie instruction. A boot time option which completely disables the kernel using the instruction, this is currently incompatible with HASH MMU, KVM, and coherent accelerators.
Some accelerators (eg. cxl, ocxl, npu) call mm_context_add_copro() to force global TLB invalidations:
static inline void mm_context_add_copro(struct mm_struct *mm)
{
/*
* If any copro is in use, increment the active CPU count
* in order to force TLB invalidations to be global as to
* propagate to the Nest MMU.
*/
if (atomic_inc_return(&mm->context.copros) == 1)
inc_mm_active_cpus(mm);
}Admittedly I haven't dug into all the details of this patch but it sounds like it might break the above if TLBIE is disabled. Do you think we should add a WARN_ON if mm_context_add_copro() is called with TLBIE disabled? Or perhaps even force TLBIE to be re-enabled if it is called with it disabled?
- Alistair
Documentation/
Quoted 27 lines
And a debugfs option can be switched at runtime and avoids using tlbie for invalidating CPU TLBs for normal process and kernel address mappings. Coherent accelerators are still managed with tlbie, as will KVM partition scope translations.
Cross-CPU TLB flushing is implemented with IPIs and tlbiel. This is a basic implementation which does not attempt to make any optimisation beyond the tlbie implementation.
This is useful for performance testing among other things. For example in certain situations on large systems, using IPIs may be faster than tlbie as they can be directed rather than broadcast. Later we may also take advantage of the IPIs to do more interesting things such as trim the mm cpumask more aggressively.
Signed-off-by Nicholas Piggin <[email protected]>--- .../admin-guide/kernel-parameters.txt | 4 + arch/powerpc/include/asm/book3s/64/tlbflush.h | 9 + arch/powerpc/kvm/book3s_hv.c | 6 + arch/powerpc/mm/book3s64/pgtable.c | 47 +++++ arch/powerpc/mm/book3s64/radix_tlb.c | 190 ++++++++++++++++-- drivers/misc/cxl/main.c | 4 + drivers/misc/ocxl/main.c | 4 + 7 files changed, 246 insertions(+), 18 deletions(-)Documentation/ renamed from Documentation/admin-guide/kernel-parameters.txt +0
-0
admin-guide/kernel-parameters.txt
arch/powerpc/
Quoted 15 lines
index
d3cbb3ae62b6..65ae16549aa3100644Documentation/admin-guide/kernel-parameters.txt +4
-0@@ -860,6 +860,10 @@ disable_radix [PPC] Disable RADIX MMU mode on POWER9 + disable_tlbie [PPC] + Disable TLBIE instruction. Currently does not work + with KVM, with HASH MMU, or with coherent accelerators. + disable_cpu_apicid= [X86,APIC,SMP] Format: <int> The number of initial APIC ID for thearch/powerpc/ renamed from arch/powerpc/include/asm/book3s/64/tlbflush.h +0
-0
include/asm/book3s/64/tlbflush.h
arch/powerpc/include/asm/book3s/64/tlbflush.h static inline void flush_tlb_pgtable(struct mmu_gather
Quoted 4 lines
index
ebf572ea621e..7aa8195b6cff100644arch/powerpc/include/asm/book3s/64/tlbflush.h +0
-0@@ -162,4 +162,13 @@ static inline void flush_tlb_pgtable(struct mmu_gather
*tlb, unsigned long addre
arch/powerpc/mm/book3s64/
Quoted 31 lines
arch/powerpc/include/asm/book3s/64/tlbflush.h +9
-0radix__flush_tlb_pwc(tlb, address); } + +extern bool tlbie_capable; +extern bool tlbie_enabled; + +static inline bool cputlb_use_tlbie (void ) +{ +return tlbie_enabled; +} +#endif /* _ASM_POWERPC_BOOK3S_64_TLBFLUSH_H */ arch/powerpc/kvm/book3s_hv.c +6
-0@@ -5462,6 +5462,12 @@ static int kvmppc_radix_possible(void)static int kvmppc_book3s_init_hv (void ) {int r; + +if (!tlbie_capable) { +pr_err ("KVM-HV: Host does not support TLBIE\n" ); +return -ENODEV ; + } +/* * FIXME!! Do we need to check on all cpus ? */ arch/powerpc/mm/book3s64/ renamed from arch/powerpc/mm/book3s64/pgtable.c +0
-0
pgtable.c
arch/powerpc/mm/book3s64/pgtable.c int pmd_move_must_withdraw(struct spinlock
Quoted 12 lines
index
351eb78eed55..75483b40fcb1100644arch/powerpc/mm/book3s64/pgtable.c +1
-0@@ -8,6 +8,7 @@#include <linux/memblock.h> #include <misc/cxl-base.h> +#include <asm/debugfs.h> #include <asm/pgalloc.h> #include <asm/tlb.h> #include <asm/trace.h> @@ -469,3 +470,49 @@ int pmd_move_must_withdraw(struct spinlock
*new_pmd_ptl,
arch/powerpc/mm/book3s64/pgtable.c int pmd_move_must_withdraw(struct spinlock *new_pmd_ptl,
Quoted 13 lines
arch/powerpc/mm/book3s64/pgtable.c +10
-0return true; } + +/* +* Does the CPU support tlbie? +*/ +bool tlbie_capable__read_mostly = true; +EXPORT_SYMBOL (tlbie_capable); + +/* +* Should tlbie be used for management of CPU TLBs, for kernel and process +* address spaces? tlbie may still be used for nMMU accelerators, and for
KVM
arch/powerpc/mm/book3s64/
Quoted 37 lines
arch/powerpc/mm/book3s64/pgtable.c +36
-0+* guest address spaces. +*/ +bool tlbie_enabled__read_mostly = true; + +static int __init setup_disable_tlbie (char *str) +{ +if (!radix_enabled ()) { +pr_err ("disable_tlbie: Unable to disable TLBIE with Hash MMU.\n" ); +return 1 ; + } + + tlbie_capable = false; + tlbie_enabled = false; + +return 1 ; +} +__setup ("disable_tlbie" , setup_disable_tlbie); + +static int __init pgtable_debugfs_setup (void ) +{ +if (!tlbie_capable) +return 0 ; + +/* +* There is no locking vs tlb flushing when changing this value. +* The tlb flushers will see one value or another, and use either +* tlbie or tlbiel with IPIs. In both cases the TLBs will be +* invalidated as expected. +*/ +debugfs_create_bool ("tlbie_enabled" ,0600 , + powerpc_debugfs_root, + &tlbie_enabled); + +return 0 ; +} +arch_initcall (pgtable_debugfs_setup);arch/powerpc/mm/book3s64/ renamed from arch/powerpc/mm/book3s64/radix_tlb.c +0
-0
radix_tlb.c
arch/powerpc/mm/book3s64/radix_tlb.c static inline void _tlbie_pid(unsigned long pid,
Quoted 4 lines
index
f9cf8ae59831..631be42abd33100644arch/powerpc/mm/book3s64/radix_tlb.c +0
-0@@ -270,6 +270,39 @@ static inline void _tlbie_pid(unsigned long pid,
unsigned long ric)
arch/powerpc/mm/book3s64/radix_tlb.c static __always_inline void _tlbie_va(unsigned long va,
Quoted 40 lines
arch/powerpc/mm/book3s64/radix_tlb.c +33
-0asmvolatile ("eieio; tlbsync; ptesync" : : :"memory" ); } +struct tlbiel_pid { +unsigned long pid; +unsigned long ric; +}; + +static void do_tlbiel_pid (void *info) +{ +struct tlbiel_pid *t = info; + +if (t->ric ==RIC_FLUSH_TLB ) +_tlbiel_pid (t->pid,RIC_FLUSH_TLB ); +else if (t->ric ==RIC_FLUSH_PWC ) +_tlbiel_pid (t->pid,RIC_FLUSH_PWC ); +else +_tlbiel_pid (t->pid,RIC_FLUSH_ALL ); +} + +static inline void _tlbiel_pid_multicast (struct mm_struct *mm, +unsigned long pid,unsigned long ric) +{ +struct cpumask *cpus =mm_cpumask (mm); +struct tlbiel_pid t = { .pid = pid, .ric = ric }; + +on_each_cpu_mask (cpus, do_tlbiel_pid, &t,1 ); +/* +* Always want the CPU translations to be invalidated with tlbiel in +* these paths, so while coprocessors must use tlbie, we can not +* optimise away the tlbiel component. +*/ +if (atomic_read (&mm->context.copros) >0 ) +_tlbie_pid (pid,RIC_FLUSH_ALL ); +} +static inline void _tlbie_lpid (unsigned long lpid,unsigned long ric) { asmvolatile ("ptesync" : : :"memory" );@@ -370,6 +403,53 @@ static __always_inline void _tlbie_va(unsigned long va,
unsigned long pid,
Quoted 41 lines
arch/powerpc/mm/book3s64/radix_tlb.c +38
-0asmvolatile ("eieio; tlbsync; ptesync" : : :"memory" ); } +struct tlbiel_va { +unsigned long pid; +unsigned long va; +unsigned long psize; +unsigned long ric; +}; + +static void do_tlbiel_va (void *info) +{ +struct tlbiel_va *t = info; + +if (t->ric ==RIC_FLUSH_TLB ) +_tlbiel_va (t->va, t->pid, t->psize,RIC_FLUSH_TLB ); +else if (t->ric ==RIC_FLUSH_PWC ) +_tlbiel_va (t->va, t->pid, t->psize,RIC_FLUSH_PWC ); +else +_tlbiel_va (t->va, t->pid, t->psize,RIC_FLUSH_ALL ); +} + +static inline void _tlbiel_va_multicast (struct mm_struct *mm, +unsigned long va,unsigned long pid, +unsigned long psize,unsigned long ric) +{ +struct cpumask *cpus =mm_cpumask (mm); +struct tlbiel_va t = { .va = va, .pid = pid, .psize = psize, .ric = ric }; +on_each_cpu_mask (cpus, do_tlbiel_va, &t,1 ); +if (atomic_read (&mm->context.copros) >0 ) +_tlbie_va (va, pid, psize,RIC_FLUSH_TLB ); +} + +struct tlbiel_va_range { +unsigned long pid; +unsigned long start; +unsigned long end; +unsigned long page_size; +unsigned long psize; +bool also_pwc; +};
arch/powerpc/mm/book3s64/radix_tlb.c static __always_inline void _tlbie_va(unsigned long va, unsigned long pid,
Quoted 10 lines
arch/powerpc/mm/book3s64/radix_tlb.c +9
-0+ +static void do_tlbiel_va_range (void *info) +{ +struct tlbiel_va_range *t = info; + +_tlbiel_va_range (t->start, t->end, t->pid, t->page_size, + t->psize, t->also_pwc); +} + static __always_inline void _tlbie_lpid_va(unsigned long va, unsigned long
lpid,
arch/powerpc/mm/book3s64/radix_tlb.c static inline void _tlbie_va_range(unsigned long start,
Quoted 3 lines
arch/powerpc/mm/book3s64/radix_tlb.c +0
-0unsigned long psize, unsigned long ric) {@@ -393,6 +473,21 @@ static inline void _tlbie_va_range(unsigned long start,
unsigned long end,
arch/powerpc/mm/book3s64/radix_tlb.c static void __flush_all_mm(struct mm_struct *mm, bool
Quoted 41 lines
arch/powerpc/mm/book3s64/radix_tlb.c +23
-4asmvolatile ("eieio; tlbsync; ptesync" : : :"memory" ); } +static inline void _tlbiel_va_range_multicast (struct mm_struct *mm, +unsigned long start,unsigned long end, +unsigned long pid,unsigned long page_size, +unsigned long psize,bool also_pwc) +{ +struct cpumask *cpus =mm_cpumask (mm); +struct tlbiel_va_range t = { .start = start, .end = end, + .pid = pid, .page_size = page_size, + .psize = psize, .also_pwc = also_pwc }; + +on_each_cpu_mask (cpus, do_tlbiel_va_range, &t,1 ); +if (atomic_read (&mm->context.copros) >0 ) +_tlbie_va_range (start, end, pid, page_size, psize, also_pwc); +} +/* * Base TLB flushing operations: * @@ -530,10 +625,14 @@ void radix__flush_tlb_mm(struct mm_struct *mm) goto local; } -if (mm_needs_flush_escalation (mm)) -_tlbie_pid (pid,RIC_FLUSH_ALL ); -else -_tlbie_pid (pid,RIC_FLUSH_TLB ); +if (cputlb_use_tlbie ()) { +if (mm_needs_flush_escalation (mm)) +_tlbie_pid (pid,RIC_FLUSH_ALL ); +else +_tlbie_pid (pid,RIC_FLUSH_TLB ); + }else { +_tlbiel_pid_multicast (mm, pid,RIC_FLUSH_TLB ); + } }else { local:_tlbiel_pid (pid,RIC_FLUSH_TLB );@@ -559,7 +658,10 @@ static void __flush_all_mm(struct mm_struct *mm, bool
fullmm)
arch/powerpc/mm/book3s64/radix_tlb.c void radix__flush_tlb_page_psize(struct mm_struct *mm,
Quoted 12 lines
arch/powerpc/mm/book3s64/radix_tlb.c +4
-1goto local; } } -_tlbie_pid (pid,RIC_FLUSH_ALL ); +if (cputlb_use_tlbie ()) +_tlbie_pid (pid,RIC_FLUSH_ALL ); +else +_tlbiel_pid_multicast (mm, pid,RIC_FLUSH_ALL ); }else { local:_tlbiel_pid (pid,RIC_FLUSH_ALL );@@ -594,7 +696,10 @@ void radix__flush_tlb_page_psize(struct mm_struct *mm,
unsigned long vmaddr,
arch/powerpc/mm/book3s64/radix_tlb.c static inline void __radix__flush_tlb_range(struct
Quoted 49 lines
arch/powerpc/mm/book3s64/radix_tlb.c +26
-2exit_flush_lazy_tlbs (mm); goto local; } -_tlbie_va (vmaddr, pid, psize,RIC_FLUSH_TLB ); +if (cputlb_use_tlbie ()) +_tlbie_va (vmaddr, pid, psize,RIC_FLUSH_TLB ); +else +_tlbiel_va_multicast (mm, vmaddr, pid, psize,RIC_FLUSH_TLB ); }else { local:_tlbiel_va (vmaddr, pid, psize,RIC_FLUSH_TLB );@@ -616,6 +721,24 @@ EXPORT_SYMBOL(radix__flush_tlb_page);#define radix__flush_all_mm radix__local_flush_all_mm#endif /* CONFIG_SMP */ +static void do_tlbiel_kernel (void *info) +{ +_tlbiel_pid (0 ,RIC_FLUSH_ALL ); +} + +static inline void _tlbiel_kernel_broadcast (void ) +{ +on_each_cpu (do_tlbiel_kernel,NULL ,1 ); +if (tlbie_capable) { +/* +* Coherent accelerators don't refcount kernel memory mappings, +* so have to always issue a tlbie for them. This is quite a +* slow path anyway. +*/ +_tlbie_pid (0 ,RIC_FLUSH_ALL ); + } +} +/* * If kernel TLBIs ever become local rather than global, then * drivers/misc/ocxl/link.c:ocxl_link_add_pe will need some work, as it @@ -623,7 +746,10 @@ EXPORT_SYMBOL(radix__flush_tlb_page);*/ void radix__flush_tlb_kernel_range (unsigned long start,unsigned long end) { -_tlbie_pid (0 ,RIC_FLUSH_ALL ); +if (cputlb_use_tlbie ()) +_tlbie_pid (0 ,RIC_FLUSH_ALL ); +else +_tlbiel_kernel_broadcast (); }EXPORT_SYMBOL (radix__flush_tlb_kernel_range);@@ -679,10 +805,14 @@ static inline void __radix__flush_tlb_range(struct
mm_struct *mm,
arch/powerpc/mm/book3s64/radix_tlb.c static inline void __radix__flush_tlb_range(struct
Quoted 19 lines
arch/powerpc/mm/book3s64/radix_tlb.c +8
-4if (local) {_tlbiel_pid (pid,RIC_FLUSH_TLB ); }else { -if (mm_needs_flush_escalation (mm)) -_tlbie_pid (pid,RIC_FLUSH_ALL ); -else -_tlbie_pid (pid,RIC_FLUSH_TLB ); +if (cputlb_use_tlbie ()) { +if (mm_needs_flush_escalation (mm)) +_tlbie_pid (pid,RIC_FLUSH_ALL ); +else +_tlbie_pid (pid,RIC_FLUSH_TLB ); + }else { +_tlbiel_pid_multicast (mm, pid,RIC_FLUSH_TLB ); + } } }else {bool hflush = flush_all_sizes;@@ -707,8 +837,8 @@ static inline void __radix__flush_tlb_range(struct
mm_struct *mm,
arch/powerpc/mm/book3s64/radix_tlb.c static inline void __radix__flush_tlb_range(struct
Quoted 10 lines
arch/powerpc/mm/book3s64/radix_tlb.c +1
-1gflush = false; } - asmvolatile ("ptesync" : : :"memory" );if (local) { + asmvolatile ("ptesync" : : :"memory" );__tlbiel_va_range (start, end, pid, page_size, mmu_virtual_psize);if (hflush) __tlbiel_va_range(hstart, hend, pid,@@ -717,7 +847,8 @@ static inline void __radix__flush_tlb_range(struct
mm_struct *mm,
arch/powerpc/mm/book3s64/radix_tlb.c static inline void __radix__flush_tlb_range(struct
Quoted 10 lines
arch/powerpc/mm/book3s64/radix_tlb.c +2
-1__tlbiel_va_range (gstart, gend, pid,PUD_SIZE ,MMU_PAGE_1G ); asmvolatile ("ptesync" : : :"memory" ); - }else { + }else if (cputlb_use_tlbie ()) { + asmvolatile ("ptesync" : : :"memory" );__tlbie_va_range (start, end, pid, page_size, mmu_virtual_psize);if (hflush) __tlbie_va_range(hstart, hend, pid,@@ -727,6 +858,15 @@ static inline void __radix__flush_tlb_range(struct
mm_struct *mm,
arch/powerpc/mm/book3s64/radix_tlb.c static __always_inline void
Quoted 16 lines
arch/powerpc/mm/book3s64/radix_tlb.c +9
-0PUD_SIZE, MMU_PAGE_1G);fixup_tlbie (); asmvolatile ("eieio; tlbsync; ptesync" : : :"memory" ); + }else { +_tlbiel_va_range_multicast (mm, + start, end, pid, page_size, mmu_virtual_psize, false); +if (hflush) +_tlbiel_va_range_multicast (mm, + hstart, hend, pid,PMD_SIZE ,MMU_PAGE_2M , false); +if (gflush) +_tlbiel_va_range_multicast (mm, + gstart, gend, pid,PUD_SIZE ,MMU_PAGE_1G , false); } }preempt_enable ();@@ -903,16 +1043,26 @@ static __always_inline void
__radix__flush_tlb_range_psize(struct mm_struct *mm,
arch/powerpc/mm/book3s64/radix_tlb.c void radix__flush_tlb_collapsed_pmd(struct mm_struct
Quoted 31 lines
arch/powerpc/mm/book3s64/radix_tlb.c +14
-4if (local) {_tlbiel_pid (pid, also_pwc ?RIC_FLUSH_ALL :RIC_FLUSH_TLB ); }else { -if (mm_needs_flush_escalation (mm)) - also_pwc = true; +if (cputlb_use_tlbie ()) { +if (mm_needs_flush_escalation (mm)) + also_pwc = true; + +_tlbie_pid (pid, + also_pwc ?RIC_FLUSH_ALL :RIC_FLUSH_TLB ); + }else { +_tlbiel_pid_multicast (mm, pid, + also_pwc ?RIC_FLUSH_ALL :RIC_FLUSH_TLB ); + } -_tlbie_pid (pid, also_pwc ?RIC_FLUSH_ALL :RIC_FLUSH_TLB ); } }else {if (local)_tlbiel_va_range (start, end, pid, page_size, psize, also_pwc); -else +else if (cputlb_use_tlbie ())_tlbie_va_range (start, end, pid, page_size, psize, also_pwc); +else +_tlbiel_va_range_multicast (mm, + start, end, pid, page_size, psize, also_pwc); }preempt_enable (); }@@ -954,7 +1104,11 @@ void radix__flush_tlb_collapsed_pmd(struct mm_struct
*mm, unsigned long addr)
arch/powerpc/mm/book3s64/radix_tlb.c void radix__flush_tlb_collapsed_pmd(struct mm_struct *mm, unsigned long addr)
Quoted 6 lines
arch/powerpc/mm/book3s64/radix_tlb.c +2
-1exit_flush_lazy_tlbs (mm); goto local; } -_tlbie_va_range (addr, end, pid,PAGE_SIZE , mmu_virtual_psize, true); +if (cputlb_use_tlbie ()) + _tlbie_va_range(addr, end, pid, PAGE_SIZE, mmu_virtual_psize,
true);
Quoted 50 lines
arch/powerpc/mm/book3s64/radix_tlb.c +3
-0+else +_tlbiel_va_range_multicast (mm, + addr, end, pid,PAGE_SIZE , mmu_virtual_psize, true); }else { local:_tlbiel_va_range (addr, end, pid,PAGE_SIZE , mmu_virtual_psize, true);drivers/misc/cxl/main.c +4
-0@@ -18,6 +18,7 @@#include <linux/sched/task.h> #include <asm/cputable.h> +#include <asm/mmu.h> #include <misc/cxl-base.h> #include "cxl.h" @@ -315,6 +316,9 @@ static int __init init_cxl(void) {int rc =0 ; +if (!tlbie_capable) +return -EINVAL ; +if ((rc =cxl_file_init ()))return rc;drivers/misc/ocxl/main.c +4
-0@@ -2,12 +2,16 @@// Copyright 2017 IBM Corp. #include <linux/module.h> #include <linux/pci.h> +#include <asm/mmu.h> #include "ocxl_internal.h" static int __init init_ocxl (void ) {int rc =0 ; +if (!tlbie_capable) +return -EINVAL ; + rc =ocxl_file_init ();if (rc)return rc;
Nicholas Piggin
reviewerRe: [PATCH 6/6] powerpc/64s/radix: introduce options to disable use of the tlbie instruction
Alistair Popple's on September 3, 2019 10:32 am:
Quoted 22 lines
Nick,
Quoted 4 lines · AEST Nicholas Piggin
Introduce two options to control the use of the tlbie instruction. A boot time option which completely disables the kernel using the instruction, this is currently incompatible with HASH MMU, KVM, and coherent accelerators.
Some accelerators (eg. cxl, ocxl, npu) call
mm_context_add_copro()to force global TLB invalidations:static inline void
mm_context_add_copro(struct mm_struct *mm){ /* * If any copro is in use, increment the active CPU count * in order to force TLB invalidations to be global as to * propagate to the Nest MMU. */ if (atomic_inc_return(&mm->context.copros) == 1) inc_mm_active_cpus(mm); }
Quoted 4 lines
Admittedly I haven't dug into all the details of this patch but it sounds like it might break the above if TLBIE is disabled. Do you think we should add a
WARN_ONifmm_context_add_copro()is called with TLBIE disabled? Or perhaps even force TLBIE to be re-enabled if it is called with it disabled?
The patch has two flags, "enabled" and "capable". If capable is false then it prevents cxl, oxcl, and KVM from loading. I think NPU is gone from the tree now. Hash MMU won't work either, but for now you can't mark !capable with hash.
If enabled is false but capable is true, then it avoids tlbie for flushing the CPU translations, but will also use it to flush nMMU coprocessors (and KVM for partition scope, but hopefully that can be made to work with tlbiel as well).
So this should be fine. Could put a BUG in there if !tlbie_capable, because we can't continue -- idea is that tlbie could be broken or not implemented or we want to test without ever using it.
Thanks,
Nick
Re: [PATCH 1/6] powerpc/64s: remove register_process_table callback
In reply to Nicholas PigginQuoted 7 lines · Nicholas Piggin
This callback is only required because the partition table init comes before process table allocation on powernv (aka bare metal aka native).
Change the order to allocate the process table first, and remove the callback.
Signed-off-by Nicholas Piggin <[email protected]>
Series applied to powerpc next, thanks.
https://git.kernel.org/powerpc/c/ed6546bdc61b7c4bd926cebd82ba52d056fcefa1
cheers
Re: [PATCH 2/6] powerpc/64s/radix: tidy up TLB flushing code
In reply to a message not in the archivearch/powerpc/mm/book3s64/pgtable.c: In function ‘flush_partition’:arch/powerpc/mm/book3s64/pgtable.c:216:3: error: implicit declaration of function ‘radix__flush_all_lpid_guest’ [-Werror=implicit-function-declaration]
radix__flush_all_lpid_guest(lpid);
^
cc1: all warnings being treated as errors
make[4]: *** [arch/powerpc/mm/book3s64/pgtable.o] Error 1
Andreas.
Signature · 3 lines
Andreas Schwab, [email protected] GPG Key fingerprint = 7578 EB47 D4E5 4D69 2510 2552 DF73 E780 A9DA AEC1 "And now for something completely different."