AMD broadcast TLB invalidation
Subsystems (4)
v15 cover AMD broadcast TLB invalidation
From: "Borislav Petkov (AMD)" <[email protected]>
Hi all,
these are Rik's patches from here:
https://lore.kernel.org/r/[email protected]
with a bunch of dhansen's and mine edits ontop.
Some stuff is still in-flight but I'm sending the current state because a bunch of things have changed and we'll need a good base to discuss the remaining changes pending.
Preliminary build and boot tests look good but that doesn't say a whole lot.
Thx.
Rik van Riel (11):
x86/mm: Consolidate full flush threshold decision
x86/mm: Add INVLPGB feature and Kconfig entry
x86/mm: Add INVLPGB support code
x86/mm: Use INVLPGB for kernel TLB flushes
x86/mm: Use broadcast TLB flushing in page reclaim
x86/mm: Add global ASID allocation helper functions x86/mm: Handle global ASID context switch and TLB flush x86/mm: Add global ASID process exit helpers
x86/mm: Enable broadcast TLB invalidation for multi-threaded processes x86/mm: Do targeted broadcast flushing from tlbbatch code x86/mm: Enable AMD translation cache extensions
arch/x86/Kconfig.cpu | 4 + arch/x86/include/asm/cpufeatures.h | 1 + arch/x86/include/asm/disabled-features.h | 8 +- arch/x86/include/asm/mmu.h | 12 + arch/x86/include/asm/mmu_context.h | 10 +- arch/x86/include/asm/msr-index.h | 2 + arch/x86/include/asm/tlb.h | 126 ++++++ arch/x86/include/asm/tlbflush.h | 96 ++++- arch/x86/kernel/cpu/amd.c | 10 + arch/x86/mm/tlb.c | 506 +++++++++++++++++++++-- tools/arch/x86/include/asm/msr-index.h | 2 + 11 files changed, 730 insertions(+), 47 deletions(-)
Signature · 1 line
2.43.0
PATCH v15 1/11 x86/mm: Consolidate full flush threshold decision
From: Rik van Riel <[email protected]>
Reduce code duplication by consolidating the decision point for whether to do individual invalidations or a full flush inside get_flush_tlb_info().
--- arch/x86/mm/tlb.c | 41 +++++++++++++++++++---------------------- 1 file changed, 19 insertions(+), 22 deletions(-)
arch/x86/mm/tlb.c +19 -22
@@ -1000,6 +1000,15 @@ static struct flush_tlb_info *get_flush_tlb_info(struct mm_struct *mm,BUG_ON (this_cpu_inc_return (flush_tlb_info_idx) !=1 );#endif +/* +* If the number of flushes is so large that a full flush +* would be faster, do a full flush. +*/ +if ((end - start) >> stride_shift > tlb_single_page_flush_ceiling) { + start =0 ; + end =TLB_FLUSH_ALL ; + } + info->start = start; info->end = end; info->mm = mm;
@@ -1026,17 +1035,8 @@ void flush_tlb_mm_range(struct mm_struct *mm, unsigned long start, bool freed_tables) {struct flush_tlb_info *info; +int cpu =get_cpu ();u64 new_tlb_gen; -int cpu; - - cpu =get_cpu (); - -/* Should we flush just the requested range? */ -if ((end ==TLB_FLUSH_ALL ) || - ((end - start) >> stride_shift) > tlb_single_page_flush_ceiling) { - start =0 ; - end =TLB_FLUSH_ALL ; - }/* This is also a barrier that synchronizes with switch_mm(). */ new_tlb_gen =inc_mm_tlb_gen (mm);
@@ -1089,22 +1089,19 @@ static void do_kernel_range_flush(void *info)void flush_tlb_kernel_range (unsigned long start,unsigned long end) { -/* Balance as user space task's flush, a bit conservative */ -if (end ==TLB_FLUSH_ALL || - (end - start) > tlb_single_page_flush_ceiling <<PAGE_SHIFT ) { -on_each_cpu (do_flush_tlb_all,NULL ,1 ); - }else { -struct flush_tlb_info *info; +struct flush_tlb_info *info; + +guard (preempt)(); -preempt_disable (); - info =get_flush_tlb_info (NULL , start, end,0 , false, -TLB_GENERATION_INVALID ); + info =get_flush_tlb_info (NULL , start, end,PAGE_SHIFT , false, +TLB_GENERATION_INVALID ); +if (info->end ==TLB_FLUSH_ALL ) +on_each_cpu (do_flush_tlb_all,NULL ,1 ); +else on_each_cpu (do_kernel_range_flush, info,1 ); -put_flush_tlb_info (); -preempt_enable (); - } +put_flush_tlb_info (); }/*
Signature · 1 line
2.43.0
PATCH v15 2/11 x86/mm: Add INVLPGB feature and Kconfig entry
In reply to Borislav PetkovFrom: Rik van Riel <[email protected]>
In addition, the CPU advertises the maximum number of pages that can be shot down with one INVLPGB instruction in CPUID. Save that information for later use.
[ bp: use cpu_has(), typos, massage. ]
--- arch/x86/Kconfig.cpu | 4 ++++ arch/x86/include/asm/cpufeatures.h | 1 + arch/x86/include/asm/disabled-features.h | 8 +++++++- arch/x86/include/asm/tlbflush.h | 3 +++ arch/x86/kernel/cpu/amd.c | 6 ++++++ 5 files changed, 21 insertions(+), 1 deletion(-)
arch/x86/Kconfig.cpu +4 -0
@@ -401,6 +401,10 @@ menuconfig PROCESSOR_SELECT This lets you choose what x86 vendor support code your kernel will include. +config BROADCAST_TLB_FLUSH + def_bool y + depends on CPU_SUP_AMD && 64BIT + config CPU_SUP_INTEL default y bool "Support Intel processors" if PROCESSOR_SELECT
arch/x86/include/asm/cpufeatures.h +1 -0
@@ -338,6 +338,7 @@#define X86_FEATURE_CLZERO (13*32+ 0)/* "clzero" CLZERO instruction */ #define X86_FEATURE_IRPERF (13*32+ 1)/* "irperf" Instructions Retired Count */ #define X86_FEATURE_XSAVEERPTR (13*32+ 2)/* "xsaveerptr" Always save/restore FP error pointers */ +#define X86_FEATURE_INVLPGB (13*32+ 3)/* INVLPGB and TLBSYNC instructions supported */ #define X86_FEATURE_RDPRU (13*32+ 4)/* "rdpru" Read processor register at user level */ #define X86_FEATURE_WBNOINVD (13*32+ 9)/* "wbnoinvd" WBNOINVD instruction */ #define X86_FEATURE_AMD_IBPB (13*32+12)/* Indirect Branch Prediction Barrier */
arch/x86/include/asm/disabled-features.h +7 -1
@@ -129,6 +129,12 @@#define DISABLE_SEV_SNP (1 << (X86_FEATURE_SEV_SNP & 31))#endif +#ifdef CONFIG_X86_BROADCAST_TLB_FLUSH +#define DISABLE_INVLPGB 0 +#else +#define DISABLE_INVLPGB (1 << (X86_FEATURE_INVLPGB & 31)) +#endif +/* * Make sure to add features to the correct mask */
@@ -146,7 +152,7 @@#define DISABLED_MASK11 (DISABLE_RETPOLINE|DISABLE_RETHUNK|DISABLE_UNRET| \ DISABLE_CALL_DEPTH_TRACKING|DISABLE_USER_SHSTK)#define DISABLED_MASK12 (DISABLE_FRED|DISABLE_LAM) -#define DISABLED_MASK13 0 +#define DISABLED_MASK13 (DISABLE_INVLPGB)#define DISABLED_MASK14 0#define DISABLED_MASK15 0#define DISABLED_MASK16 (DISABLE_PKU|DISABLE_OSPKE|DISABLE_LA57|DISABLE_UMIP| \
arch/x86/include/asm/tlbflush.h +3 -0
@@ -183,6 +183,9 @@ static inline void cr4_init_shadow(void)extern unsigned long mmu_cr4_features;extern u32 *trampoline_cr4_features; +/* How many pages can be invalidated with one INVLPGB. */ +extern u16 invlpgb_count_max; +extern void initialize_tlbstate_and_flush (void );/*
arch/x86/kernel/cpu/amd.c +6 -0
@@ -29,6 +29,8 @@#include "cpu.h" +u16 invlpgb_count_max__ro_after_init ; +static inline int rdmsrl_amd_safe (unsigned msr,unsigned long long *p) {u32 gprs[8 ] = {0 };
@@ -1139,6 +1141,10 @@ static void cpu_detect_tlb_amd(struct cpuinfo_x86 *c) tlb_lli_2m[ENTRIES ] = eax & mask; tlb_lli_4m[ENTRIES ] = tlb_lli_2m[ENTRIES ] >>1 ; + +/* Max number of pages INVLPGB can invalidate in one shot */ +if (cpu_has (c,X86_FEATURE_INVLPGB )) + invlpgb_count_max = (cpuid_edx (0x80000008 ) &0xffff ) +1 ; }static const struct cpu_dev amd_cpu_dev = {
Signature · 1 line
2.43.0
Borislav Petkov
maintainerRe: [PATCH v15 02/11] x86/mm: Add INVLPGB feature and Kconfig entry
Quoted 17 lines · Borislav Petkov
arch/x86/include/asm/cpufeatures.h +1
-0@@ -338,6 +338,7 @@#define X86_FEATURE_CLZERO (13*32+ 0)/* "clzero" CLZERO instruction */ #define X86_FEATURE_IRPERF (13*32+ 1)/* "irperf" Instructions Retired Count */ #define X86_FEATURE_XSAVEERPTR (13*32+ 2)/* "xsaveerptr" Always save/restore FP error pointers */ +#define X86_FEATURE_INVLPGB (13*32+ 3)/* INVLPGB and TLBSYNC instructions supported */ #define X86_FEATURE_RDPRU (13*32+ 4)/* "rdpru" Read processor register at user level */ #define X86_FEATURE_WBNOINVD (13*32+ 9)/* "wbnoinvd" WBNOINVD instruction */ #define X86_FEATURE_AMD_IBPB (13*32+12)/* Indirect Branch Prediction Barrier */ arch/x86/include/asm/disabled-features.h +0
-0
arch/x86/include/asm/disabled-features.h line 132
Quoted 4 lines
arch/x86/include/asm/disabled-features.h +1
-0@@ -129,6 +129,12 @@#define DISABLE_SEV_SNP (1 << (X86_FEATURE_SEV_SNP & 31))#endif +#ifdef CONFIG_X86_BROADCAST_TLB_FLUSH
Manali caught this: I forgot this one place yesterday when converting because of the tip patch zapping and branch rebasing tetris...
With that fixed, Tom's patch with the guest ASID valid bit set works too:
https://lore.kernel.org/r/20250304120449.GHZ8bsYYyEBOKQIxBm@fat_crate.local
I'll push a new branch soon.
Signature · 4 lines
Regards/Gruss,
Boris.
https://people.kernel.org/tglx/notes-about-netiquettePATCH v15 3/11 x86/mm: Add INVLPGB support code
In reply to Borislav PetkovFrom: Rik van Riel <[email protected]>
Add helper functions and definitions needed to use broadcast TLB invalidation on AMD CPUs.
[ bp:
- Cleanup commit message
- Improve and expand comments
- push the preemption guards inside the invlpgb* helpers
- merge improvements from dhansen
- add !CONFIG_BROADCAST_TLB_FLUSH function stubs because Clang can't do DCE properly yet and looks at the inline asm and complains about it getting a u64 argument on 32-bit code ]
--- arch/x86/include/asm/tlb.h | 126 +++++++++++++++++++++++++++++++++++++ 1 file changed, 126 insertions(+)
arch/x86/include/asm/tlb.h +126 -0
@@ -6,6 +6,9 @@static inline void tlb_flush (struct mmu_gather *tlb);#include <asm-generic/tlb.h> +#include <linux/kernel.h> +#include <vdso/bits.h> +#include <vdso/page.h> static inline void tlb_flush (struct mmu_gather *tlb) {
@@ -25,4 +28,127 @@ static inline void invlpg(unsigned long addr) asmvolatile ("invlpg (%0)" ::"r" (addr) :"memory" ); } +enum addr_stride { +PTE_STRIDE =0 , +PMD_STRIDE =1 +}; + +#ifdef CONFIG_BROADCAST_TLB_FLUSH +/* +* INVLPGB does broadcast TLB invalidation across all the CPUs in the system. +* +* The INVLPGB instruction is weakly ordered, and a batch of invalidations can +* be done in a parallel fashion. +* +* The instruction takes the number of extra pages to invalidate, beyond +* the first page, while __invlpgb gets the more human readable number of +* pages to invalidate. +* +* The bits in rax[0:2] determine respectively which components of the address +* (VA, PCID, ASID) get compared when flushing. If neither bits are set, *any* +* address in the specified range matches. +* +* TLBSYNC is used to ensure that pending INVLPGB invalidations initiated from +* this CPU have completed. +*/ +static inline void __invlpgb (unsigned long asid,unsigned long pcid, +unsigned long addr,u16 nr_pages, +enum addr_stride stride,u8 flags) +{ +u32 edx = (pcid <<16 ) | asid; +u32 ecx = (stride <<31 ) | (nr_pages -1 ); +u64 rax = addr | flags; + +/* The low bits in rax are for flags. Verify addr is clean. */ +VM_WARN_ON_ONCE (addr & ~PAGE_MASK ); + +/* INVLPGB; supported in binutils >= 2.36. */ + asmvolatile (".byte 0x0f, 0x01, 0xfe" ::"a" (rax),"c" (ecx),"d" (edx)); +} + +static inline void __tlbsync (void ) +{ +/* +* TLBSYNC waits for INVLPGB instructions originating on the same CPU +* to have completed. Print a warning if the task has been migrated, +* and might not be waiting on all the INVLPGBs issued during this TLB +* invalidation sequence. +*/ +cant_migrate (); + +/* TLBSYNC: supported in binutils >= 0.36. */ + asmvolatile (".byte 0x0f, 0x01, 0xff" :::"memory" ); +} +#else +/* Some compilers (I'm looking at you clang!) simply can't do DCE */ +static inline void __invlpgb (unsigned long asid,unsigned long pcid, +unsigned long addr,u16 nr_pages, +enum addr_stride s,u8 flags) { } +static inline void __tlbsync (void ) { } +#endif + +/* +* INVLPGB can be targeted by virtual address, PCID, ASID, or any combination +* of the three. For example: +* - FLAG_VA | FLAG_INCLUDE_GLOBAL: invalidate all TLB entries at the address +* - FLAG_PCID: invalidate all TLB entries matching the PCID +* +* The first is used to invalidate (kernel) mappings at a particular +* address across all processes. +* +* The latter invalidates all TLB entries matching a PCID. +*/ +#define INVLPGB_FLAG_VA BIT(0) +#define INVLPGB_FLAG_PCID BIT(1) +#define INVLPGB_FLAG_ASID BIT(2) +#define INVLPGB_FLAG_INCLUDE_GLOBAL BIT(3) +#define INVLPGB_FLAG_FINAL_ONLY BIT(4) +#define INVLPGB_FLAG_INCLUDE_NESTED BIT(5) + +/* The implied mode when all bits are clear: */ +#define INVLPGB_MODE_ALL_NONGLOBALS 0UL + +static inline void invlpgb_flush_user_nr_nosync (unsigned long pcid, +unsigned long addr, +u16 nr,bool stride) +{ +enum addr_stride str = stride ?PMD_STRIDE :PTE_STRIDE ; +u8 flags =INVLPGB_FLAG_PCID |INVLPGB_FLAG_VA ; + +__invlpgb (0 , pcid, addr, nr, str, flags); +} + +/* Flush all mappings for a given PCID, not including globals. */ +static inline void invlpgb_flush_single_pcid_nosync (unsigned long pcid) +{ +__invlpgb (0 , pcid,0 ,1 ,PTE_STRIDE ,INVLPGB_FLAG_PCID ); +} + +/* Flush all mappings, including globals, for all PCIDs. */ +static inline void invlpgb_flush_all (void ) +{ +/* +* TLBSYNC at the end needs to make sure all flushes done on the +* current CPU have been executed system-wide. Therefore, make +* sure nothing gets migrated in-between but disable preemption +* as it is cheaper. +*/ +guard (preempt)(); +__invlpgb (0 ,0 ,0 ,1 ,PTE_STRIDE ,INVLPGB_FLAG_INCLUDE_GLOBAL ); +__tlbsync (); +} + +/* Flush addr, including globals, for all PCIDs. */ +static inline void invlpgb_flush_addr_nosync (unsigned long addr,u16 nr) +{ +__invlpgb (0 ,0 , addr, nr,PTE_STRIDE ,INVLPGB_FLAG_INCLUDE_GLOBAL ); +} + +/* Flush all mappings for all PCIDs except globals. */ +static inline void invlpgb_flush_all_nonglobals (void ) +{ +guard (preempt)(); +__invlpgb (0 ,0 ,0 ,1 ,PTE_STRIDE ,INVLPGB_MODE_ALL_NONGLOBALS ); +__tlbsync (); +}#endif /* _ASM_X86_TLB_H */
Signature · 1 line
2.43.0
PATCH v15 4/11 x86/mm: Use INVLPGB for kernel TLB flushes
In reply to Borislav PetkovFrom: Rik van Riel <[email protected]>
Use broadcast TLB invalidation for kernel addresses when available. Remove the need to send IPIs for kernel TLB flushes.
[ bp: Integrate dhansen's comments additions, merge the
flush_tlb_all() change into this one too. ]
--- arch/x86/mm/tlb.c | 48 +++++++++++++++++++++++++++++++++++++++++++---- 1 file changed, 44 insertions(+), 4 deletions(-)
arch/x86/mm/tlb.c +44 -4
@@ -1064,7 +1064,6 @@ void flush_tlb_mm_range(struct mm_struct *mm, unsigned long start,mmu_notifier_arch_invalidate_secondary_tlbs (mm, start, end); } -static void do_flush_tlb_all (void *info) {count_vm_tlb_event (NR_TLB_REMOTE_FLUSH_RECEIVED );
@@ -1074,7 +1073,32 @@ static void do_flush_tlb_all(void *info)void flush_tlb_all (void ) {count_vm_tlb_event (NR_TLB_REMOTE_FLUSH ); -on_each_cpu (do_flush_tlb_all,NULL ,1 ); + +/* First try (faster) hardware-assisted TLB invalidation. */ +if (cpu_feature_enabled (X86_FEATURE_INVLPGB )) +invlpgb_flush_all (); +else +/* Fall back to the IPI-based invalidation. */ +on_each_cpu (do_flush_tlb_all,NULL ,1 ); +} + +/* Flush an arbitrarily large range of memory with INVLPGB. */ +static void invlpgb_kernel_range_flush (struct flush_tlb_info *info) +{ +unsigned long addr, nr; + +for (addr = info->start; addr < info->end; addr += nr <<PAGE_SHIFT ) { + nr = (info->end - addr) >>PAGE_SHIFT ; + +/* +* INVLPGB has a limit on the size of ranges it can +* flush. Break up large flushes. +*/ + nr =clamp_val (nr,1 , invlpgb_count_max); + +invlpgb_flush_addr_nosync (addr, nr); + } +__tlbsync (); }static void do_kernel_range_flush (void *info)
@@ -1087,6 +1111,22 @@ static void do_kernel_range_flush(void *info)flush_tlb_one_kernel (addr); } +static void kernel_tlb_flush_all (struct flush_tlb_info *info) +{ +if (cpu_feature_enabled (X86_FEATURE_INVLPGB )) +invlpgb_flush_all (); +else +on_each_cpu (do_flush_tlb_all,NULL ,1 ); +} + +static void kernel_tlb_flush_range (struct flush_tlb_info *info) +{ +if (cpu_feature_enabled (X86_FEATURE_INVLPGB )) +invlpgb_kernel_range_flush (info); +else +on_each_cpu (do_kernel_range_flush, info,1 ); +} +void flush_tlb_kernel_range (unsigned long start,unsigned long end) {struct flush_tlb_info *info;
@@ -1097,9 +1137,9 @@ void flush_tlb_kernel_range(unsigned long start, unsigned long end) TLB_GENERATION_INVALID);if (info->end ==TLB_FLUSH_ALL ) -on_each_cpu (do_flush_tlb_all,NULL ,1 ); +kernel_tlb_flush_all (info);else -on_each_cpu (do_kernel_range_flush, info,1 ); +kernel_tlb_flush_range (info);put_flush_tlb_info (); }
Signature · 1 line
2.43.0
PATCH v15 5/11 x86/mm: Use broadcast TLB flushing in page reclaim
In reply to Borislav PetkovFrom: Rik van Riel <[email protected]>
Page reclaim tracks only the CPU(s) where the TLB needs to be flushed, rather than all the individual mappings that may be getting invalidated.
Use broadcast TLB flushing when that is available.
[ bp: Massage commit message. ]
--- arch/x86/mm/tlb.c | 4 +++- 1 file changed, 3 insertions(+), 1 deletion(-)
arch/x86/mm/tlb.c +3 -1
@@ -1320,7 +1320,9 @@ void arch_tlbbatch_flush(struct arch_tlbflush_unmap_batch *batch)* a local TLB flush is needed. Optimize this use-case by calling * flush_tlb_func_local() directly in this case. */ -if (cpumask_any_but (&batch->cpumask, cpu) < nr_cpu_ids) { +if (cpu_feature_enabled (X86_FEATURE_INVLPGB )) { +invlpgb_flush_all_nonglobals (); + }else if (cpumask_any_but (&batch->cpumask, cpu) < nr_cpu_ids) {flush_tlb_multi (&batch->cpumask, info); }else if (cpumask_test_cpu (cpu, &batch->cpumask)) {lockdep_assert_irqs_enabled ();
Signature · 1 line
2.43.0
PATCH v15 6/11 x86/mm: Add global ASID allocation helper functions
In reply to Borislav PetkovFrom: Rik van Riel <[email protected]>
Add functions to manage global ASID space. Multithreaded processes that are simultaneously active on 4 or more CPUs can get a global ASID, resulting in the same PCID being used for that process on every CPU.
This in turn will allow the kernel to use hardware-assisted TLB flushing through AMD INVLPGB or Intel RAR for these processes.
[ bp:
- Extend use_global_asid() comment
- s/X86_BROADCAST_TLB_FLUSH/BROADCAST_TLB_FLUSH/g
- other touchups ]
--- arch/x86/include/asm/mmu.h | 12 +++ arch/x86/include/asm/mmu_context.h | 2 + arch/x86/include/asm/tlbflush.h | 37 +++++++ arch/x86/mm/tlb.c | 154 ++++++++++++++++++++++++++++- 4 files changed, 202 insertions(+), 3 deletions(-)
arch/x86/include/asm/mmu.h +12 -0
@@ -69,6 +69,18 @@ typedef struct {u16 pkey_allocation_map;s16 execute_only_pkey;#endif + +#ifdef CONFIG_BROADCAST_TLB_FLUSH +/* +* The global ASID will be a non-zero value when the process has +* the same ASID across all CPUs, allowing it to make use of +* hardware-assisted remote TLB invalidation like AMD INVLPGB. +*/ +u16 global_asid; + +/* The process is transitioning to a new global ASID number. */ +bool asid_transition; +#endif } mm_context_t;#define INIT_MM_CONTEXT (mm) \
arch/x86/include/asm/mmu_context.h +2 -0
@@ -139,6 +139,8 @@ static inline void mm_reset_untag_mask(struct mm_struct *mm)#define enter_lazy_tlb enter_lazy_tlbextern void enter_lazy_tlb (struct mm_struct *mm,struct task_struct *tsk); +extern void mm_free_global_asid (struct mm_struct *mm); +/* * Init a new mm. Used on mm copies, like at fork() * and on mm's that are brand-new, like at execve().
arch/x86/include/asm/tlbflush.h +37 -0
@@ -6,6 +6,7 @@#include <linux/mmu_notifier.h> #include <linux/sched.h> +#include <asm/barrier.h> #include <asm/processor.h> #include <asm/cpufeature.h> #include <asm/special_insns.h>
@@ -234,6 +235,42 @@ void flush_tlb_one_kernel(unsigned long addr);void flush_tlb_multi (const struct cpumask *cpumask,const struct flush_tlb_info *info); +static inline bool is_dyn_asid (u16 asid) +{ +return asid <TLB_NR_DYN_ASIDS ; +} + +#ifdef CONFIG_BROADCAST_TLB_FLUSH +static inline u16 mm_global_asid (struct mm_struct *mm) +{ +u16 asid; + +if (!cpu_feature_enabled (X86_FEATURE_INVLPGB )) +return 0 ; + + asid =smp_load_acquire (&mm->context.global_asid); + +/* mm->context.global_asid is either 0, or a global ASID */ +VM_WARN_ON_ONCE (asid &&is_dyn_asid (asid)); + +return asid; +} + +static inline void mm_assign_global_asid (struct mm_struct *mm,u16 asid) +{ +/* +* Notably flush_tlb_mm_range() -> broadcast_tlb_flush() -> +* finish_asid_transition() needs to observe asid_transition = true +* once it observes global_asid. +*/ + mm->context.asid_transition = true; +smp_store_release (&mm->context.global_asid, asid); +} +#else +static inline u16 mm_global_asid (struct mm_struct *mm) {return 0 ; } +static inline void mm_assign_global_asid (struct mm_struct *mm,u16 asid) { } +#endif /* CONFIG_BROADCAST_TLB_FLUSH */ +#ifdef CONFIG_PARAVIRT #include <asm/paravirt.h> #endif
arch/x86/mm/tlb.c +151 -3
170 lines not on this page · raw message
Signature · 1 line
2.43.0
PATCH v15 7/11 x86/mm: Handle global ASID context switch and TLB flush
In reply to Borislav PetkovFrom: Rik van Riel <[email protected]>
Do context switch and TLB flush support for processes that use a global ASID and PCID across all CPUs.
At both context switch time and TLB flush time, it needs to be checked whether a task is switching to a global ASID, and, if so, reload the TLB with the new ASID as appropriate.
In both code paths, the TLB flush is avoided if a global ASID is used, because the global ASIDs are always kept up to date across CPUs, even when the process is not running on a CPU.
[ bp:
- Massage
- :%s/\<static_cpu_has\>/cpu_feature_enabled/cgi
]
--- arch/x86/include/asm/tlbflush.h | 14 ++++++ arch/x86/mm/tlb.c | 77 ++++++++++++++++++++++++++++++--- 2 files changed, 84 insertions(+), 7 deletions(-)
arch/x86/include/asm/tlbflush.h +14 -0
@@ -240,6 +240,11 @@ static inline bool is_dyn_asid(u16 asid)return asid <TLB_NR_DYN_ASIDS ; } +static inline bool is_global_asid (u16 asid) +{ +return !is_dyn_asid (asid); +} +#ifdef CONFIG_BROADCAST_TLB_FLUSH static inline u16 mm_global_asid (struct mm_struct *mm) {
@@ -266,9 +271,18 @@ static inline void mm_assign_global_asid(struct mm_struct *mm, u16 asid) mm->context.asid_transition = true; smp_store_release(&mm->context.global_asid, asid); } + +static inline bool mm_in_asid_transition (struct mm_struct *mm) +{ +if (!cpu_feature_enabled (X86_FEATURE_INVLPGB )) +return false; + +return mm &&READ_ONCE (mm->context.asid_transition); +}#else static inline u16 mm_global_asid (struct mm_struct *mm) {return 0 ; }static inline void mm_assign_global_asid (struct mm_struct *mm,u16 asid) { } +static inline bool mm_in_asid_transition (struct mm_struct *mm) {return false; }#endif /* CONFIG_BROADCAST_TLB_FLUSH */ #ifdef CONFIG_PARAVIRT
arch/x86/mm/tlb.c +70 -7
141 lines not on this page · raw message
Signature · 1 line
2.43.0
PATCH v15 8/11 x86/mm: Add global ASID process exit helpers
In reply to Borislav PetkovFrom: Rik van Riel <[email protected]>
A global ASID is allocated for the lifetime of a process. Free the global ASID at process exit time.
[ bp: Massage, create helpers, hide details inside them. ]
--- arch/x86/include/asm/mmu_context.h | 8 +++++++- arch/x86/include/asm/tlbflush.h | 9 +++++++++ 2 files changed, 16 insertions(+), 1 deletion(-)
arch/x86/include/asm/mmu_context.h +7 -1
@@ -2,7 +2,6 @@#ifndef _ASM_X86_MMU_CONTEXT_H#define _ASM_X86_MMU_CONTEXT_H -#include <asm/desc.h> #include <linux/atomic.h> #include <linux/mm_types.h> #include <linux/pkeys.h>
@@ -13,6 +12,7 @@#include <asm/paravirt.h> #include <asm/debugreg.h> #include <asm/gsseg.h> +#include <asm/desc.h> extern atomic64_t last_mm_ctx_id;
@@ -139,6 +139,9 @@ static inline void mm_reset_untag_mask(struct mm_struct *mm)#define enter_lazy_tlb enter_lazy_tlbextern void enter_lazy_tlb (struct mm_struct *mm,struct task_struct *tsk); +#define mm_init_global_asid mm_init_global_asid +extern void mm_init_global_asid (struct mm_struct *mm); +extern void mm_free_global_asid (struct mm_struct *mm);/*
@@ -163,6 +166,8 @@ static inline int init_new_context(struct task_struct *tsk, mm->context.execute_only_pkey = -1; }#endif + +mm_init_global_asid (mm);mm_reset_untag_mask (mm);init_new_context_ldt (mm);return 0 ;
@@ -172,6 +177,7 @@ static inline int init_new_context(struct task_struct *tsk,static inline void destroy_context (struct mm_struct *mm) {destroy_context_ldt (mm); +mm_free_global_asid (mm); } extern void switch_mm(struct mm_struct *prev, struct mm_struct *next,
arch/x86/include/asm/tlbflush.h +9 -0
@@ -261,6 +261,14 @@ static inline u16 mm_global_asid(struct mm_struct *mm)return asid; } +static inline void mm_init_global_asid (struct mm_struct *mm) +{ +if (cpu_feature_enabled (X86_FEATURE_INVLPGB )) { + mm->context.global_asid =0 ; + mm->context.asid_transition = false; + } +} +static inline void mm_assign_global_asid (struct mm_struct *mm,u16 asid) {/*
@@ -281,6 +289,7 @@ static inline bool mm_in_asid_transition(struct mm_struct *mm) }#else static inline u16 mm_global_asid (struct mm_struct *mm) {return 0 ; } +static inline void mm_init_global_asid (struct mm_struct *mm) { }static inline void mm_assign_global_asid (struct mm_struct *mm,u16 asid) { }static inline bool mm_in_asid_transition (struct mm_struct *mm) {return false; }#endif /* CONFIG_BROADCAST_TLB_FLUSH */
Signature · 1 line
2.43.0
PATCH v15 9/11 x86/mm: Enable broadcast TLB invalidation for multi-threaded processes
In reply to Borislav PetkovFrom: Rik van Riel <[email protected]>
There is not enough room in the 12-bit ASID address space to hand out broadcast ASIDs to every process. Only hand out broadcast ASIDs to processes when they are observed to be simultaneously running on 4 or more CPUs.
This also allows single threaded process to continue using the cheaper, local TLB invalidation instructions like INVLPGB.
Due to the structure of flush_tlb_mm_range(), the INVLPGB flushing is done in a generically named broadcast_tlb_flush() function which can later also be used for Intel RAR.
Combined with the removal of unnecessary lru_add_drain calls() (see
https://lore.kernel.org/r/20241219153253.3da9e8aa@fangorn) this results in a nice performance boost for the will-it-scale tlb_flush2_threads test on an AMD Milan system with 36 cores:
- vanilla kernel: 527k loops/second - lru_add_drain removal: 731k loops/second - only INVLPGB: 527k loops/second
- lru_add_drain + INVLPGB: 1157k loops/second
Profiling with only the INVLPGB changes showed while TLB invalidation went down from 40% of the total CPU time to only around 4% of CPU time, the contention simply moved to the LRU lock.
Fixing both at the same time about doubles the number of iterations per second from this case.
Comparing will-it-scale tlb_flush2_threads with several different numbers of threads on a 72 CPU AMD Milan shows similar results. The number represents the total number of loops per second across all the threads:
threads tip INVLPGB
1 315k 304k
2 423k 424k
4 644k 1032k
8 652k 1267k
16 737k 1368k
32 759k 1199k
64 636k 1094k
72 609k 993k
1 and 2 thread performance is similar with and without INVLPGB, because INVLPGB is only used on processes using 4 or more CPUs simultaneously.
The number is the median across 5 runs.
Some numbers closer to real world performance can be found at Phoronix, thanks to Michael:
https://www.phoronix.com/news/AMD-INVLPGB-Linux-Benefits
[ bp:
- Massage
- :%s/\<static_cpu_has\>/cpu_feature_enabled/cgi
- :%s/\<clear_asid_transition\>/mm_clear_asid_transition/cgi
- Fold in a 0day bot fix: https://lore.kernel.org/oe-kbuild-all/[email protected]
]
WIP
--- arch/x86/include/asm/tlbflush.h | 6 ++ arch/x86/mm/tlb.c | 104 +++++++++++++++++++++++++++++++- 2 files changed, 109 insertions(+), 1 deletion(-)
arch/x86/include/asm/tlbflush.h +6 -0
@@ -280,6 +280,11 @@ static inline void mm_assign_global_asid(struct mm_struct *mm, u16 asid)smp_store_release (&mm->context.global_asid, asid); } +static inline void mm_clear_asid_transition (struct mm_struct *mm) +{ +WRITE_ONCE (mm->context.asid_transition, false); +} +static inline bool mm_in_asid_transition (struct mm_struct *mm) {if (!cpu_feature_enabled (X86_FEATURE_INVLPGB ))
@@ -291,6 +296,7 @@ static inline bool mm_in_asid_transition(struct mm_struct *mm)static inline u16 mm_global_asid (struct mm_struct *mm) {return 0 ; }static inline void mm_init_global_asid (struct mm_struct *mm) { }static inline void mm_assign_global_asid (struct mm_struct *mm,u16 asid) { } +static inline void mm_clear_asid_transition (struct mm_struct *mm) { }static inline bool mm_in_asid_transition (struct mm_struct *mm) {return false; }#endif /* CONFIG_BROADCAST_TLB_FLUSH */
arch/x86/mm/tlb.c +103 -1
118 lines not on this page · raw message
Signature · 1 line
2.43.0
PATCH v15 10/11 x86/mm: Do targeted broadcast flushing from tlbbatch code
In reply to Borislav PetkovFrom: Rik van Riel <[email protected]>
Instead of doing a system-wide TLB flush from arch_tlbbatch_flush(), queue up asynchronous, targeted flushes from arch_tlbbatch_add_pending().
This also allows to avoid adding the CPUs of processes using broadcast flushing to the batch->cpumask, and will hopefully further reduce TLB flushing from the reclaim and compaction paths.
[ bp:
- Massage
- :%s/\<static_cpu_has\>/cpu_feature_enabled/cgi
- merge in improvements from dhansen ]
--- arch/x86/include/asm/tlb.h | 10 ++-- arch/x86/include/asm/tlbflush.h | 27 ++++++---- arch/x86/mm/tlb.c | 88 +++++++++++++++++++++++++++++++-- 3 files changed, 108 insertions(+), 17 deletions(-)
arch/x86/include/asm/tlb.h +5 -5
@@ -108,9 +108,9 @@ static inline void __tlbsync(void) { }/* The implied mode when all bits are clear: */ #define INVLPGB_MODE_ALL_NONGLOBALS 0UL -static inline void invlpgb_flush_user_nr_nosync (unsigned long pcid, -unsigned long addr, -u16 nr,bool stride) +static inline void __invlpgb_flush_user_nr_nosync (unsigned long pcid, +unsigned long addr, +u16 nr,bool stride) {enum addr_stride str = stride ?PMD_STRIDE :PTE_STRIDE ;u8 flags =INVLPGB_FLAG_PCID |INVLPGB_FLAG_VA ;
@@ -119,7 +119,7 @@ static inline void invlpgb_flush_user_nr_nosync(unsigned long pcid, }/* Flush all mappings for a given PCID, not including globals. */ -static inline void invlpgb_flush_single_pcid_nosync (unsigned long pcid) +static inline void __invlpgb_flush_single_pcid_nosync (unsigned long pcid) {__invlpgb (0 , pcid,0 ,1 ,PTE_STRIDE ,INVLPGB_FLAG_PCID ); }
@@ -139,7 +139,7 @@ static inline void invlpgb_flush_all(void) }/* Flush addr, including globals, for all PCIDs. */ -static inline void invlpgb_flush_addr_nosync (unsigned long addr,u16 nr) +static inline void __invlpgb_flush_addr_nosync (unsigned long addr,u16 nr) {__invlpgb (0 ,0 , addr, nr,PTE_STRIDE ,INVLPGB_FLAG_INCLUDE_GLOBAL ); }
arch/x86/include/asm/tlbflush.h +18 -9
@@ -105,6 +105,9 @@ struct tlb_state {* need to be invalidated. */ bool invalidate_other; +#ifdef CONFIG_BROADCAST_TLB_FLUSH +bool need_tlbsync; +#endif #ifdef CONFIG_ADDRESS_MASKING /*
@@ -292,12 +295,24 @@ static inline bool mm_in_asid_transition(struct mm_struct *mm)return mm &&READ_ONCE (mm->context.asid_transition); } + +static inline bool cpu_need_tlbsync (void ) +{ +return this_cpu_read (cpu_tlbstate.need_tlbsync); +} + +static inline void cpu_set_tlbsync (bool state) +{ +this_cpu_write (cpu_tlbstate.need_tlbsync, state); +}#else static inline u16 mm_global_asid (struct mm_struct *mm) {return 0 ; }static inline void mm_init_global_asid (struct mm_struct *mm) { }static inline void mm_assign_global_asid (struct mm_struct *mm,u16 asid) { }static inline void mm_clear_asid_transition (struct mm_struct *mm) { }static inline bool mm_in_asid_transition (struct mm_struct *mm) {return false; } +static inline bool cpu_need_tlbsync (void ) {return false; } +static inline void cpu_set_tlbsync (bool state) { }#endif /* CONFIG_BROADCAST_TLB_FLUSH */ #ifdef CONFIG_PARAVIRT
@@ -347,21 +362,15 @@ static inline u64 inc_mm_tlb_gen(struct mm_struct *mm)return atomic64_inc_return (&mm->context.tlb_gen); } -static inline void arch_tlbbatch_add_pending (struct arch_tlbflush_unmap_batch *batch, -struct mm_struct *mm, -unsigned long uaddr) -{ -inc_mm_tlb_gen (mm); -cpumask_or (&batch->cpumask, &batch->cpumask,mm_cpumask (mm)); -mmu_notifier_arch_invalidate_secondary_tlbs (mm,0 ,-1UL ); -} -static inline void arch_flush_tlb_batched_pending (struct mm_struct *mm) {flush_tlb_mm (mm); }extern void arch_tlbbatch_flush (struct arch_tlbflush_unmap_batch *batch); +extern void arch_tlbbatch_add_pending (struct arch_tlbflush_unmap_batch *batch, +struct mm_struct *mm, +unsigned long uaddr); static inline bool pte_flags_need_flush(unsigned long oldflags, unsigned long newflags,
arch/x86/mm/tlb.c +85 -3
124 lines not on this page · raw message
Signature · 1 line
2.43.0
PATCH v15 11/11 x86/mm: Enable AMD translation cache extensions
In reply to Borislav PetkovFrom: Rik van Riel <[email protected]>
With AMD TCE (translation cache extensions) only the intermediate mappings that cover the address range zapped by INVLPG / INVLPGB get invalidated, rather than all intermediate mappings getting zapped at every TLB invalidation.
This can help reduce the TLB miss rate, by keeping more intermediate mappings in the cache.
From the AMD manual:
Translation Cache Extension (TCE) Bit. Bit 15, read/write. Setting this bit to 1 changes how the INVLPG, INVLPGB, and INVPCID instructions operate on TLB entries. When this bit is 0, these instructions remove the target PTE from the TLB as well as all upper-level table entries that are cached in the TLB, whether or not they are associated with the target PTE. When this bit is set, these instructions will remove the target PTE and only those upper-level entries that lead to the target PTE in the page table hierarchy, leaving unrelated upper-level entries intact.
[ bp: use cpu_has()... I know, it is a mess. ]
--- arch/x86/include/asm/msr-index.h | 2 ++ arch/x86/kernel/cpu/amd.c | 4 ++++ tools/arch/x86/include/asm/msr-index.h | 2 ++ 3 files changed, 8 insertions(+)
arch/x86/include/asm/msr-index.h +2 -0
@@ -25,6 +25,7 @@#define _EFER_SVME 12/* Enable virtualization */ #define _EFER_LMSLE 13/* Long Mode Segment Limit Enable */ #define _EFER_FFXSR 14/* Enable Fast FXSAVE/FXRSTOR */ +#define _EFER_TCE 15/* Enable Translation Cache Extensions */ #define _EFER_AUTOIBRS 21/* Enable Automatic IBRS */ #define EFER_SCE (1<<_EFER_SCE)
@@ -34,6 +35,7 @@#define EFER_SVME (1<<_EFER_SVME)#define EFER_LMSLE (1<<_EFER_LMSLE)#define EFER_FFXSR (1<<_EFER_FFXSR) +#define EFER_TCE (1<<_EFER_TCE)#define EFER_AUTOIBRS (1<<_EFER_AUTOIBRS)/*
arch/x86/kernel/cpu/amd.c +4 -0
@@ -1075,6 +1075,10 @@ static void init_amd(struct cpuinfo_x86 *c)/* AMD CPUs don't need fencing after x2APIC/TSC_DEADLINE MSR writes. */ clear_cpu_cap (c,X86_FEATURE_APIC_MSRS_FENCE ); + +/* Enable Translation Cache Extension */ +if (cpu_has (c,X86_FEATURE_TCE )) +msr_set_bit (MSR_EFER , _EFER_TCE); }#ifdef CONFIG_X86_32
tools/arch/x86/include/asm/msr-index.h +2 -0
@@ -25,6 +25,7 @@#define _EFER_SVME 12/* Enable virtualization */ #define _EFER_LMSLE 13/* Long Mode Segment Limit Enable */ #define _EFER_FFXSR 14/* Enable Fast FXSAVE/FXRSTOR */ +#define _EFER_TCE 15/* Enable Translation Cache Extensions */ #define _EFER_AUTOIBRS 21/* Enable Automatic IBRS */ #define EFER_SCE (1<<_EFER_SCE)
@@ -34,6 +35,7 @@#define EFER_SVME (1<<_EFER_SVME)#define EFER_LMSLE (1<<_EFER_LMSLE)#define EFER_FFXSR (1<<_EFER_FFXSR) +#define EFER_TCE (1<<_EFER_TCE)#define EFER_AUTOIBRS (1<<_EFER_AUTOIBRS)/*
Signature · 1 line
2.43.0