x86/mm/cpa: Further optimizations
Peter Zijlstra
maintainerPATCH 4/8 x86/mm/cpa: Use flush_tlb_kernel_range()
In reply to Peter ZijlstraBoth cpa_flush_range() and cpa_flush_array() have a well specified range, use that to do a range based TLB invalidate.
--- arch/x86/mm/pageattr.c | 9 +++++---- 1 file changed, 5 insertions(+), 4 deletions(-)
arch/x86/mm/pageattr.c +5 -4
@@ -293,7 +293,7 @@ static void cpa_flush_range(unsigned lonBUG_ON (irqs_disabled () && !early_boot_irqs_disabled);WARN_ON (PAGE_ALIGN (start) != start); -flush_tlb_all (); +flush_tlb_kernel_range (start, start +PAGE_SIZE * numpages);if (!cache)return ;
@@ -315,14 +315,15 @@ static void cpa_flush_range(unsigned lon } } -static void cpa_flush_array (unsigned long *start,int numpages,int cache, +static void cpa_flush_array (unsigned long baddr,unsigned long *start, +int numpages,int cache,int in_flags,struct page **pages) {unsigned int i, level;BUG_ON (irqs_disabled () && !early_boot_irqs_disabled); -flush_tlb_all (); +flush_tlb_kernel_range (baddr, baddr +PAGE_SIZE * numpages);if (!cache)return ;
@@ -1754,7 +1755,7 @@ static int change_page_attr_set_clr(unsi*/ if (!ret &&boot_cpu_has (X86_FEATURE_CLFLUSH )) {if (cpa.flags & (CPA_PAGES_ARRAY |CPA_ARRAY )) { -cpa_flush_array (addr, numpages, cache, +cpa_flush_array (baddr, addr, numpages, cache, cpa.flags, pages); }else cpa_flush_range (baddr, numpages, cache);
Peter Zijlstra
maintainerPATCH 1/8 x86/mm/cpa: Use flush_tlb_all()
In reply to Peter ZijlstraInstead of open-coding it..
--- arch/x86/mm/pageattr.c | 12 +----------- 1 file changed, 1 insertion(+), 11 deletions(-)
arch/x86/mm/pageattr.c +1 -11
@@ -285,16 +285,6 @@ static void cpa_flush_all(unsigned longon_each_cpu (__cpa_flush_all, (void *) cache,1 ); } -static void __cpa_flush_range (void *arg) -{ -/* -* We could optimize that further and do individual per page -* tlb invalidates for a low number of pages. Caveat: we must -* flush the high aliases on 64bit as well. -*/ -__flush_tlb_all (); -} -static void cpa_flush_range (unsigned long start,int numpages,int cache) {unsigned int i, level;
@@ -303,7 +293,7 @@ static void cpa_flush_range(unsigned lonBUG_ON (irqs_disabled () && !early_boot_irqs_disabled);WARN_ON (PAGE_ALIGN (start) != start); -on_each_cpu (__cpa_flush_range,NULL ,1 ); +flush_tlb_all ();if (!cache)return ;
Peter Zijlstra
maintainerPATCH 6/8 x86/mm/cpa: Move CLFLUSH test into cpa_flush_array()
In reply to Peter ZijlstraRather than guarding cpa_flush_array() users with a CLFLUSH test, put it inside.
--- arch/x86/mm/pageattr.c | 29 +++++++++++++++++------------ 1 file changed, 17 insertions(+), 12 deletions(-)
arch/x86/mm/pageattr.c +17 -12
@@ -328,6 +328,11 @@ static void cpa_flush_array(unsigned lonBUG_ON (irqs_disabled () && !early_boot_irqs_disabled); +if (!static_cpu_has (X86_FEATURE_CLFLUSH )) { +cpa_flush_all (cache); +return ; + } +flush_tlb_kernel_range (baddr, baddr +PAGE_SIZE * numpages);if (!cache)
@@ -1753,19 +1758,19 @@ static int change_page_attr_set_clr(unsi cache = !!pgprot2cachemode (mask_set);/* -* On success we use CLFLUSH, when the CPU supports it to -* avoid the WBINVD. If the CPU does not support it and in the -* error case we fall back to cpa_flush_all (which uses -* WBINVD): -*/ -if (!ret &&boot_cpu_has (X86_FEATURE_CLFLUSH )) { -if (cpa.flags & (CPA_PAGES_ARRAY |CPA_ARRAY )) { -cpa_flush_array (baddr, addr, numpages, cache, - cpa.flags, pages); - }else -cpa_flush_range (baddr, numpages, cache); - }else +* On error; flush everything to be sure. +*/ +if (ret) {cpa_flush_all (cache); + goto out; + } + +if (cpa.flags & (CPA_PAGES_ARRAY |CPA_ARRAY )) { +cpa_flush_array (baddr, addr, numpages, cache, + cpa.flags, pages); + }else { +cpa_flush_range (baddr, numpages, cache); + } out:return ret;
Peter Zijlstra
maintainerPATCH 7/8 x86/mm/cpa: Factor common code between cpa_flush_*()
In reply to Peter ZijlstraThe start of cpa_flush_range() and cpa_flush_array() is the same, use a common function.
--- arch/x86/mm/pageattr.c | 29 +++++++++++++---------------- 1 file changed, 13 insertions(+), 16 deletions(-)
arch/x86/mm/pageattr.c +13 -16
@@ -285,22 +285,28 @@ static void cpa_flush_all(unsigned longon_each_cpu (__cpa_flush_all, (void *) cache,1 ); } -static void cpa_flush_range (unsigned long start,int numpages,int cache) +static bool __cpa_flush_range (unsigned long start,int numpages,int cache) { -unsigned int i, level; -unsigned long addr; -BUG_ON (irqs_disabled () && !early_boot_irqs_disabled); +WARN_ON (PAGE_ALIGN (start) != start);if (!static_cpu_has (X86_FEATURE_CLFLUSH )) {cpa_flush_all (cache); -return ; +return true; }flush_tlb_kernel_range (start, start +PAGE_SIZE * numpages); -if (!cache) +return !cache; +} + +static void cpa_flush_range (unsigned long start,int numpages,int cache) +{ +unsigned int i, level; +unsigned long addr; + +if (__cpa_flush_range (start, numpages, cache))return ;/*
@@ -326,16 +332,7 @@ static void cpa_flush_array(unsigned lon {unsigned int i, level; -BUG_ON (irqs_disabled () && !early_boot_irqs_disabled); - -if (!static_cpu_has (X86_FEATURE_CLFLUSH )) { -cpa_flush_all (cache); -return ; - } - -flush_tlb_kernel_range (baddr, baddr +PAGE_SIZE * numpages); - -if (!cache) +if (__cpa_flush_range (baddr, numpages, cache))return ;/*
Peter Zijlstra
maintainerPATCH 5/8 x86/mm/cpa: Move CLFLUSH test into cpa_flush_range()
In reply to Peter ZijlstraRather than guarding all cpa_flush_range() uses with a CLFLUSH test, put it inside.
--- arch/x86/mm/pageattr.c | 15 +++++++-------- 1 file changed, 7 insertions(+), 8 deletions(-)
arch/x86/mm/pageattr.c +7 -8
@@ -293,6 +293,11 @@ static void cpa_flush_range(unsigned lonBUG_ON (irqs_disabled () && !early_boot_irqs_disabled);WARN_ON (PAGE_ALIGN (start) != start); +if (!static_cpu_has (X86_FEATURE_CLFLUSH )) { +cpa_flush_all (cache); +return ; + } +flush_tlb_kernel_range (start, start +PAGE_SIZE * numpages);if (!cache)
@@ -2075,10 +2080,7 @@ static int __set_memory_enc_dec(unsigned/* * Before changing the encryption attribute, we need to flush caches. */ -if (static_cpu_has (X86_FEATURE_CLFLUSH )) -cpa_flush_range (start, numpages,1 ); -else -cpa_flush_all (1 ); +cpa_flush_range (start, numpages,1 ); ret =__change_page_attr_set_clr (&cpa,1 );
@@ -2089,10 +2091,7 @@ static int __set_memory_enc_dec(unsigned* in case TLB flushing gets optimized in the cpa_flush_range() * path use the same logic as above. */ -if (static_cpu_has (X86_FEATURE_CLFLUSH )) -cpa_flush_range (start, numpages,0 ); -else -cpa_flush_all (0 ); +cpa_flush_range (start, numpages,0 );return ret; }
Peter Zijlstra
maintainerPATCH 2/8 x86/mm/cpa: Move flush_tlb_all()
In reply to Peter ZijlstraThere is an atom errata, where we do a local TLB invalidate right before we return and then do a global TLB invalidate.
Move the global invalidate up a little bit and avoid the local invalidate entirely.
This does put the global invalidate under pgd_lock, but that shouldn't matter.
--- arch/x86/mm/pageattr.c | 44 +++++++++++++++++--------------------------- 1 file changed, 17 insertions(+), 27 deletions(-)
arch/x86/mm/pageattr.c +17 -27
@@ -996,14 +996,24 @@ __split_large_page(struct cpa_data *cpa,__set_pmd_pte (kpte, address,mk_pte (base,__pgprot (_KERNPG_TABLE)));/* -* Intel Atom errata AAH41 workaround. +* Do a global flush tlb after splitting the large page +* and before we do the actual change page attribute in the PTE. * -* The real fix should be in hw or in a microcode update, but -* we also probabilistically try to reduce the window of having -* a large TLB mixed with 4K TLBs while instruction fetches are -* going on. +* Without this, we violate the TLB application note, that says: +* "The TLBs may contain both ordinary and large-page +* translations for a 4-KByte range of linear addresses. This +* may occur if software modifies the paging structures so that +* the page size used for the address range changes. If the two +* translations differ with respect to page frame or attributes +* (e.g., permissions), processor behavior is undefined and may +* be implementation-specific." +* +* We do this global tlb flush inside the cpa_lock, so that we +* don't allow any other cpu, with stale tlb entries change the +* page attribute in parallel, that also falls into the +* just split large page entry. */ -__flush_tlb_all (); +flush_tlb_all ();spin_unlock (&pgd_lock);return 0 ;
@@ -1528,28 +1538,8 @@ static int __change_page_attr(struct cpa* We have to split the large page: */ err =split_large_page (cpa, kpte, address); -if (!err) { -/* -* Do a global flush tlb after splitting the large page -* and before we do the actual change page attribute in the PTE. -* -* With out this, we violate the TLB application note, that says -* "The TLBs may contain both ordinary and large-page -* translations for a 4-KByte range of linear addresses. This -* may occur if software modifies the paging structures so that -* the page size used for the address range changes. If the two -* translations differ with respect to page frame or attributes -* (e.g., permissions), processor behavior is undefined and may -* be implementation-specific." -* -* We do this global tlb flush inside the cpa_lock, so that we -* don't allow any other cpu, with stale tlb entries change the -* page attribute in parallel, that also falls into the -* just split large page entry. -*/ -flush_tlb_all (); +if (!err) goto repeat; - }return err; }
Peter Zijlstra
maintainerPATCH 3/8 x86/mm/cpa: Unconditionally avoid WBINDV when we can
In reply to Peter ZijlstraCAT has happened, WBINDV is bad (even before CAT blowing away the entire cache on a multi-core platform wasn't nice), try not to use it ever.
--- arch/x86/mm/pageattr.c | 18 ++---------------- 1 file changed, 2 insertions(+), 16 deletions(-)
arch/x86/mm/pageattr.c +2 -16
@@ -319,26 +319,12 @@ static void cpa_flush_array(unsigned lon int in_flags, struct page **pages) {unsigned int i, level; -#ifdef CONFIG_PREEMPT -/* -* Avoid wbinvd() because it causes latencies on all CPUs, -* regardless of any CPU isolation that may be in effect. -* -* This should be extended for CAT enabled systems independent of -* PREEMPT because wbinvd() does not respect the CAT partitions and -* this is exposed to unpriviledged users through the graphics -* subsystem. -*/ -unsigned long do_wbinvd =0 ; -#else -unsigned long do_wbinvd = cache && numpages >=1024 ;/* 4M threshold */ -#endif BUG_ON (irqs_disabled () && !early_boot_irqs_disabled); -on_each_cpu (__cpa_flush_all, (void *) do_wbinvd,1 ); +flush_tlb_all (); -if (!cache || do_wbinvd) +if (!cache)return ;/*
Peter Zijlstra
maintainerPATCH 8/8 x86/mm/cpa: Optimize __cpa_flush_range()
In reply to Peter ZijlstraIf we IPI for WBINDV, then we might as well kill the entire TLB too. But if we don't have to invalidate cache, there is no reason not to use a range TLB flush.
--- arch/x86/mm/pageattr.c | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-)
arch/x86/mm/pageattr.c +1 -1
@@ -291,7 +291,7 @@ static bool __cpa_flush_range(unsigned lWARN_ON (PAGE_ALIGN (start) != start); -if (!static_cpu_has (X86_FEATURE_CLFLUSH )) { +if (cache && !static_cpu_has (X86_FEATURE_CLFLUSH )) {cpa_flush_all (cache);return true; }
Peter Zijlstra
maintainercover x86/mm/cpa: Further optimizations
Thomas' CPA patches made me look at the tlb invalidate crud around there. These patches are the result of cleaning some of that up a bit. They are based on top of Thomas' patches.
Dave, can you have a wee look at the Atom errata thing; this does grow the amount of code between the page-table update and the TLB flush a bit, but all that PTI crud we did a while back already made it a non-trivial amount of code.
--- pageattr.c | 128 ++++++++++++++++++++++--------------------------------------- 1 file changed, 48 insertions(+), 80 deletions(-)
Peter Zijlstra
maintainerRe: [PATCH 1/8] x86/mm/cpa: Use flush_tlb_all()
In reply to Peter ZijlstraQuoted 12 lines · Peter Zijlstra
Instead of open-coding it..
Signed-off-by Peter Zijlstra (Intel) <[email protected]>--- arch/x86/mm/pageattr.c | 12 +----------- 1 file changed, 1 insertion(+), 11 deletions(-)arch/x86/mm/pageattr.c +0
-0@@ -285,16 +285,6 @@ static void cpa_flush_all(unsigned longon_each_cpu (__cpa_flush_all, (void *) cache,1 ); }
arch/x86/mm/pageattr.c static void cpa_flush_all(unsigned long · line 296
Quoted 10 lines
arch/x86/mm/pageattr.c +0
-9-static void __cpa_flush_range (void *arg) -{ -/* -* We could optimize that further and do individual per page -* tlb invalidates for a low number of pages. Caveat: we must -* flush the high aliases on 64bit as well. -*/ -__flush_tlb_all (); -}
Hmm,.. so in patch #4 I do switch to flush_tlb_kernel_range(). What are those high aliases that comment talks about?
Re: [PATCH 1/8] x86/mm/cpa: Use flush_tlb_all()
Quoted 14 lines · Peter Zijlstra
Quoted 13 lines · Peter Zijlstra
Instead of open-coding it..
Signed-off-by Peter Zijlstra (Intel) <[email protected]>--- arch/x86/mm/pageattr.c | 12 +----------- 1 file changed, 1 insertion(+), 11 deletions(-)arch/x86/mm/pageattr.c +0
-0@@ -285,16 +285,6 @@ static void cpa_flush_all(unsigned longon_each_cpu (__cpa_flush_all, (void *) cache,1 ); }
Quoted 12 lines
arch/x86/mm/pageattr.c static void cpa_flush_all(unsigned long · line 296
Quoted 9 lines
arch/x86/mm/pageattr.c +0
-9-static void __cpa_flush_range (void *arg) -{ -/* -* We could optimize that further and do individual per page -* tlb invalidates for a low number of pages. Caveat: we must -* flush the high aliases on 64bit as well. -*/ -__flush_tlb_all (); -}Hmm,.. so in patch #4 I do switch to
flush_tlb_kernel_range(). What are those high aliases that comment talks about?
We have two mappings for the kernel. The 'real one' and the direct mapping alias and for most of the operations, we have to make sure that the table entries are identical in both maps.
The comments in that code probably need some care.
Thanks,
tglx
Re: [PATCH 2/8] x86/mm/cpa: Move flush_tlb_all()
In reply to Peter ZijlstraQuoted 5 lines · Peter Zijlstra
arch/x86/mm/pageattr.c +0
-1@@ -1528,28 +1538,8 @@ static int __change_page_attr(struct cpa* We have to split the large page: */ err =split_large_page (cpa, kpte, address); -if (!err) {
...
arch/x86/mm/pageattr.c static int __change_page_attr(struct cpa · line 1552
Quoted 4 lines
arch/x86/mm/pageattr.c +1
-2-flush_tlb_all (); +if (!err) goto repeat; - }
I _think_ this is the only use of split_large_page()'s error code. We can just make it return void now, right?
BTW, I also really like doing the "final" TLB flush as close as possible to the actual PTE manipulation. Makes it much less likely that we break it.
cover x86/mm/cpa: Further optimizations
In reply to Peter ZijlstraQuoted 3 lines · Peter Zijlstra
Dave, can you have a wee look at the Atom errata thing; this does grow the amount of code between the page-table update and the TLB flush a bit, but all that PTI crud we did a while back already made it a non-trivial amount of code.
Oh, were you worried that the act of sending the IPIs widens the window vs. just doing the local flush? We definitely do the smp_call_function() dance *before* we do the local TLB invalidation in in on_each_cpu().
But, the existing flush was not a "true" fix anyway. It just tended to work around things.
If the issue pops back up, I'd prefer that we just do:
__flush_tlb_all(); flush_tlb_all();
Preferably with an actual model/family check for the "AAH41" workaround.
The real value of what you did was moving the flush_tlb_all() closer,
and that remains either way.
BTW, on all of these:
Peter Zijlstra
maintainerRe: [PATCH 2/8] x86/mm/cpa: Move flush_tlb_all()
In reply to Dave HansenQuoted 7 lines · Dave Hansen
Quoted 5 lines · Peter Zijlstra
arch/x86/mm/pageattr.c +0
-1@@ -1528,28 +1538,8 @@ static int __change_page_attr(struct cpa* We have to split the large page: */ err =split_large_page (cpa, kpte, address); -if (!err) {...
Quoted 7 lines
arch/x86/mm/pageattr.c static int __change_page_attr(struct cpa · line 1552
Quoted 4 lines
arch/x86/mm/pageattr.c +1
-2-flush_tlb_all (); +if (!err) goto repeat; - }I _think_ this is the only use of split_large_page()'s error code. We can just make it return void now, right?
Then how does it decide to goto repeat or error out?
Quoted 2 lines
BTW, I also really like doing the "final" TLB flush as close as possible to the actual PTE manipulation. Makes it much less likely that we break it.
Yes.
Re: [PATCH 2/8] x86/mm/cpa: Move flush_tlb_all()
Quoted 8 lines · Peter Zijlstra
Quoted 7 lines · Dave Hansen
Quoted 5 lines · Peter Zijlstra
arch/x86/mm/pageattr.c +0
-1@@ -1528,28 +1538,8 @@ static int __change_page_attr(struct cpa* We have to split the large page: */ err =split_large_page (cpa, kpte, address); -if (!err) {...
Quoted 7 lines
Quoted 6 lines
arch/x86/mm/pageattr.c static int __change_page_attr(struct cpa · line 1552
Quoted 4 lines
arch/x86/mm/pageattr.c +1
-2-flush_tlb_all (); +if (!err) goto repeat; - }I _think_ this is the only use of split_large_page()'s error code. We can just make it return void now, right?
Then how does it decide to goto repeat or error out?
D'oh. Good point. I was misreading the diff.
tip-bot for Peter Zijlstra
merge bot[tip:x86/mm] x86/mm/cpa: Use flush_tlb_all()
In reply to Peter Zijlstrax86/mm/cpa: Use flush_tlb_all()
Commit-ID: c6185b1f21a47af94617fde3af7e803817b522a9 Gitweb: https://git.kernel.org/tip/c6185b1f21a47af94617fde3af7e803817b522a9 Author: Peter Zijlstra <[email protected]> AuthorDate: Wed, 19 Sep 2018 10:50:17 +0200 Committer: Thomas Gleixner <[email protected]> CommitDate: Thu, 27 Sep 2018 20:39:40 +0200
x86/mm/cpa: Use flush_tlb_all()
Instead of open-coding it..
--- arch/x86/mm/pageattr.c | 12 +----------- 1 file changed, 1 insertion(+), 11 deletions(-)
arch/x86/mm/pageattr.c +1 -11
@@ -285,16 +285,6 @@ static void cpa_flush_all(unsigned long cache)on_each_cpu (__cpa_flush_all, (void *) cache,1 ); } -static void __cpa_flush_range (void *arg) -{ -/* -* We could optimize that further and do individual per page -* tlb invalidates for a low number of pages. Caveat: we must -* flush the high aliases on 64bit as well. -*/ -__flush_tlb_all (); -} -static void cpa_flush_range (unsigned long start,int numpages,int cache) {unsigned int i, level;
@@ -303,7 +293,7 @@ static void cpa_flush_range(unsigned long start, int numpages, int cache)BUG_ON (irqs_disabled () && !early_boot_irqs_disabled);WARN_ON (PAGE_ALIGN (start) != start); -on_each_cpu (__cpa_flush_range,NULL ,1 ); +flush_tlb_all ();if (!cache)return ;
tip-bot for Peter Zijlstra
merge bot[tip:x86/mm] x86/mm/cpa: Move flush_tlb_all()
In reply to Peter Zijlstrax86/mm/cpa: Move flush_tlb_all()
Commit-ID: c0a759abf5a686a37b9204c13b7e281fe516c8f0 Gitweb: https://git.kernel.org/tip/c0a759abf5a686a37b9204c13b7e281fe516c8f0 Author: Peter Zijlstra <[email protected]> AuthorDate: Wed, 19 Sep 2018 10:50:18 +0200 Committer: Thomas Gleixner <[email protected]> CommitDate: Thu, 27 Sep 2018 20:39:40 +0200
x86/mm/cpa: Move flush_tlb_all()
There is an atom errata, where we do a local TLB invalidate right before we return and then do a global TLB invalidate.
Move the global invalidate up a little bit and avoid the local invalidate entirely.
This does put the global invalidate under pgd_lock, but that shouldn't matter.
--- arch/x86/mm/pageattr.c | 44 +++++++++++++++++--------------------------- 1 file changed, 17 insertions(+), 27 deletions(-)
arch/x86/mm/pageattr.c +17 -27
@@ -999,14 +999,24 @@ __split_large_page(struct cpa_data *cpa, pte_t *kpte, unsigned long address,__set_pmd_pte (kpte, address,mk_pte (base,__pgprot (_KERNPG_TABLE)));/* -* Intel Atom errata AAH41 workaround. +* Do a global flush tlb after splitting the large page +* and before we do the actual change page attribute in the PTE. * -* The real fix should be in hw or in a microcode update, but -* we also probabilistically try to reduce the window of having -* a large TLB mixed with 4K TLBs while instruction fetches are -* going on. +* Without this, we violate the TLB application note, that says: +* "The TLBs may contain both ordinary and large-page +* translations for a 4-KByte range of linear addresses. This +* may occur if software modifies the paging structures so that +* the page size used for the address range changes. If the two +* translations differ with respect to page frame or attributes +* (e.g., permissions), processor behavior is undefined and may +* be implementation-specific." +* +* We do this global tlb flush inside the cpa_lock, so that we +* don't allow any other cpu, with stale tlb entries change the +* page attribute in parallel, that also falls into the +* just split large page entry. */ -__flush_tlb_all (); +flush_tlb_all ();spin_unlock (&pgd_lock);return 0 ;
@@ -1531,28 +1541,8 @@ repeat:* We have to split the large page: */ err =split_large_page (cpa, kpte, address); -if (!err) { -/* -* Do a global flush tlb after splitting the large page -* and before we do the actual change page attribute in the PTE. -* -* With out this, we violate the TLB application note, that says -* "The TLBs may contain both ordinary and large-page -* translations for a 4-KByte range of linear addresses. This -* may occur if software modifies the paging structures so that -* the page size used for the address range changes. If the two -* translations differ with respect to page frame or attributes -* (e.g., permissions), processor behavior is undefined and may -* be implementation-specific." -* -* We do this global tlb flush inside the cpa_lock, so that we -* don't allow any other cpu, with stale tlb entries change the -* page attribute in parallel, that also falls into the -* just split large page entry. -*/ -flush_tlb_all (); +if (!err) goto repeat; - }return err; }
tip-bot for Peter Zijlstra
merge bot[tip:x86/mm] x86/mm/cpa: Unconditionally avoid WBINDV when we can
In reply to Peter Zijlstrax86/mm/cpa: Unconditionally avoid WBINDV when we can
Commit-ID: ddd07b750382adc2b78fdfbec47af8a6e0d8ef37 Gitweb: https://git.kernel.org/tip/ddd07b750382adc2b78fdfbec47af8a6e0d8ef37 Author: Peter Zijlstra <[email protected]> AuthorDate: Wed, 19 Sep 2018 10:50:19 +0200 Committer: Thomas Gleixner <[email protected]> CommitDate: Thu, 27 Sep 2018 20:39:41 +0200
x86/mm/cpa: Unconditionally avoid WBINDV when we can
CAT has happened, WBINDV is bad (even before CAT blowing away the entire cache on a multi-core platform wasn't nice), try not to use it ever.
--- arch/x86/mm/pageattr.c | 18 ++---------------- 1 file changed, 2 insertions(+), 16 deletions(-)
arch/x86/mm/pageattr.c +2 -16
@@ -319,26 +319,12 @@ static void cpa_flush_array(unsigned long *start, int numpages, int cache, int in_flags, struct page **pages) {unsigned int i, level; -#ifdef CONFIG_PREEMPT -/* -* Avoid wbinvd() because it causes latencies on all CPUs, -* regardless of any CPU isolation that may be in effect. -* -* This should be extended for CAT enabled systems independent of -* PREEMPT because wbinvd() does not respect the CAT partitions and -* this is exposed to unpriviledged users through the graphics -* subsystem. -*/ -unsigned long do_wbinvd =0 ; -#else -unsigned long do_wbinvd = cache && numpages >=1024 ;/* 4M threshold */ -#endif BUG_ON (irqs_disabled () && !early_boot_irqs_disabled); -on_each_cpu (__cpa_flush_all, (void *) do_wbinvd,1 ); +flush_tlb_all (); -if (!cache || do_wbinvd) +if (!cache)return ;/*
tip-bot for Peter Zijlstra
merge bot[tip:x86/mm] x86/mm/cpa: Use flush_tlb_kernel_range()
In reply to Peter Zijlstrax86/mm/cpa: Use flush_tlb_kernel_range()
Commit-ID: a7295fd53c39ce781a9792c9dd2c8747bf274160 Gitweb: https://git.kernel.org/tip/a7295fd53c39ce781a9792c9dd2c8747bf274160 Author: Peter Zijlstra <[email protected]> AuthorDate: Wed, 19 Sep 2018 10:50:20 +0200 Committer: Thomas Gleixner <[email protected]> CommitDate: Thu, 27 Sep 2018 20:39:41 +0200
x86/mm/cpa: Use flush_tlb_kernel_range()
Both cpa_flush_range() and cpa_flush_array() have a well specified range, use that to do a range based TLB invalidate.
--- arch/x86/mm/pageattr.c | 9 +++++---- 1 file changed, 5 insertions(+), 4 deletions(-)
arch/x86/mm/pageattr.c +5 -4
@@ -293,7 +293,7 @@ static void cpa_flush_range(unsigned long start, int numpages, int cache)BUG_ON (irqs_disabled () && !early_boot_irqs_disabled);WARN_ON (PAGE_ALIGN (start) != start); -flush_tlb_all (); +flush_tlb_kernel_range (start, start +PAGE_SIZE * numpages);if (!cache)return ;
@@ -315,14 +315,15 @@ static void cpa_flush_range(unsigned long start, int numpages, int cache) } } -static void cpa_flush_array (unsigned long *start,int numpages,int cache, +static void cpa_flush_array (unsigned long baddr,unsigned long *start, +int numpages,int cache,int in_flags,struct page **pages) {unsigned int i, level;BUG_ON (irqs_disabled () && !early_boot_irqs_disabled); -flush_tlb_all (); +flush_tlb_kernel_range (baddr, baddr +PAGE_SIZE * numpages);if (!cache)return ;
@@ -1757,7 +1758,7 @@ static int change_page_attr_set_clr(unsigned long *addr, int numpages,*/ if (!ret &&boot_cpu_has (X86_FEATURE_CLFLUSH )) {if (cpa.flags & (CPA_PAGES_ARRAY |CPA_ARRAY )) { -cpa_flush_array (addr, numpages, cache, +cpa_flush_array (baddr, addr, numpages, cache, cpa.flags, pages); }else cpa_flush_range (baddr, numpages, cache);
tip-bot for Peter Zijlstra
merge bot[tip:x86/mm] x86/mm/cpa: Move CLFLUSH test into cpa_flush_range()
In reply to Peter Zijlstrax86/mm/cpa: Move CLFLUSH test into cpa_flush_range()
Commit-ID: 5f464b33b17219a233af1267c621632225bc7acc Gitweb: https://git.kernel.org/tip/5f464b33b17219a233af1267c621632225bc7acc Author: Peter Zijlstra <[email protected]> AuthorDate: Wed, 19 Sep 2018 10:50:21 +0200 Committer: Thomas Gleixner <[email protected]> CommitDate: Thu, 27 Sep 2018 20:39:41 +0200
x86/mm/cpa: Move CLFLUSH test into cpa_flush_range()
Rather than guarding all cpa_flush_range() uses with a CLFLUSH test, put it inside.
--- arch/x86/mm/pageattr.c | 15 +++++++-------- 1 file changed, 7 insertions(+), 8 deletions(-)
arch/x86/mm/pageattr.c +7 -8
@@ -293,6 +293,11 @@ static void cpa_flush_range(unsigned long start, int numpages, int cache)BUG_ON (irqs_disabled () && !early_boot_irqs_disabled);WARN_ON (PAGE_ALIGN (start) != start); +if (!static_cpu_has (X86_FEATURE_CLFLUSH )) { +cpa_flush_all (cache); +return ; + } +flush_tlb_kernel_range (start, start +PAGE_SIZE * numpages);if (!cache)
@@ -2078,10 +2083,7 @@ static int __set_memory_enc_dec(unsigned long addr, int numpages, bool enc)/* * Before changing the encryption attribute, we need to flush caches. */ -if (static_cpu_has (X86_FEATURE_CLFLUSH )) -cpa_flush_range (start, numpages,1 ); -else -cpa_flush_all (1 ); +cpa_flush_range (start, numpages,1 ); ret =__change_page_attr_set_clr (&cpa,1 );
@@ -2092,10 +2094,7 @@ static int __set_memory_enc_dec(unsigned long addr, int numpages, bool enc)* in case TLB flushing gets optimized in the cpa_flush_range() * path use the same logic as above. */ -if (static_cpu_has (X86_FEATURE_CLFLUSH )) -cpa_flush_range (start, numpages,0 ); -else -cpa_flush_all (0 ); +cpa_flush_range (start, numpages,0 );return ret; }
tip-bot for Peter Zijlstra
merge bot[tip:x86/mm] x86/mm/cpa: Move CLFLUSH test into cpa_flush_array()
In reply to Peter Zijlstrax86/mm/cpa: Move CLFLUSH test into cpa_flush_array()
Commit-ID: fce2ce9544e9f098ba828442221ce99c2a5ecb0f Gitweb: https://git.kernel.org/tip/fce2ce9544e9f098ba828442221ce99c2a5ecb0f Author: Peter Zijlstra <[email protected]> AuthorDate: Wed, 19 Sep 2018 10:50:22 +0200 Committer: Thomas Gleixner <[email protected]> CommitDate: Thu, 27 Sep 2018 20:39:42 +0200
x86/mm/cpa: Move CLFLUSH test into cpa_flush_array()
Rather than guarding cpa_flush_array() users with a CLFLUSH test, put it inside.
--- arch/x86/mm/pageattr.c | 27 ++++++++++++++++----------- 1 file changed, 16 insertions(+), 11 deletions(-)
arch/x86/mm/pageattr.c +16 -11
@@ -328,6 +328,11 @@ static void cpa_flush_array(unsigned long baddr, unsigned long *start,BUG_ON (irqs_disabled () && !early_boot_irqs_disabled); +if (!static_cpu_has (X86_FEATURE_CLFLUSH )) { +cpa_flush_all (cache); +return ; + } +flush_tlb_kernel_range (baddr, baddr +PAGE_SIZE * numpages);if (!cache)
@@ -1756,19 +1761,19 @@ static int change_page_attr_set_clr(unsigned long *addr, int numpages, cache = !!pgprot2cachemode (mask_set);/* -* On success we use CLFLUSH, when the CPU supports it to -* avoid the WBINVD. If the CPU does not support it and in the -* error case we fall back to cpa_flush_all (which uses -* WBINVD): +* On error; flush everything to be sure. */ -if (!ret &&boot_cpu_has (X86_FEATURE_CLFLUSH )) { -if (cpa.flags & (CPA_PAGES_ARRAY |CPA_ARRAY )) { -cpa_flush_array (baddr, addr, numpages, cache, - cpa.flags, pages); - }else -cpa_flush_range (baddr, numpages, cache); - }else +if (ret) {cpa_flush_all (cache); + goto out; + } + +if (cpa.flags & (CPA_PAGES_ARRAY |CPA_ARRAY )) { +cpa_flush_array (baddr, addr, numpages, cache, + cpa.flags, pages); + }else { +cpa_flush_range (baddr, numpages, cache); + } out:return ret;
tip-bot for Peter Zijlstra
merge bot[tip:x86/mm] x86/mm/cpa: Factor common code between cpa_flush_*()
In reply to Peter Zijlstrax86/mm/cpa: Factor common code between cpa_flush_*()
Commit-ID: 47e262ac5b84015c4a101ff51767c464fb7497a6 Gitweb: https://git.kernel.org/tip/47e262ac5b84015c4a101ff51767c464fb7497a6 Author: Peter Zijlstra <[email protected]> AuthorDate: Wed, 19 Sep 2018 10:50:23 +0200 Committer: Thomas Gleixner <[email protected]> CommitDate: Thu, 27 Sep 2018 20:39:42 +0200
x86/mm/cpa: Factor common code between cpa_flush_*()
The start of cpa_flush_range() and cpa_flush_array() is the same, use a common function.
--- arch/x86/mm/pageattr.c | 29 +++++++++++++---------------- 1 file changed, 13 insertions(+), 16 deletions(-)
arch/x86/mm/pageattr.c +13 -16
@@ -285,22 +285,28 @@ static void cpa_flush_all(unsigned long cache)on_each_cpu (__cpa_flush_all, (void *) cache,1 ); } -static void cpa_flush_range (unsigned long start,int numpages,int cache) +static bool __cpa_flush_range (unsigned long start,int numpages,int cache) { -unsigned int i, level; -unsigned long addr; -BUG_ON (irqs_disabled () && !early_boot_irqs_disabled); +WARN_ON (PAGE_ALIGN (start) != start);if (!static_cpu_has (X86_FEATURE_CLFLUSH )) {cpa_flush_all (cache); -return ; +return true; }flush_tlb_kernel_range (start, start +PAGE_SIZE * numpages); -if (!cache) +return !cache; +} + +static void cpa_flush_range (unsigned long start,int numpages,int cache) +{ +unsigned int i, level; +unsigned long addr; + +if (__cpa_flush_range (start, numpages, cache))return ;/*
@@ -326,16 +332,7 @@ static void cpa_flush_array(unsigned long baddr, unsigned long *start, {unsigned int i, level; -BUG_ON (irqs_disabled () && !early_boot_irqs_disabled); - -if (!static_cpu_has (X86_FEATURE_CLFLUSH )) { -cpa_flush_all (cache); -return ; - } - -flush_tlb_kernel_range (baddr, baddr +PAGE_SIZE * numpages); - -if (!cache) +if (__cpa_flush_range (baddr, numpages, cache))return ;/*
tip-bot for Peter Zijlstra
merge bot[tip:x86/mm] x86/mm/cpa: Optimize __cpa_flush_range()
In reply to Peter Zijlstrax86/mm/cpa: Optimize __cpa_flush_range()
Commit-ID: 7904ba8a66f400182a204893c92098994e22a88d Gitweb: https://git.kernel.org/tip/7904ba8a66f400182a204893c92098994e22a88d Author: Peter Zijlstra <[email protected]> AuthorDate: Wed, 19 Sep 2018 10:50:24 +0200 Committer: Thomas Gleixner <[email protected]> CommitDate: Thu, 27 Sep 2018 20:39:42 +0200
x86/mm/cpa: Optimize __cpa_flush_range()
If we IPI for WBINDV, then we might as well kill the entire TLB too. But if we don't have to invalidate cache, there is no reason not to use a range TLB flush.
--- arch/x86/mm/pageattr.c | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-)
arch/x86/mm/pageattr.c +1 -1
@@ -291,7 +291,7 @@ static bool __cpa_flush_range(unsigned long start, int numpages, int cache)WARN_ON (PAGE_ALIGN (start) != start); -if (!static_cpu_has (X86_FEATURE_CLFLUSH )) { +if (cache && !static_cpu_has (X86_FEATURE_CLFLUSH )) {cpa_flush_all (cache);return true; }