Use INVPCID to flush global mappings
cover Use INVPCID to flush global mappings
I wrote these patches sometime ago to explore PCID and INVPCID. I haven't thought through whether how to use both in Xen yet. But seeing Juergen laid out his thought on PCID and INVPCID I think some of the patches can be useful.
I had done some benchmark on the speed in one of my older branch by inserting some trace points before and after the flush. It showed that twiddling CR4.PGE is 3 to 5 times slower than invpcid.
This series is in incomplete -- obviously we have CR4.PGE twiddling in a few other places. But if you think it is beneficial I can try to convert those places as well.
Wei.
Wei Liu (4):
x86: introduce cpu_has_invpcid
x86: report if PCID and INVPCID are supported
x86: add invpcid.h
x86: use invpcid to do global flush
xen/arch/x86/flushtlb.c | 22 ++++++++++++--- xen/arch/x86/setup.c | 7 +++++ xen/include/asm-x86/cpufeature.h | 1 + xen/include/asm-x86/invpcid.h | 61 ++++++++++++++++++++++++++++++++++++++++ 4 files changed, 87 insertions(+), 4 deletions(-) create mode 100644 xen/include/asm-x86/invpcid.h
Signature · 7 lines
2.11.0 _______________________________________________ Xen-devel mailing list [email protected] https://lists.xenproject.org/mailman/listinfo/xen-devel
RFC PATCH 1/4 x86: introduce cpu_has_invpcid
--- xen/include/asm-x86/cpufeature.h | 1 + 1 file changed, 1 insertion(+)
xen/include/asm-x86/cpufeature.h +1 -0
@@ -93,6 +93,7 @@#define cpu_has_avx2 boot_cpu_has(X86_FEATURE_AVX2)#define cpu_has_smep boot_cpu_has(X86_FEATURE_SMEP)#define cpu_has_bmi2 boot_cpu_has(X86_FEATURE_BMI2) +#define cpu_has_invpcid boot_cpu_has(X86_FEATURE_INVPCID)#define cpu_has_rtm boot_cpu_has(X86_FEATURE_RTM)#define cpu_has_fpu_sel (!boot_cpu_has(X86_FEATURE_NO_FPU_SEL))#define cpu_has_mpx boot_cpu_has(X86_FEATURE_MPX)
Signature · 7 lines
2.11.0 _______________________________________________ Xen-devel mailing list [email protected] https://lists.xenproject.org/mailman/listinfo/xen-devel
Re: [PATCH RFC 1/4] x86: introduce cpu_has_invpcid
xen/include/asm-x86/cpufeature.h line 99
Quoted 11 lines
Quoted 1 line
Quoted 1 line · On 15.02.18 at 13:10, <[email protected]> wrote:
xen/include/asm-x86/cpufeature.h +1
-0@@ -93,6 +93,7 @@#define cpu_has_avx2 boot_cpu_has(X86_FEATURE_AVX2)#define cpu_has_smep boot_cpu_has(X86_FEATURE_SMEP)#define cpu_has_bmi2 boot_cpu_has(X86_FEATURE_BMI2) +#define cpu_has_invpcid boot_cpu_has(X86_FEATURE_INVPCID)#define cpu_has_rtm boot_cpu_has(X86_FEATURE_RTM)#define cpu_has_fpu_sel (!boot_cpu_has(X86_FEATURE_NO_FPU_SEL))#define cpu_has_mpx boot_cpu_has(X86_FEATURE_MPX)
Please fold into whatever patch is first using the new macro.
Jan
_______________________________________________
Xen-devel mailing list
[email protected]
https://lists.xenproject.org/mailman/listinfo/xen-devel
RFC PATCH 2/4 x86: report if PCID and INVPCID are supported
In reply to Wei Liu--- xen/arch/x86/setup.c | 7 +++++++ 1 file changed, 7 insertions(+)
xen/arch/x86/setup.c +7 -0
@@ -1701,6 +1701,13 @@ void __init noreturn __start_xen(unsigned long mbi_p) cpu_has_nx ?XENLOG_INFO :XENLOG_WARNING "Warning: " , cpu_has_nx ? "" : "not "); + +printk (XENLOG_INFO +"PCID (Process-Context IDentifier) %ssupported\n" , + cpu_has_pcid ?"" :"not " ); + +printk (XENLOG_INFO "INVPCID %ssupported\n" , cpu_has_invpcid ?"" :"not " ); +/* * We're going to setup domain0 using the module(s) that we stashed safely * above our heap. The second module, if present, is an initrd ramdisk.
Signature · 7 lines
2.11.0 _______________________________________________ Xen-devel mailing list [email protected] https://lists.xenproject.org/mailman/listinfo/xen-devel
RFC PATCH 3/4 x86: add invpcid.h
In reply to Wei LiuProvide the functions needed for different modes.
--- xen/include/asm-x86/invpcid.h | 61 +++++++++++++++++++++++++++++++++++++++++++ 1 file changed, 61 insertions(+) create mode 100644 xen/include/asm-x86/invpcid.h
xen/include/asm-x86/invpcid.h new file +61 -0
@@ -0,0 +1,61 @@ +#ifndef _ASM_X86_INVPCID_H_ +#define _ASM_X86_INVPCID_H_ + +#include <xen/types.h> + +#define INVPCID_TYPE_INDIV_ADDR 0 +#define INVPCID_TYPE_SINGLE_CTXT 1 +#define INVPCID_TYPE_ALL_INCL_GLOBAL 2 +#define INVPCID_TYPE_ALL_NON_GLOBAL 3 + +struct invpcid_desc { +uint64_t pcid:12 ; +uint64_t reserved:52 ; +uint64_t addr; +}; + +static inline void invpcid (unsigned long pcid,unsigned long addr, +unsigned long type) +{ +struct invpcid_desc desc = { .pcid = pcid, .addr = addr }; + + asmvolatile ("invpcid (%0), %1" + : :"r" (&desc),"r" (type) :"memory" ); +} + +/* Flush all mappings for a given PCID and addr, not including globals */ +static inline void invpcid_flush_one (unsigned long pcid, +unsigned long addr) +{ +invpcid (pcid, addr,INVPCID_TYPE_INDIV_ADDR ); +} + +/* Flush all mappings for a given PCID, not including globals */ +static inline void invpcid_flush_single_context (unsigned long pcid) +{ +invpcid (pcid,0 ,INVPCID_TYPE_SINGLE_CTXT ); +} + +/* Flush all mappings, including globals, for all PCIDs */ +static inline void invpcid_flush_all (void ) +{ +invpcid (0 ,0 ,INVPCID_TYPE_ALL_INCL_GLOBAL ); +} + +/* Flush all mappings for all PCIDs, excluding globals */ +static inline void invpcid_flush_all_nonglobals (void ) +{ +invpcid (0 ,0 ,INVPCID_TYPE_ALL_NON_GLOBAL ); +} + +#endif /* _ASM_X86_INVPCID_H_ */ + +/* +* Local variables: +* mode: C +* c-file-style: "BSD" +* c-basic-offset: 4 +* tab-width: 4 +* indent-tabs-mode: nil +* End: +*/
Signature · 7 lines
2.11.0 _______________________________________________ Xen-devel mailing list [email protected] https://lists.xenproject.org/mailman/listinfo/xen-devel
Re: [PATCH RFC 3/4] x86: add invpcid.h
Quoted 27 lines · Wei Liu
Provide the functions needed for different modes.
Signed-off-by Wei Liu <[email protected]>--- xen/include/asm-x86/invpcid.h | 61 +++++++++++++++++++++++++++++++++++++++++++ 1 file changed, 61 insertions(+) create mode 100644 xen/include/asm-x86/invpcid.hxen/include/asm-x86/invpcid.h new file +13
-0@@ -0,0 +1,61 @@ +#ifndef _ASM_X86_INVPCID_H_ +#define _ASM_X86_INVPCID_H_ + +#include <xen/types.h> + +#define INVPCID_TYPE_INDIV_ADDR 0 +#define INVPCID_TYPE_SINGLE_CTXT 1 +#define INVPCID_TYPE_ALL_INCL_GLOBAL 2 +#define INVPCID_TYPE_ALL_NON_GLOBAL 3 + +struct invpcid_desc { + uint64_t pcid:12; + uint64_t reserved:52;
xen/include/asm-x86/invpcid.h line 23
Quoted 10 lines
xen/include/asm-x86/invpcid.h new file +10
-0+uint64_t addr; +}; + +static inline void invpcid (unsigned long pcid,unsigned long addr, +unsigned long type) +{ +struct invpcid_desc desc = { .pcid = pcid, .addr = addr }; + + asmvolatile ("invpcid (%0), %1" + : :"r" (&desc),"r" (type) :"memory" );
invpcid %[desc], %[type]
And you can use [desc] "m" (desc) for the constraint. The structure will be built on the stack, meaning that an %rsp based memory reference is more efficient than forcing the use of a register.
We probably also need a -DHAVE_GAS_INVPCID, as INVPCID is newer than some of the instruction groups we already check for.
~Andrew
_______________________________________________
Xen-devel mailing list
[email protected]
https://lists.xenproject.org/mailman/listinfo/xen-devel
Re: [PATCH RFC 3/4] x86: add invpcid.h
Quoted 41 lines · Andrew Cooper
Quoted 27 lines · Wei Liu
Provide the functions needed for different modes.
Signed-off-by Wei Liu <[email protected]>--- xen/include/asm-x86/invpcid.h | 61 +++++++++++++++++++++++++++++++++++++++++++ 1 file changed, 61 insertions(+) create mode 100644 xen/include/asm-x86/invpcid.hxen/include/asm-x86/invpcid.h new file +13
-0@@ -0,0 +1,61 @@ +#ifndef _ASM_X86_INVPCID_H_ +#define _ASM_X86_INVPCID_H_ + +#include <xen/types.h> + +#define INVPCID_TYPE_INDIV_ADDR 0 +#define INVPCID_TYPE_SINGLE_CTXT 1 +#define INVPCID_TYPE_ALL_INCL_GLOBAL 2 +#define INVPCID_TYPE_ALL_NON_GLOBAL 3 + +struct invpcid_desc { + uint64_t pcid:12; + uint64_t reserved:52;xen/include/asm-x86/invpcid.h line 23
Quoted 10 lines
xen/include/asm-x86/invpcid.h new file +10
-0+uint64_t addr; +}; + +static inline void invpcid (unsigned long pcid,unsigned long addr, +unsigned long type) +{ +struct invpcid_desc desc = { .pcid = pcid, .addr = addr }; + + asmvolatile ("invpcid (%0), %1" + : :"r" (&desc),"r" (type) :"memory" );invpcid %[desc], %[type]
Quoted 3 lines
And you can use [desc] "m" (desc) for the constraint. The structure will be built on the stack, meaning that an %rsp based memory reference is more efficient than forcing the use of a register.
NP.
Quoted 4 lines
We probably also need a -DHAVE_GAS_INVPCID, as INVPCID is newer than some of the instruction groups we already check for.
Or we can just use the byte code directly -- it is the same for both 64 and 32 bit, then manually specify the ModRM byte. That's what Linux does.
Wei.
_______________________________________________
Xen-devel mailing list
[email protected]
https://lists.xenproject.org/mailman/listinfo/xen-devel
Re: [PATCH RFC 3/4] x86: add invpcid.h
Quoted 44 lines · Wei Liu
Quoted 40 lines · Andrew Cooper
Quoted 27 lines · Wei Liu
Provide the functions needed for different modes.
Signed-off-by Wei Liu <[email protected]>--- xen/include/asm-x86/invpcid.h | 61 +++++++++++++++++++++++++++++++++++++++++++ 1 file changed, 61 insertions(+) create mode 100644 xen/include/asm-x86/invpcid.hxen/include/asm-x86/invpcid.h new file +13
-0@@ -0,0 +1,61 @@ +#ifndef _ASM_X86_INVPCID_H_ +#define _ASM_X86_INVPCID_H_ + +#include <xen/types.h> + +#define INVPCID_TYPE_INDIV_ADDR 0 +#define INVPCID_TYPE_SINGLE_CTXT 1 +#define INVPCID_TYPE_ALL_INCL_GLOBAL 2 +#define INVPCID_TYPE_ALL_NON_GLOBAL 3 + +struct invpcid_desc { + uint64_t pcid:12; + uint64_t reserved:52;xen/include/asm-x86/invpcid.h line 23
Quoted 10 lines
xen/include/asm-x86/invpcid.h new file +10
-0+uint64_t addr; +}; + +static inline void invpcid (unsigned long pcid,unsigned long addr, +unsigned long type) +{ +struct invpcid_desc desc = { .pcid = pcid, .addr = addr }; + + asmvolatile ("invpcid (%0), %1" + : :"r" (&desc),"r" (type) :"memory" );invpcid %[desc], %[type]
Quoted 3 lines
And you can use [desc] "m" (desc) for the constraint. The structure will be built on the stack, meaning that an %rsp based memory reference is more efficient than forcing the use of a register.
Quoted 8 lines
NP.
Quoted 3 lines
We probably also need a -DHAVE_GAS_INVPCID, as INVPCID is newer than some of the instruction groups we already check for.
Or we can just use the byte code directly -- it is the same for both 64 and 32 bit, then manually specify the ModRM byte. That's what Linux does.
See the vmx __invept() wrapper, which is very similar, but doesn't need the BUG() handling.
~Andrew
_______________________________________________
Xen-devel mailing list
[email protected]
https://lists.xenproject.org/mailman/listinfo/xen-devel
Re: [PATCH RFC 3/4] x86: add invpcid.h
In reply to Wei LiuQuoted 2 lines
Quoted 1 line
Quoted 1 line · On 15.02.18 at 13:10, <[email protected]> wrote:
Provide the functions needed for different modes.
Do we really need all of these? Let's not have dead code sit around and risk it becoming stale.
Jan
_______________________________________________
Xen-devel mailing list
[email protected]
https://lists.xenproject.org/mailman/listinfo/xen-devel
Re: [PATCH RFC 3/4] x86: add invpcid.h
Quoted 6 lines · Jan Beulich
Quoted 2 lines
Quoted 1 line
Quoted 1 line · On 15.02.18 at 13:10, <[email protected]> wrote:
Provide the functions needed for different modes.
Do we really need all of these? Let's not have dead code sit around and risk it becoming stale.
They will be needed when we want to use PCID and INVPCID at the same time, which seems to be the plan?
Wei.
_______________________________________________
Xen-devel mailing list
[email protected]
https://lists.xenproject.org/mailman/listinfo/xen-devel
Re: [PATCH RFC 3/4] x86: add invpcid.h
Quoted 11 lines
Quoted 1 line
Quoted 1 line · On 15.02.18 at 13:35, <[email protected]> wrote:
Quoted 6 lines · Jan Beulich
Quoted 2 lines
Quoted 1 line
Quoted 1 line · On 15.02.18 at 13:10, <[email protected]> wrote:
Provide the functions needed for different modes.
Do we really need all of these? Let's not have dead code sit around and risk it becoming stale.
They will be needed when we want to use PCID and INVPCID at the same time, which seems to be the plan?
I know of INVPCID plans only, not any PCID ones.
Jan
_______________________________________________
Xen-devel mailing list
[email protected]
https://lists.xenproject.org/mailman/listinfo/xen-devel
RFC PATCH 4/4 x86: use invpcid to do global flush
In reply to Wei Liu--- xen/arch/x86/flushtlb.c | 22 ++++++++++++++++++---- 1 file changed, 18 insertions(+), 4 deletions(-)
xen/arch/x86/flushtlb.c +18 -4
@@ -9,6 +9,7 @@#include <xen/sched.h> #include <xen/softirq.h> +#include <asm/invpcid.h> #include <asm/flushtlb.h> #include <asm/page.h>
@@ -120,11 +121,24 @@ unsigned int flush_area_local(const void *va, unsigned int flags)else {u32 t =pre_flush (); -unsigned long cr4 =read_cr4 (); -write_cr4 (cr4 & ~X86_CR4_PGE ); -barrier (); -write_cr4 (cr4); +if ( !cpu_has_invpcid ) + { +unsigned long cr4 =read_cr4 (); + +write_cr4 (cr4 & ~X86_CR4_PGE ); +barrier (); +write_cr4 (cr4); + } +else + { +/* +* Using invpcid to flush all mappings works +* regardless of whether PCID is enabled or not. +* It is faster than read-modify-write CR4. +*/ +invpcid_flush_all (); + }post_flush (t); }
Signature · 7 lines
2.11.0 _______________________________________________ Xen-devel mailing list [email protected] https://lists.xenproject.org/mailman/listinfo/xen-devel
Quoted 11 lines · Wei Liu
I wrote these patches sometime ago to explore PCID and INVPCID. I haven't thought through whether how to use both in Xen yet. But seeing Juergen laid out his thought on PCID and INVPCID I think some of the patches can be useful.
I had done some benchmark on the speed in one of my older branch by inserting some trace points before and after the flush. It showed that twiddling CR4.PGE is 3 to 5 times slower than invpcid.
This series is in incomplete -- obviously we have CR4.PGE twiddling in a few other places. But if you think it is beneficial I can try to convert those places as well.
I just did a little experiment by replacing the %cr4 based TLB flush in Jan's XPTI patches by invpcid, on top of my last XPTI speedup patch (no fancy ALTERNATIVE, just a plain replacement).
Doing a parallel build of the hypervisor in dom0 showed following data:
real user sys
xpti=no 61.2 167.7 71.9 xpti=yes 112.1 170.1 141.8 + my speedup 103.0 171.2 131.2 + invpcid 99.0 170.2 122.0
So system time reduction due to invpcid is quite nice.
Juergen
_______________________________________________
Xen-devel mailing list
[email protected]
https://lists.xenproject.org/mailman/listinfo/xen-devel