Patch "kaiser: enhanced by kernel and user PCIDs" has been added to the 4.4-stable tree
Subsystems (5)
This is a note to let you know that I've just added the patch titled
kaiser: enhanced by kernel and user PCIDs
to the 4.4-stable tree which can be found at:
http://www.kernel.org/git/?p=linux/kernel/git/stable/stable-queue.git;a=summary
The filename of the patch is:
kaiser-enhanced-by-kernel-and-user-pcids.patch
and it can be found in the queue-4.4 subdirectory.
If you, or anyone else, feels it should not be added to the stable tree, please let <[email protected]> know about it.
Quoted 1 line
From foo@baz Wed Jan 3 18:58:12 CET 2018
From: Dave Hansen <[email protected]>
Date: Wed, 30 Aug 2017 16:23:00 -0700
Subject: kaiser: enhanced by kernel and user PCIDs
From: Dave Hansen <[email protected]>
Merged performance improvements to Kaiser, using distinct kernel and user Process Context Identifiers to minimize the TLB flushing.
--- arch/x86/entry/entry_64.S | 10 ++++- arch/x86/entry/entry_64_compat.S | 1 arch/x86/include/asm/cpufeature.h | 1 arch/x86/include/asm/kaiser.h | 15 ++++++- arch/x86/include/asm/pgtable_types.h | 26 +++++++++++++ arch/x86/include/asm/tlbflush.h | 54 +++++++++++++++++++++++----- arch/x86/include/uapi/asm/processor-flags.h | 3 + arch/x86/kernel/cpu/common.c | 34 +++++++++++++++++ arch/x86/kvm/x86.c | 3 + arch/x86/mm/kaiser.c | 7 +++ arch/x86/mm/tlb.c | 46 ++++++++++++++++++++++- 11 files changed, 182 insertions(+), 18 deletions(-)
arch/x86/entry/entry_64.S +8 -2
@@ -1291,7 +1291,10 @@ ENTRY(nmi) /* %rax is saved above, so OK to clobber here */ movq %cr3, %rax pushq %rax - andq $(~KAISER_SHADOW_PGD_OFFSET), %rax + /* mask off "user" bit of pgd address and 12 PCID bits: */ + andq $(~(X86_CR3_PCID_ASID_MASK | KAISER_SHADOW_PGD_OFFSET)), %rax + /* Add back kernel PCID and "no flush" bit */ + orq X86_CR3_PCID_KERN_VAR, %rax movq %rax, %cr3 #endif call do_nmi
@@ -1532,7 +1535,10 @@ end_repeat_nmi: /* %rax is saved above, so OK to clobber here */ movq %cr3, %rax pushq %rax - andq $(~KAISER_SHADOW_PGD_OFFSET), %rax + /* mask off "user" bit of pgd address and 12 PCID bits: */ + andq $(~(X86_CR3_PCID_ASID_MASK | KAISER_SHADOW_PGD_OFFSET)), %rax + /* Add back kernel PCID and "no flush" bit */ + orq X86_CR3_PCID_KERN_VAR, %rax movq %rax, %cr3 #endif
arch/x86/entry/entry_64_compat.S +1 -0
@@ -13,6 +13,7 @@ #include <asm/irqflags.h> #include <asm/asm.h> #include <asm/smap.h> +#include <asm/pgtable_types.h> #include <asm/kaiser.h> #include <linux/linkage.h> #include <linux/err.h>
arch/x86/include/asm/cpufeature.h +1 -0
@@ -187,6 +187,7 @@#define X86_FEATURE_ARAT ( 7*32+ 1)/* Always Running APIC Timer */ #define X86_FEATURE_CPB ( 7*32+ 2)/* AMD Core Performance Boost */ #define X86_FEATURE_EPB ( 7*32+ 3)/* IA32_ENERGY_PERF_BIAS support */ +#define X86_FEATURE_INVPCID_SINGLE ( 7*32+ 4)/* Effectively INVPCID && CR4.PCIDE=1 */ #define X86_FEATURE_PLN ( 7*32+ 5)/* Intel Power Limit Notification */ #define X86_FEATURE_PTS ( 7*32+ 6)/* Intel Package Thermal Status */ #define X86_FEATURE_DTHERM ( 7*32+ 7)/* Digital Thermal Sensor */
arch/x86/include/asm/kaiser.h +13 -2
@@ -1,5 +1,8 @@#ifndef _ASM_X86_KAISER_H#define _ASM_X86_KAISER_H + +#include <uapi/asm/processor-flags.h> /* For PCID constants */ +/* * This file includes the definitions for the KAISER feature. * KAISER is a counter measure against x86_64 side channel attacks on
@@ -21,13 +24,21 @@ .macro _SWITCH_TO_KERNEL_CR3 reg movq %cr3, \reg -andq $(~KAISER_SHADOW_PGD_OFFSET), \reg +andq $(~(X86_CR3_PCID_ASID_MASK | KAISER_SHADOW_PGD_OFFSET)), \reg +orq X86_CR3_PCID_KERN_VAR, \reg movq \reg, %cr3 .endm .macro _SWITCH_TO_USER_CR3 reg movq %cr3, \reg -orq $(KAISER_SHADOW_PGD_OFFSET), \reg +andq $(~(X86_CR3_PCID_ASID_MASK | KAISER_SHADOW_PGD_OFFSET)), \reg +/* + * This can obviously be one instruction by putting the + * KAISER_SHADOW_PGD_OFFSET bit in the X86_CR3_PCID_USER_VAR. + * But, just leave it now for simplicity. + */ +orq X86_CR3_PCID_USER_VAR, \reg +orq $(KAISER_SHADOW_PGD_OFFSET), \reg movq \reg, %cr3 .endm
arch/x86/include/asm/pgtable_types.h +26 -0
@@ -106,6 +106,32 @@ _PAGE_SOFT_DIRTY)#define _HPAGE_CHG_MASK (_PAGE_CHG_MASK | _PAGE_PSE) +/* The ASID is the lower 12 bits of CR3 */ +#define X86_CR3_PCID_ASID_MASK (_AC((1<<12)-1,UL)) + +/* Mask for all the PCID-related bits in CR3: */ +#define X86_CR3_PCID_MASK (X86_CR3_PCID_NOFLUSH | X86_CR3_PCID_ASID_MASK) +#if defined(CONFIG_KAISER ) && defined(CONFIG_X86_64 ) +#define X86_CR3_PCID_ASID_KERN (_AC(0x4,UL)) +#define X86_CR3_PCID_ASID_USER (_AC(0x6,UL)) + +#define X86_CR3_PCID_KERN_FLUSH (X86_CR3_PCID_ASID_KERN) +#define X86_CR3_PCID_USER_FLUSH (X86_CR3_PCID_ASID_USER) +#define X86_CR3_PCID_KERN_NOFLUSH (X86_CR3_PCID_NOFLUSH | X86_CR3_PCID_ASID_KERN) +#define X86_CR3_PCID_USER_NOFLUSH (X86_CR3_PCID_NOFLUSH | X86_CR3_PCID_ASID_USER) +#else +#define X86_CR3_PCID_ASID_KERN (_AC(0x0,UL)) +#define X86_CR3_PCID_ASID_USER (_AC(0x0,UL)) +/* +* PCIDs are unsupported on 32-bit and none of these bits can be +* set in CR3: +*/ +#define X86_CR3_PCID_KERN_FLUSH (0) +#define X86_CR3_PCID_USER_FLUSH (0) +#define X86_CR3_PCID_KERN_NOFLUSH (0) +#define X86_CR3_PCID_USER_NOFLUSH (0) +#endif +/* * The cache modes defined here are used to translate between pure SW usage * and the HW defined cache mode bits and/or PAT entries.
arch/x86/include/asm/tlbflush.h +45 -9
79 lines not on this page · raw message
arch/x86/include/uapi/asm/processor-flags.h +2 -1
@@ -77,7 +77,8 @@#define X86_CR3_PWT _BITUL(X86_CR3_PWT_BIT)#define X86_CR3_PCD_BIT 4/* Page Cache Disable */ #define X86_CR3_PCD _BITUL(X86_CR3_PCD_BIT) -#define X86_CR3_PCID_MASK _AC(0x00000fff,UL)/* PCID Mask */ +#define X86_CR3_PCID_NOFLUSH_BIT 63/* Preserve old PCID */ +#define X86_CR3_PCID_NOFLUSH _BITULL(X86_CR3_PCID_NOFLUSH_BIT)/* * Intel CPU features in CR4
arch/x86/kernel/cpu/common.c +34 -0
@@ -321,11 +321,45 @@ static __always_inline void setup_smap(s } } +/* +* These can have bit 63 set, so we can not just use a plain "or" +* instruction to get their value or'd into CR3. It would take +* another register. So, we use a memory reference to these +* instead. +* +* This is also handy because systems that do not support +* PCIDs just end up or'ing a 0 into their CR3, which does +* no harm. +*/ +__aligned(PAGE_SIZE) unsigned long X86_CR3_PCID_KERN_VAR =0 ; +__aligned(PAGE_SIZE) unsigned long X86_CR3_PCID_USER_VAR =0 ; +static void setup_pcid (struct cpuinfo_x86 *c) {if (cpu_has (c,X86_FEATURE_PCID )) {if (cpu_has (c,X86_FEATURE_PGE )) {cr4_set_bits (X86_CR4_PCIDE ); +/* +* These variables are used by the entry/exit +* code to change PCIDs. +*/ +#ifdef CONFIG_KAISER +X86_CR3_PCID_KERN_VAR =X86_CR3_PCID_KERN_NOFLUSH ; +X86_CR3_PCID_USER_VAR =X86_CR3_PCID_USER_NOFLUSH ; +#endif +/* +* INVPCID has two "groups" of types: +* 1/2: Invalidate an individual address +* 3/4: Invalidate all contexts +* +* 1/2 take a PCID, but 3/4 do not. So, 3/4 +* ignore the PCID argument in the descriptor. +* But, we have to be careful not to call 1/2 +* with an actual non-zero PCID in them before +* we do the above cr4_set_bits(). +*/ +if (cpu_has (c,X86_FEATURE_INVPCID )) +set_cpu_cap (c,X86_FEATURE_INVPCID_SINGLE ); }else {/* * flush_tlb_all(), as currently implemented, won't
arch/x86/kvm/x86.c +2 -1
@@ -759,7 +759,8 @@ int kvm_set_cr4(struct kvm_vcpu *vcpu, ureturn 1 ;/* PCID can not be enabled when cr3[11:0]!=000H or EFER.LMA=0 */ -if ((kvm_read_cr3 (vcpu) &X86_CR3_PCID_MASK ) || !is_long_mode (vcpu)) +if ((kvm_read_cr3 (vcpu) &X86_CR3_PCID_ASID_MASK ) || + !is_long_mode (vcpu))return 1 ; }
arch/x86/mm/kaiser.c +7 -0
@@ -240,6 +240,8 @@ static void __init kaiser_init_all_pgds( }while (0 ) extern char __per_cpu_user_mapped_start[], __per_cpu_user_mapped_end[]; +extern unsigned long X86_CR3_PCID_KERN_VAR ; +extern unsigned long X86_CR3_PCID_USER_VAR ;/* * If anything in here fails, we will likely die on one of the * first kernel->user transitions and init will die. But, we
@@ -290,6 +292,11 @@ void __init kaiser_init(void)kaiser_add_user_map_early (&debug_idt_table,sizeof (gate_desc) *NR_VECTORS , __PAGE_KERNEL); + +kaiser_add_user_map_early (&X86_CR3_PCID_KERN_VAR ,PAGE_SIZE , + __PAGE_KERNEL); +kaiser_add_user_map_early (&X86_CR3_PCID_USER_VAR ,PAGE_SIZE , + __PAGE_KERNEL); }/* Add a mapping to the shadow mapping, and synchronize the mappings */
arch/x86/mm/tlb.c +43 -3
70 lines not on this page · raw message
Patches currently in stable-queue which might be from [email protected] are
queue-4.4/x86-paravirt-dont-patch-flush_tlb_single.patch queue-4.4/kaiser-merged-update.patch queue-4.4/kaiser-enhanced-by-kernel-and-user-pcids.patch
Dave Hansen
maintainerQuoted 8 lines · On 01/03/2018 10:03 AM, [email protected] wrote:
0700
Subject: kaiser: enhanced by kernel and user PCIDsFrom: Dave Hansen <[email protected]>
Merged performance improvements to Kaiser, using distinct kernel and user Process Context Identifiers to minimize the TLB flushing.
For this and the 4.9 version:
Greg KH
maintainerQuoted 7 lines · Dave Hansen
Quoted 5 lines · On 01/03/2018 10:03 AM, [email protected] wrote:
0700
Subject: kaiser: enhanced by kernel and user PCIDsFrom: Dave Hansen <[email protected]>
Quoted 6 lines
Quoted 2 lines
Merged performance improvements to Kaiser, using distinct kernel and user Process Context Identifiers to minimize the TLB flushing.
For this and the 4.9 version:
Signed-off-by Dave Hansen <[email protected]>
Thanks for this, and the other sob.
greg k-h