arch/x86: Optionally flush L1D on context switch
PATCH 1/3 arch/x86/kvm: Refactor l1d flush lifecycle management
In reply to Balbir SinghSplit out the allocation and free routines to be used in a follow up set of patches (to reuse for L1D flushing).
--- arch/x86/include/asm/cacheflush.h | 3 +++ arch/x86/kernel/Makefile | 1 + arch/x86/kernel/l1d_flush.c | 36 +++++++++++++++++++++++++++++++ arch/x86/kvm/vmx/vmx.c | 25 +++------------------ 4 files changed, 43 insertions(+), 22 deletions(-) create mode 100644 arch/x86/kernel/l1d_flush.c
arch/x86/include/asm/cacheflush.h +3 -0
@@ -6,6 +6,9 @@#include <asm-generic/cacheflush.h> #include <asm/special_insns.h> +#define L1D_CACHE_ORDER 4void clflush_cache_range (void *addr,unsigned int size); +void *alloc_l1d_flush_pages (void ); +void cleanup_l1d_flush_pages (void *l1d_flush_pages);#endif /* _ASM_X86_CACHEFLUSH_H */
arch/x86/kernel/Makefile +1 -0
@@ -160,3 +160,4 @@ ifeq ($(CONFIG_X86_64),y) endif obj-$(CONFIG_IMA_SECURE_AND_OR_TRUSTED_BOOT) += ima_arch.o +obj-y += l1d_flush.o
arch/x86/kernel/l1d_flush.c new file +36 -0
@@ -0,0 +1,36 @@ +#include <linux/mm.h> +#include <asm/cacheflush.h> + +void *alloc_l1d_flush_pages (void ) +{ +struct page *page; +void *l1d_flush_pages =NULL ; +int i; + +/* +* This allocation for l1d_flush_pages is not tied to a VM/task's +* lifetime and so should not be charged to a memcg. +*/ + page =alloc_pages (GFP_KERNEL ,L1D_CACHE_ORDER ); +if (!page) +return NULL ; + l1d_flush_pages =page_address (page); + +/* +* Initialize each page with a different pattern in +* order to protect against KSM in the nested +* virtualization case. +*/ +for (i =0 ; i <1u <<L1D_CACHE_ORDER ; ++i) { +memset (l1d_flush_pages + i *PAGE_SIZE , i +1 , +PAGE_SIZE ); + } +return l1d_flush_pages; +} +EXPORT_SYMBOL_GPL (alloc_l1d_flush_pages); + +void cleanup_l1d_flush_pages (void *l1d_flush_pages) +{ +free_pages ((unsigned long )l1d_flush_pages,L1D_CACHE_ORDER ); +} +EXPORT_SYMBOL_GPL (cleanup_l1d_flush_pages);
arch/x86/kvm/vmx/vmx.c +3 -22
@@ -203,14 +203,10 @@ static const struct { [VMENTER_L1D_FLUSH_NOT_REQUIRED] = {"not required", false}, }; -#define L1D_CACHE_ORDER 4static void *vmx_l1d_flush_pages;static int vmx_setup_l1d_flush (enum vmx_l1d_flush_state l1tf) { -struct page *page; -unsigned int i; -if (!boot_cpu_has_bug (X86_BUG_L1TF )) { l1tf_vmx_mitigation =VMENTER_L1D_FLUSH_NOT_REQUIRED ;return 0 ;
@@ -253,24 +249,9 @@ static int vmx_setup_l1d_flush(enum vmx_l1d_flush_state l1tf)if (l1tf !=VMENTER_L1D_FLUSH_NEVER && !vmx_l1d_flush_pages && !boot_cpu_has (X86_FEATURE_FLUSH_L1D )) { -/* -* This allocation for vmx_l1d_flush_pages is not tied to a VM -* lifetime and so should not be charged to a memcg. -*/ - page =alloc_pages (GFP_KERNEL ,L1D_CACHE_ORDER ); -if (!page) + vmx_l1d_flush_pages =alloc_l1d_flush_pages (); +if (!vmx_l1d_flush_pages)return -ENOMEM ; - vmx_l1d_flush_pages =page_address (page); - -/* -* Initialize each page with a different pattern in -* order to protect against KSM in the nested -* virtualization case. -*/ -for (i =0 ; i <1u <<L1D_CACHE_ORDER ; ++i) { -memset (vmx_l1d_flush_pages + i *PAGE_SIZE , i +1 , -PAGE_SIZE ); - } } l1tf_vmx_mitigation = l1tf;
@@ -7992,7 +7973,7 @@ static struct kvm_x86_ops vmx_x86_ops __ro_after_init = {static void vmx_cleanup_l1d_flush (void ) {if (vmx_l1d_flush_pages) { -free_pages ((unsigned long )vmx_l1d_flush_pages,L1D_CACHE_ORDER ); +cleanup_l1d_flush_pages (vmx_l1d_flush_pages); vmx_l1d_flush_pages =NULL ; }/* Restore state so sysfs ignores VMX */
Signature · 1 line
2.17.1
PATCH 3/3 arch/x86: Optionally flush L1D on context switch
In reply to Balbir SinghImplement a mechanism to selectively flush the L1D cache. The goal is to allow tasks that are paranoid due to the recent snoop assisted data sampling vulnerabilites, to flush their L1D on being switched out. This protects their data from being snooped or leaked via side channels after the task has context switched out.
There are two scenarios we might want to protect against, a task leaving the CPU with data still in L1D (which is the main concern of this patch), the second scenario is a malicious task coming in (not so well trusted) for which we want to clean up the cache before it starts. Only the case for the former is addressed.
Add arch specific prctl()'s to opt-in to the L1D cache on context switch out, the existing mechanisms of tracking prev_mm via cpu_tlbstate is reused. cond_ibpb() is refactored and renamed into cond_mitigation().
A new thread_info flag TIF_SPEC_FLUSH_L1D is added to track tasks which opt-into L1D flushing. cpu_tlbstate.last_user_mm_ibpb is renamed to cpu_tlbstate.last_user_mm_spec, this is used to convert the TIF flags into mm state (per cpu via last_user_mm_spec) in cond_mitigation(), which then used to do decide when to call flush_l1d().
The current version benefited from discussions with Kees and Thomas. Thomas suggested and provided the code snippet for refactoring the existing cond_ibpb() code.
--- arch/x86/include/asm/thread_info.h | 6 +- arch/x86/include/asm/tlbflush.h | 2 +- arch/x86/include/uapi/asm/prctl.h | 3 + arch/x86/kernel/process_64.c | 10 +++- arch/x86/mm/tlb.c | 92 +++++++++++++++++++++++------- 5 files changed, 89 insertions(+), 24 deletions(-)
arch/x86/include/asm/thread_info.h +5 -1
@@ -84,7 +84,7 @@ struct thread_info {#define TIF_SYSCALL_AUDIT 7/* syscall auditing active */ #define TIF_SECCOMP 8/* secure computing */ #define TIF_SPEC_IB 9/* Indirect branch speculation mitigation */ -#define TIF_SPEC_FORCE_UPDATE 10/* Force speculation MSR update in context switch */ +#define TIF_SPEC_FLUSH_L1D 10/* Flush L1D on mm switches (processes) */ #define TIF_USER_RETURN_NOTIFY 11/* notify kernel of userspace return */ #define TIF_UPROBE 12/* breakpointed or singlestepping */ #define TIF_PATCH_PENDING 13/* pending live patching update */
@@ -96,6 +96,7 @@ struct thread_info {#define TIF_MEMDIE 20/* is terminating due to OOM killer */ #define TIF_POLLING_NRFLAG 21/* idle is polling for TIF_NEED_RESCHED */ #define TIF_IO_BITMAP 22/* uses I/O bitmap */ +#define TIF_SPEC_FORCE_UPDATE 23/* Force speculation MSR update in context switch */ #define TIF_FORCED_TF 24/* true if TF in eflags artificially */ #define TIF_BLOCKSTEP 25/* set when we want DEBUGCTLMSR_BTF */ #define TIF_LAZY_MMU_UPDATES 27/* task is updating the mmu lazily */
@@ -132,6 +133,7 @@ struct thread_info {#define _TIF_ADDR32 (1 << TIF_ADDR32)#define _TIF_X32 (1 << TIF_X32)#define _TIF_FSCHECK (1 << TIF_FSCHECK) +#define _TIF_SPEC_FLUSH_L1D (1 << TIF_SPEC_FLUSH_L1D)/* Work to do before invoking the actual syscall. */ #define _TIF_WORK_SYSCALL_ENTRY \
@@ -239,6 +241,8 @@ extern void arch_task_cache_init(void);extern int arch_dup_task_struct (struct task_struct *dst,struct task_struct *src);extern void arch_release_task_struct (struct task_struct *tsk);extern void arch_setup_new_exec (void ); +extern int enable_l1d_flush_for_task (struct task_struct *tsk); +extern int disable_l1d_flush_for_task (struct task_struct *tsk);#define arch_setup_new_exec arch_setup_new_exec#endif /* !__ASSEMBLY__ */
arch/x86/include/asm/tlbflush.h +1 -1
@@ -172,7 +172,7 @@ struct tlb_state {/* Last user mm for optimizing IBPB */ union {struct mm_struct *last_user_mm; -unsigned long last_user_mm_ibpb; +unsigned long last_user_mm_spec; };u16 loaded_mm_asid;
arch/x86/include/uapi/asm/prctl.h +3 -0
@@ -14,4 +14,7 @@#define ARCH_MAP_VDSO_32 0x2002#define ARCH_MAP_VDSO_64 0x2003 +#define ARCH_SET_L1D_FLUSH 0x3001 +#define ARCH_GET_L1D_FLUSH 0x3002 +#endif /* _ASM_X86_PRCTL_H */
arch/x86/kernel/process_64.c +9 -1
@@ -699,7 +699,15 @@ long do_arch_prctl_64(struct task_struct *task, int option, unsigned long arg2)case ARCH_MAP_VDSO_64 :return prctl_map_vdso (&vdso_image_64, arg2);#endif - +case ARCH_GET_L1D_FLUSH : +return test_ti_thread_flag (&task->thread_info,TIF_SPEC_FLUSH_L1D ); +case ARCH_SET_L1D_FLUSH : { +if (arg2 >=1 ) +return enable_l1d_flush_for_task (task); +else +return disable_l1d_flush_for_task (task); +break ; + }default : ret = -EINVAL ;break ;
arch/x86/mm/tlb.c +71 -21
165 lines not on this page · raw message
Signature · 1 line
2.17.1
cover arch/x86: Optionally flush L1D on context switch
Provide a mechanisn to flush the L1D cache on context switch. The goal is to allow tasks that are paranoid due to the recent snoop assisted data sampling vulnerabilites, to flush their L1D on being switched out. This protects their data from being snooped or leaked via side channels after the task has context switched out.
The core of the patches is patch 3, the first two refactor the code so that common bits can be reused.
Changelog:
- Refactor the code and reuse cond_ibpb() - code bits provided by tglx
- Merge mm state tracking for ibpb and l1d flush
- Rename TIF_L1D_FLUSH to TIF_SPEC_FLUSH_L1D
Changelog RFC:
- Reuse existing code for allocation and flush
- Simplify the goto logic in the actual l1d_flush function
- Optimize the code path with jump labels/static functions
The RFC patch was previously posted at
https://lore.kernel.org/lkml/[email protected]/
Balbir Singh (3):
arch/x86/kvm: Refactor l1d flush lifecycle management arch/x86: Refactor tlbflush and l1d flush arch/x86: Optionally flush L1D on context switch
arch/x86/include/asm/cacheflush.h | 6 ++ arch/x86/include/asm/thread_info.h | 6 +- arch/x86/include/asm/tlbflush.h | 2 +- arch/x86/include/uapi/asm/prctl.h | 3 + arch/x86/kernel/Makefile | 1 + arch/x86/kernel/l1d_flush.c | 85 +++++++++++++++++++++++++++ arch/x86/kernel/process_64.c | 10 +++- arch/x86/kvm/vmx/vmx.c | 56 +++--------------- arch/x86/mm/tlb.c | 92 +++++++++++++++++++++++------- 9 files changed, 189 insertions(+), 72 deletions(-) create mode 100644 arch/x86/kernel/l1d_flush.c
Signature · 1 line
2.17.1
PATCH 2/3 arch/x86: Refactor tlbflush and l1d flush
Refactor the existing assembly bits into smaller helper functions and also abstract L1D_FLUSH into a helper function. Use these functions in kvm for L1D flushing.
--- arch/x86/include/asm/cacheflush.h | 3 ++ arch/x86/kernel/l1d_flush.c | 49 +++++++++++++++++++++++++++++++ arch/x86/kvm/vmx/vmx.c | 31 ++++--------------- 3 files changed, 57 insertions(+), 26 deletions(-)
arch/x86/include/asm/cacheflush.h +3 -0
@@ -10,5 +10,8 @@void clflush_cache_range (void *addr,unsigned int size);void *alloc_l1d_flush_pages (void );void cleanup_l1d_flush_pages (void *l1d_flush_pages); +void populate_tlb_with_flush_pages (void *l1d_flush_pages); +void flush_l1d_cache_sw (void *l1d_flush_pages); +int flush_l1d_cache_hw (void );#endif /* _ASM_X86_CACHEFLUSH_H */
arch/x86/kernel/l1d_flush.c +49 -0
52 lines not on this page · raw message
arch/x86/kvm/vmx/vmx.c +5 -26
@@ -5956,8 +5956,6 @@ static int vmx_handle_exit(struct kvm_vcpu *vcpu,*/ static void vmx_l1d_flush (struct kvm_vcpu *vcpu) { -int size =PAGE_SIZE <<L1D_CACHE_ORDER ; -/* * This code is only executed when the the flush mode is 'cond' or * 'always'
@@ -5986,32 +5984,13 @@ static void vmx_l1d_flush(struct kvm_vcpu *vcpu) vcpu->stat.l1d_flush++; -if (static_cpu_has (X86_FEATURE_FLUSH_L1D )) { -wrmsrl (MSR_IA32_FLUSH_CMD ,L1D_FLUSH ); +if (flush_l1d_cache_hw ())return ; - } - asmvolatile ( -/* First ensure the pages are in the TLB */ -"xorl %%eax, %%eax\n" -".Lpopulate_tlb:\n\t" -"movzbl (%[flush_pages], %%" _ASM_AX"), %%ecx\n\t" -"addl $4096, %%eax\n\t" -"cmpl %%eax, %[size]\n\t" -"jne .Lpopulate_tlb\n\t" -"xorl %%eax, %%eax\n\t" -"cpuid\n\t" -/* Now fill the cache */ -"xorl %%eax, %%eax\n" -".Lfill_cache:\n" -"movzbl (%[flush_pages], %%" _ASM_AX"), %%ecx\n\t" -"addl $64, %%eax\n\t" -"cmpl %%eax, %[size]\n\t" -"jne .Lfill_cache\n\t" -"lfence\n" - :: [flush_pages]"r" (vmx_l1d_flush_pages), - [size]"r" (size) - :"eax" ,"ebx" ,"ecx" ,"edx" ); +preempt_disable (); +populate_tlb_with_flush_pages (vmx_l1d_flush_pages); +flush_l1d_cache_sw (vmx_l1d_flush_pages); +preempt_enable (); }static void update_cr8_intercept (struct kvm_vcpu *vcpu,int tpr,int irr)
Signature · 1 line
2.17.1
Quoted 3 lines · Balbir Singh
Provide a mechanisn to flush the L1D cache on context switch. The goal is to allow tasks that are paranoid due to the recent snoop assisted data sampling vulnerabilites, to flush their L1D on being switched out.
Hi Balbir,
Just curious, is it really vulnerabilities, plural? I thought there was only one: CVE-2020-0550 (Snoop-assisted L1 Data Sampling).
(There was a similar one without the "snoop": L1D Eviction Sampling, but it's supposed to get fixed in microcode).
Signature · 1 line
Josh
Quoted 11 lines · Josh Poimboeuf
CAUTION: This email originated from outside of the organization. Do not click links or open attachments unless you can confirm the sender and know the content is safe.
Quoted 3 lines · Balbir Singh
Provide a mechanisn to flush the L1D cache on context switch. The goal is to allow tasks that are paranoid due to the recent snoop assisted data sampling vulnerabilites, to flush their L1D on being switched out.
Quoted 8 lines
Hi Balbir,
Just curious, is it really vulnerabilities, plural? I thought there was only one: CVE-2020-0550 (Snoop-assisted L1 Data Sampling).
(There was a similar one without the "snoop": L1D Eviction Sampling, but it's supposed to get fixed in microcode).
Hi, Josh
Yes, that CVE the motivation, the mitigation for CVE-2020-0550 does suggest flushing the cache on context switch. But in general, as we begin to find more ways of evicting data or snopping data, a generic mechanism is more useful and that is why I am making it an opt-in.
Balbir Singh.
Quoted 21 lines · Balbir
Quoted 11 lines · Josh Poimboeuf
CAUTION: This email originated from outside of the organization. Do not click links or open attachments unless you can confirm the sender and know the content is safe.
Quoted 3 lines · Balbir Singh
Provide a mechanisn to flush the L1D cache on context switch. The goal is to allow tasks that are paranoid due to the recent snoop assisted data sampling vulnerabilites, to flush their L1D on being switched out.
Quoted 8 lines
Hi Balbir,
Just curious, is it really vulnerabilities, plural? I thought there was only one: CVE-2020-0550 (Snoop-assisted L1 Data Sampling).
(There was a similar one without the "snoop": L1D Eviction Sampling, but it's supposed to get fixed in microcode).
Quoted 6 lines
Hi, Josh
Yes, that CVE the motivation, the mitigation for CVE-2020-0550 does suggest flushing the cache on context switch. But in general, as we begin to find more ways of evicting data or snopping data, a generic mechanism is more useful and that is why I am making it an opt-in.
Ok. I think it would be a good idea to expand on that justification more precisely in the commit message. That would help both reviewers of the code and users of the new option understand what level of paranoia they're opting in to :-)
Signature · 1 line
Josh
Quoted 10 lines · Josh Poimboeuf
Quoted 4 lines · Balbir
Yes, that CVE the motivation, the mitigation for CVE-2020-0550 does suggest flushing the cache on context switch. But in general, as we begin to find more ways of evicting data or snopping data, a generic mechanism is more useful and that is why I am making it an opt-in.
Ok. I think it would be a good idea to expand on that justification more precisely in the commit message. That would help both reviewers of the code and users of the new option understand what level of paranoia they're opting in to :-)
The commit message is mostly useful for reviewers and people who have to do code archeaology.
Documentation/admin-guide/hw-vuln/ has plenty of space to host a document with explanations. paranoia.rst comes to my mind. :)
Thanks,
tglx
Borislav Petkov
maintainerQuoted 5 lines · Balbir Singh
arch/x86/mm/tlb.c +5
-0+int enable_l1d_flush_for_task (struct task_struct *tsk) +{ +struct page *page; +int ret =0 ; +
arch/x86/mm/tlb.c void leave_mm(int cpu) · line 174
Quoted 10 lines
arch/x86/mm/tlb.c +10
-0+if (static_cpu_has (X86_FEATURE_FLUSH_L1D )) + goto done; + + page =READ_ONCE (l1d_flush_pages); +if (unlikely (!page)) { +mutex_lock (&l1d_flush_mutex); +if (!l1d_flush_pages) { + l1d_flush_pages =alloc_l1d_flush_pages (); +if (!l1d_flush_pages) +return -ENOMEM ;
Do I see it correctly that you'll return here with l1d_flush_mutex still held?
arch/x86/mm/tlb.c void leave_mm(int cpu) · line 178
Quoted 4 lines
arch/x86/mm/tlb.c +4
-0+ } +mutex_unlock (&l1d_flush_mutex); + } +/* I don't think we need to worry about KSM */
Pls use passive voice: no "we" or "I", etc.
Signature · 4 lines
Regards/Gruss,
Boris.
https://people.kernel.org/tglx/notes-about-netiquetteQuoted 17 lines · Borislav Petkov
CAUTION: This email originated from outside of the organization. Do not click links or open attachments unless you can confirm the sender and know the content is safe.
Quoted 10 lines · Balbir Singh
arch/x86/mm/tlb.c +10
-0+int enable_l1d_flush_for_task (struct task_struct *tsk) +{ +struct page *page; +int ret =0 ; + +if (static_cpu_has (X86_FEATURE_FLUSH_L1D )) + goto done; + + page =READ_ONCE (l1d_flush_pages); +if (unlikely (!page)) {
Quoted 8 lines
arch/x86/mm/tlb.c void leave_mm(int cpu) · line 174
Quoted 5 lines
arch/x86/mm/tlb.c +5
-0+mutex_lock (&l1d_flush_mutex); +if (!l1d_flush_pages) { + l1d_flush_pages =alloc_l1d_flush_pages (); +if (!l1d_flush_pages) +return -ENOMEM ;Do I see it correctly that you'll return here with
l1d_flush_mutexstill held?
Darn.. :(
I had goto done to release the mutex earlier, let me fix that :)
Thanks,
Balbir Singh
Quoted 13 lines
arch/x86/mm/tlb.c void leave_mm(int cpu) · line 178
Quoted 5 lines
arch/x86/mm/tlb.c +4
-0+ } +mutex_unlock (&l1d_flush_mutex); + } +/* I don't think we need to worry about KSM */ Pls use passive voice: no "we" or "I", etc.
--
Regards/Gruss,
Boris.
Quoted 21 lines · Thomas Gleixner
CAUTION: This email originated from outside of the organization. Do not click links or open attachments unless you can confirm the sender and know the content is safe.
Quoted 9 lines · Josh Poimboeuf
Quoted 7 lines · Balbir
Yes, that CVE the motivation, the mitigation for CVE-2020-0550 does suggest
flushing the cache on context switch. But in general, as we begin to find more
ways of evicting data or snopping data, a generic mechanism is more useful and
that is why I am making it an opt-in.Quoted 4 lines
Ok. I think it would be a good idea to expand on that justification more precisely in the commit message. That would help both reviewers of the code and users of the new option understand what level of paranoia they're opting in to :-)
Quoted 5 lines
The commit message is mostly useful for reviewers and people who have to do code archeaology.
Documentation/admin-guide/hw-vuln/has plenty of space to host a document with explanations.paranoia.rstcomes to my mind. :)
I hope people don't go looking for aliens in there :) I'll write up some documentation if that helps, starting with something simple.
Balbir
Quoted 5 lines
Thanks,
tglx