Optionally flush L1D on context switch
Subsystems (5)
v5 cover Optionally flush L1D on context switch
Provide a mechanism to flush the L1D cache on context switch. The goal is to allow tasks that are paranoid due to the recent snoop assisted data sampling vulnerabilites, to flush their L1D on being switched out. This protects their data from being snooped or leaked via side channels after the task has context switched out.
Changelog v5:
- Based on Tom's recommendation, restrict the patches to Intel CPUs only ([email protected])
- Update reviewed-by tags based on v4.
Changelog v4:
- Refactor the L1D flushing code even further, pages are now allocated once and never freed. Simplify the exported functions.
- Change the name prefixs to be more consistent (l1d_flush_*)
- Refactoring of the code done in the spirit of the comments, prctl still requires arch bits for get/set L1D flush and ofcourse in the arch switch_mm bits flushing the L1D cache.
Changelog v3:
- Refactor the return value of what flush_l1d_cache_hw() returns
- Refactor the code, so that the generic setup bits come first (patch 3 from previous posting is now patches 3 and 4)
- Move from arch_prctl() to the prctl() interface as recommend in the reviews.
Changelog v2:
- Fix a miss of mutex_unlock (caught by Borislav Petkov <[email protected]>)
- Add documentation about the changes (Josh Poimboeuf <[email protected]>)
Changelog:
- Refactor the code and reuse cond_ibpb() - code bits provided by tglx
- Merge mm state tracking for ibpb and l1d flush
- Rename TIF_L1D_FLUSH to TIF_SPEC_FLUSH_L1D
Changelog RFC:
- Reuse existing code for allocation and flush
- Simplify the goto logic in the actual l1d_flush function
- Optimize the code path with jump labels/static functions
The previous version of these patches are posted at:
https://lore.kernel.org/lkml/[email protected]/
Balbir Singh (6):
arch/x86/kvm: Refactor l1d flush lifecycle management arch/x86/kvm: Refactor tlbflush and l1d flush arch/x86/mm: Refactor cond_ibpb() to support other use cases arch/x86/kvm: Refactor L1D flushing
Optionally flush L1D on context switch
Documentation: Add L1D flushing Documentation
Documentation/admin-guide/hw-vuln/index.rst | 1 + .../admin-guide/hw-vuln/l1d_flush.rst | 40 ++++++ arch/x86/include/asm/cacheflush.h | 8 ++ arch/x86/include/asm/thread_info.h | 7 +- arch/x86/include/asm/tlbflush.h | 2 +- arch/x86/kernel/Makefile | 1 + arch/x86/kernel/l1d_flush.c | 120 ++++++++++++++++++ arch/x86/kvm/vmx/vmx.c | 62 +-------- arch/x86/mm/tlb.c | 83 +++++++++--- include/uapi/linux/prctl.h | 4 + kernel/sys.c | 20 +++ 11 files changed, 266 insertions(+), 82 deletions(-) create mode 100644 Documentation/admin-guide/hw-vuln/l1d_flush.rst create mode 100644 arch/x86/kernel/l1d_flush.c
Signature · 1 line
2.17.1
PATCH v5 1/6 arch/x86/kvm: Refactor l1d flush lifecycle management
Split out the allocation and free routines to be used in a follow up set of patches (to reuse for L1D flushing).
--- arch/x86/include/asm/cacheflush.h | 3 +++ arch/x86/kernel/Makefile | 1 + arch/x86/kernel/l1d_flush.c | 36 +++++++++++++++++++++++++++++++ arch/x86/kvm/vmx/vmx.c | 25 +++------------------ 4 files changed, 43 insertions(+), 22 deletions(-) create mode 100644 arch/x86/kernel/l1d_flush.c
arch/x86/include/asm/cacheflush.h +3 -0
@@ -6,6 +6,9 @@#include <asm-generic/cacheflush.h> #include <asm/special_insns.h> +#define L1D_CACHE_ORDER 4void clflush_cache_range (void *addr,unsigned int size); +void *l1d_flush_alloc_pages (void ); +void l1d_flush_cleanup_pages (void *l1d_flush_pages);#endif /* _ASM_X86_CACHEFLUSH_H */
arch/x86/kernel/Makefile +1 -0
@@ -158,3 +158,4 @@ ifeq ($(CONFIG_X86_64),y) endif obj-$(CONFIG_IMA_SECURE_AND_OR_TRUSTED_BOOT) += ima_arch.o +obj-y += l1d_flush.o
arch/x86/kernel/l1d_flush.c new file +36 -0
@@ -0,0 +1,36 @@ +#include <linux/mm.h> +#include <asm/cacheflush.h> + +void *l1d_flush_alloc_pages (void ) +{ +struct page *page; +void *l1d_flush_pages =NULL ; +int i; + +/* +* This allocation for l1d_flush_pages is not tied to a VM/task's +* lifetime and so should not be charged to a memcg. +*/ + page =alloc_pages (GFP_KERNEL ,L1D_CACHE_ORDER ); +if (!page) +return NULL ; + l1d_flush_pages =page_address (page); + +/* +* Initialize each page with a different pattern in +* order to protect against KSM in the nested +* virtualization case. +*/ +for (i =0 ; i <1u <<L1D_CACHE_ORDER ; ++i) { +memset (l1d_flush_pages + i *PAGE_SIZE , i +1 , +PAGE_SIZE ); + } +return l1d_flush_pages; +} +EXPORT_SYMBOL_GPL (l1d_flush_alloc_pages); + +void l1d_flush_cleanup_pages (void *l1d_flush_pages) +{ +free_pages ((unsigned long )l1d_flush_pages,L1D_CACHE_ORDER ); +} +EXPORT_SYMBOL_GPL (l1d_flush_cleanup_pages);
arch/x86/kvm/vmx/vmx.c +3 -22
@@ -203,14 +203,10 @@ static const struct { [VMENTER_L1D_FLUSH_NOT_REQUIRED] = {"not required", false}, }; -#define L1D_CACHE_ORDER 4static void *vmx_l1d_flush_pages;static int vmx_setup_l1d_flush (enum vmx_l1d_flush_state l1tf) { -struct page *page; -unsigned int i; -if (!boot_cpu_has_bug (X86_BUG_L1TF )) { l1tf_vmx_mitigation =VMENTER_L1D_FLUSH_NOT_REQUIRED ;return 0 ;
@@ -253,24 +249,9 @@ static int vmx_setup_l1d_flush(enum vmx_l1d_flush_state l1tf)if (l1tf !=VMENTER_L1D_FLUSH_NEVER && !vmx_l1d_flush_pages && !boot_cpu_has (X86_FEATURE_FLUSH_L1D )) { -/* -* This allocation for vmx_l1d_flush_pages is not tied to a VM -* lifetime and so should not be charged to a memcg. -*/ - page =alloc_pages (GFP_KERNEL ,L1D_CACHE_ORDER ); -if (!page) + vmx_l1d_flush_pages =l1d_flush_alloc_pages (); +if (!vmx_l1d_flush_pages)return -ENOMEM ; - vmx_l1d_flush_pages =page_address (page); - -/* -* Initialize each page with a different pattern in -* order to protect against KSM in the nested -* virtualization case. -*/ -for (i =0 ; i <1u <<L1D_CACHE_ORDER ; ++i) { -memset (vmx_l1d_flush_pages + i *PAGE_SIZE , i +1 , -PAGE_SIZE ); - } } l1tf_vmx_mitigation = l1tf;
@@ -8099,7 +8080,7 @@ static struct kvm_x86_init_ops vmx_init_ops __initdata = {static void vmx_cleanup_l1d_flush (void ) {if (vmx_l1d_flush_pages) { -free_pages ((unsigned long )vmx_l1d_flush_pages,L1D_CACHE_ORDER ); +l1d_flush_cleanup_pages (vmx_l1d_flush_pages); vmx_l1d_flush_pages =NULL ; }/* Restore state so sysfs ignores VMX */
Signature · 1 line
2.17.1
PATCH v5 2/6 arch/x86/kvm: Refactor tlbflush and l1d flush
In reply to Balbir SinghRefactor the existing assembly bits into smaller helper functions and also abstract L1D_FLUSH into a helper function. Use these functions in kvm for L1D flushing.
--- arch/x86/include/asm/cacheflush.h | 3 ++ arch/x86/kernel/l1d_flush.c | 54 +++++++++++++++++++++++++++++++ arch/x86/kvm/vmx/vmx.c | 29 ++--------------- 3 files changed, 60 insertions(+), 26 deletions(-)
arch/x86/include/asm/cacheflush.h +3 -0
@@ -8,7 +8,10 @@#define L1D_CACHE_ORDER 4void clflush_cache_range (void *addr,unsigned int size); +void l1d_flush_populate_tlb (void *l1d_flush_pages);void *l1d_flush_alloc_pages (void );void l1d_flush_cleanup_pages (void *l1d_flush_pages); +void l1d_flush_sw (void *l1d_flush_pages); +int l1d_flush_hw (void );#endif /* _ASM_X86_CACHEFLUSH_H */
arch/x86/kernel/l1d_flush.c +54 -0
57 lines not on this page · raw message
arch/x86/kvm/vmx/vmx.c +3 -26
@@ -6031,8 +6031,6 @@ static int vmx_handle_exit(struct kvm_vcpu *vcpu,*/ static void vmx_l1d_flush (struct kvm_vcpu *vcpu) { -int size =PAGE_SIZE <<L1D_CACHE_ORDER ; -/* * This code is only executed when the the flush mode is 'cond' or * 'always'
@@ -6061,32 +6059,11 @@ static void vmx_l1d_flush(struct kvm_vcpu *vcpu) vcpu->stat.l1d_flush++; -if (static_cpu_has (X86_FEATURE_FLUSH_L1D )) { -wrmsrl (MSR_IA32_FLUSH_CMD ,L1D_FLUSH ); +if (!l1d_flush_hw ())return ; - } - asmvolatile ( -/* First ensure the pages are in the TLB */ -"xorl %%eax, %%eax\n" -".Lpopulate_tlb:\n\t" -"movzbl (%[flush_pages], %%" _ASM_AX"), %%ecx\n\t" -"addl $4096, %%eax\n\t" -"cmpl %%eax, %[size]\n\t" -"jne .Lpopulate_tlb\n\t" -"xorl %%eax, %%eax\n\t" -"cpuid\n\t" -/* Now fill the cache */ -"xorl %%eax, %%eax\n" -".Lfill_cache:\n" -"movzbl (%[flush_pages], %%" _ASM_AX"), %%ecx\n\t" -"addl $64, %%eax\n\t" -"cmpl %%eax, %[size]\n\t" -"jne .Lfill_cache\n\t" -"lfence\n" - :: [flush_pages]"r" (vmx_l1d_flush_pages), - [size]"r" (size) - :"eax" ,"ebx" ,"ecx" ,"edx" ); +l1d_flush_populate_tlb (vmx_l1d_flush_pages); +l1d_flush_sw (vmx_l1d_flush_pages); }static void update_cr8_intercept (struct kvm_vcpu *vcpu,int tpr,int irr)
Signature · 1 line
2.17.1
PATCH v5 3/6 arch/x86/mm: Refactor cond_ibpb() to support other use cases
In reply to Balbir Singhcond_ibpb() has the necessary bits required to track the previous mm in switch_mm_irqs_off(). This can be reused for other use cases like L1D flushing (on context switch out).
--- arch/x86/include/asm/tlbflush.h | 2 +- arch/x86/mm/tlb.c | 43 +++++++++++++++++---------------- 2 files changed, 23 insertions(+), 22 deletions(-)
arch/x86/include/asm/tlbflush.h +1 -1
@@ -83,7 +83,7 @@ struct tlb_state {/* Last user mm for optimizing IBPB */ union {struct mm_struct *last_user_mm; -unsigned long last_user_mm_ibpb; +unsigned long last_user_mm_spec; };u16 loaded_mm_asid;
arch/x86/mm/tlb.c +22 -21
105 lines not on this page · raw message
Signature · 1 line
2.17.1
PATCH v5 4/6 arch/x86/kvm: Refactor L1D flushing
In reply to Balbir SinghMove out the initialization function to l1d_flush_init_once() so that it can be reused for subsequent patches. The side-effect of this patch is that the memory allocated for l1d flush pages is no longer freed up and the memory allocated once is shared amongst callers.
l1d_flush_sw/hw() are now abstracted under arch_l1d_flush(). vmx_l1d_flush_mutex however continues to exist as it also used from other code paths.
--- arch/x86/include/asm/cacheflush.h | 12 ++++--- arch/x86/kernel/l1d_flush.c | 56 ++++++++++++++++++++++++------- arch/x86/kvm/vmx/vmx.c | 20 ++--------- 3 files changed, 53 insertions(+), 35 deletions(-)
arch/x86/include/asm/cacheflush.h +7 -5
@@ -7,11 +7,13 @@#include <asm/special_insns.h> #define L1D_CACHE_ORDER 4 + +enum l1d_flush_options { +L1D_FLUSH_POPULATE_TLB =0x1 , +}; +void clflush_cache_range (void *addr,unsigned int size); -void l1d_flush_populate_tlb (void *l1d_flush_pages); -void *l1d_flush_alloc_pages (void ); -void l1d_flush_cleanup_pages (void *l1d_flush_pages); -void l1d_flush_sw (void *l1d_flush_pages); -int l1d_flush_hw (void ); +int l1d_flush_init_once (void ); +void arch_l1d_flush (enum l1d_flush_options options);#endif /* _ASM_X86_CACHEFLUSH_H */
arch/x86/kernel/l1d_flush.c +43 -13
92 lines not on this page · raw message
arch/x86/kvm/vmx/vmx.c +3 -17
@@ -203,8 +203,6 @@ static const struct { [VMENTER_L1D_FLUSH_NOT_REQUIRED] = {"not required", false}, }; -static void *vmx_l1d_flush_pages; -static int vmx_setup_l1d_flush (enum vmx_l1d_flush_state l1tf) {if (!boot_cpu_has_bug (X86_BUG_L1TF )) {
@@ -247,12 +245,9 @@ static int vmx_setup_l1d_flush(enum vmx_l1d_flush_state l1tf) l1tf =VMENTER_L1D_FLUSH_ALWAYS ; } -if (l1tf !=VMENTER_L1D_FLUSH_NEVER && !vmx_l1d_flush_pages && - !boot_cpu_has (X86_FEATURE_FLUSH_L1D )) { - vmx_l1d_flush_pages =l1d_flush_alloc_pages (); -if (!vmx_l1d_flush_pages) +if (l1tf !=VMENTER_L1D_FLUSH_NEVER ) +if (l1d_flush_init_once ())return -ENOMEM ; - } l1tf_vmx_mitigation = l1tf;
@@ -6058,12 +6053,7 @@ static void vmx_l1d_flush(struct kvm_vcpu *vcpu) } vcpu->stat.l1d_flush++; - -if (!l1d_flush_hw ()) -return ; - -l1d_flush_populate_tlb (vmx_l1d_flush_pages); -l1d_flush_sw (vmx_l1d_flush_pages); +arch_l1d_flush (L1D_FLUSH_POPULATE_TLB ); }static void update_cr8_intercept (struct kvm_vcpu *vcpu,int tpr,int irr)
@@ -8056,10 +8046,6 @@ static struct kvm_x86_init_ops vmx_init_ops __initdata = {static void vmx_cleanup_l1d_flush (void ) { -if (vmx_l1d_flush_pages) { -l1d_flush_cleanup_pages (vmx_l1d_flush_pages); - vmx_l1d_flush_pages =NULL ; - }/* Restore state so sysfs ignores VMX */ l1tf_vmx_mitigation =VMENTER_L1D_FLUSH_AUTO ; }
Signature · 1 line
2.17.1
kbuild test robot
build botRe: [PATCH v5 4/6] arch/x86/kvm: Refactor L1D flushing
I love your patch! Perhaps something to improve:
Hi Balbir,
I love your patch! Perhaps something to improve:
[auto build test WARNING on tip/auto-latest]
[also build test WARNING on linus/master v5.7-rc4 next-20200505] [cannot apply to kvm/linux-next tip/x86/core]
[if your patch is applied to the wrong git tree, please drop us a note to help improve the system. BTW, we also suggest to use '--base' option to specify the base tree in git format-patch, please see https://stackoverflow.com/a/37406982]
url: https://github.com/0day-ci/linux/commits/Balbir-Singh/Optionally-flush-L1D-on-context-switch/20200505-044116 base: https://git.kernel.org/pub/scm/linux/kernel/git/tip/tip.git 9a31ac1743a00b816d5393acf61ce16713d319a1
If you fix the issue, kindly add following tag as appropriate
cppcheck warnings: (new ones prefixed by >>)
>> arch/x86/kernel/l1d_flush.c:7:8: warning: Local variable 'l1d_flush_pages' shadows outer variable [shadowVariable]
void *l1d_flush_pages = NULL;
^
arch/x86/kernel/l1d_flush.c:83:14: note: Shadowed declaration
static void *l1d_flush_pages;
^
arch/x86/kernel/l1d_flush.c:7:8: note: Shadow variable
void *l1d_flush_pages = NULL;
^vim +/l1d_flush_pages +7 arch/x86/kernel/l1d_flush.c
5228007f200e15 Balbir Singh 2020-05-04 3
604f3d173bf63d Balbir Singh 2020-05-04 4 static void *l1d_flush_alloc_pages(void)
5228007f200e15 Balbir Singh 2020-05-04 5 {
5228007f200e15 Balbir Singh 2020-05-04 6 struct page *page;
5228007f200e15 Balbir Singh 2020-05-04 @7 void *l1d_flush_pages = NULL;
5228007f200e15 Balbir Singh 2020-05-04 8 int i;
5228007f200e15 Balbir Singh 2020-05-04 9
5228007f200e15 Balbir Singh 2020-05-04 10 /*
5228007f200e15 Balbir Singh 2020-05-04 11 * This allocation for l1d_flush_pages is not tied to a VM/task's
5228007f200e15 Balbir Singh 2020-05-04 12 * lifetime and so should not be charged to a memcg.
5228007f200e15 Balbir Singh 2020-05-04 13 */
5228007f200e15 Balbir Singh 2020-05-04 14 page = alloc_pages(GFP_KERNEL, L1D_CACHE_ORDER);
5228007f200e15 Balbir Singh 2020-05-04 15 if (!page)
5228007f200e15 Balbir Singh 2020-05-04 16 return NULL;
5228007f200e15 Balbir Singh 2020-05-04 17 l1d_flush_pages = page_address(page);
5228007f200e15 Balbir Singh 2020-05-04 18
5228007f200e15 Balbir Singh 2020-05-04 19 /*
5228007f200e15 Balbir Singh 2020-05-04 20 * Initialize each page with a different pattern in
5228007f200e15 Balbir Singh 2020-05-04 21 * order to protect against KSM in the nested
5228007f200e15 Balbir Singh 2020-05-04 22 * virtualization case.
5228007f200e15 Balbir Singh 2020-05-04 23 */
5228007f200e15 Balbir Singh 2020-05-04 24 for (i = 0; i < 1u << L1D_CACHE_ORDER; ++i) {
5228007f200e15 Balbir Singh 2020-05-04 25 memset(l1d_flush_pages + i * PAGE_SIZE, i + 1,
5228007f200e15 Balbir Singh 2020-05-04 26 PAGE_SIZE);
5228007f200e15 Balbir Singh 2020-05-04 27 }
5228007f200e15 Balbir Singh 2020-05-04 28 return l1d_flush_pages;
5228007f200e15 Balbir Singh 2020-05-04 29 }
5228007f200e15 Balbir Singh 2020-05-04 30 :::::: The code@line 7 was first introduced by commit
:::::: 5228007f200e157bfc7a4add32def7c7f32c0550 arch/x86/kvm: Refactor l1d flush lifecycle management
:::::: TO: Balbir Singh <[email protected]>
:::::: CC: 0day robot <[email protected]>
---
0-DAY CI Kernel Test Service, Intel Corporation
https://lists.01.org/hyperkitty/list/kbuild-all(a)lists.01.org
PATCH v5 5/6 Optionally flush L1D on context switch
In reply to Balbir SinghImplement a mechanism to selectively flush the L1D cache. The goal is to allow tasks that are paranoid due to the recent snoop assisted data sampling vulnerabilites, to flush their L1D on being switched out. This protects their data from being snooped or leaked via side channels after the task has context switched out.
There are two scenarios we might want to protect against, a task leaving the CPU with data still in L1D (which is the main concern of this patch), the second scenario is a malicious task coming in (not so well trusted) for which we want to clean up the cache before it starts. Only the case for the former is addressed.
A new thread_info flag TIF_SPEC_FLUSH_L1D is added to track tasks which opt-into L1D flushing. cpu_tlbstate.last_user_mm_spec is used to convert the TIF flags into mm state (per cpu via last_user_mm_spec) in cond_mitigation(), which then used to do decide when to call flush_l1d().
Add prctl()'s to opt-in to the L1D cache on context switch out, the existing mechanisms of tracking prev_mm via cpu_tlbstate is reused to track state of the tasks and to flush the L1D cache. The prctl interface is generic and can be ported over to other architectures.
--- arch/x86/include/asm/thread_info.h | 7 ++++- arch/x86/mm/tlb.c | 44 ++++++++++++++++++++++++++++-- include/uapi/linux/prctl.h | 4 +++ kernel/sys.c | 20 ++++++++++++++ 4 files changed, 72 insertions(+), 3 deletions(-)
arch/x86/include/asm/thread_info.h +6 -1
@@ -84,7 +84,7 @@ struct thread_info {#define TIF_SYSCALL_AUDIT 7/* syscall auditing active */ #define TIF_SECCOMP 8/* secure computing */ #define TIF_SPEC_IB 9/* Indirect branch speculation mitigation */ -#define TIF_SPEC_FORCE_UPDATE 10/* Force speculation MSR update in context switch */ +#define TIF_SPEC_FLUSH_L1D 10/* Flush L1D on mm switches (processes) */ #define TIF_USER_RETURN_NOTIFY 11/* notify kernel of userspace return */ #define TIF_UPROBE 12/* breakpointed or singlestepping */ #define TIF_PATCH_PENDING 13/* pending live patching update */
@@ -96,6 +96,7 @@ struct thread_info {#define TIF_MEMDIE 20/* is terminating due to OOM killer */ #define TIF_POLLING_NRFLAG 21/* idle is polling for TIF_NEED_RESCHED */ #define TIF_IO_BITMAP 22/* uses I/O bitmap */ +#define TIF_SPEC_FORCE_UPDATE 23/* Force speculation MSR update in context switch */ #define TIF_FORCED_TF 24/* true if TF in eflags artificially */ #define TIF_BLOCKSTEP 25/* set when we want DEBUGCTLMSR_BTF */ #define TIF_LAZY_MMU_UPDATES 27/* task is updating the mmu lazily */
@@ -132,6 +133,7 @@ struct thread_info {#define _TIF_ADDR32 (1 << TIF_ADDR32)#define _TIF_X32 (1 << TIF_X32)#define _TIF_FSCHECK (1 << TIF_FSCHECK) +#define _TIF_SPEC_FLUSH_L1D (1 << TIF_SPEC_FLUSH_L1D)/* Work to do before invoking the actual syscall. */ #define _TIF_WORK_SYSCALL_ENTRY \
@@ -235,6 +237,9 @@ static inline int arch_within_stack_frames(const void * const stack, current_thread_info()->status & TS_COMPAT)#endif +extern int arch_prctl_l1d_flush_set (struct task_struct *tsk,unsigned long enable); +extern int arch_prctl_l1d_flush_get (struct task_struct *tsk); +extern void arch_task_cache_init (void );extern int arch_dup_task_struct (struct task_struct *dst,struct task_struct *src);extern void arch_release_task_struct (struct task_struct *tsk);
arch/x86/mm/tlb.c +42 -2
77 lines not on this page · raw message
include/uapi/linux/prctl.h +4 -0
@@ -238,4 +238,8 @@ struct prctl_mm_map {#define PR_SET_IO_FLUSHER 57#define PR_GET_IO_FLUSHER 58 +/* Flush L1D on context switch (mm) */ +#define PR_SET_L1D_FLUSH 59 +#define PR_GET_L1D_FLUSH 60 +#endif /* _LINUX_PRCTL_H */
kernel/sys.c +20 -0
@@ -2262,6 +2262,16 @@ int __weak arch_prctl_spec_ctrl_set(struct task_struct *t, unsigned long which,return -EINVAL ; } +int __weak arch_prctl_l1d_flush_set (struct task_struct *tsk,unsigned long enable) +{ +return -EINVAL ; +} + +int __weak arch_prctl_l1d_flush_get (struct task_struct *t) +{ +return -EINVAL ; +} +#define PR_IO_FLUSHER (PF_MEMALLOC_NOIO | PF_LESS_THROTTLE) SYSCALL_DEFINE5(prctl, int, option, unsigned long, arg2, unsigned long, arg3,
@@ -2514,6 +2524,16 @@ SYSCALL_DEFINE5(prctl, int, option, unsigned long, arg2, unsigned long, arg3, error = (current->flags &PR_IO_FLUSHER ) ==PR_IO_FLUSHER ;break ; +case PR_SET_L1D_FLUSH : +if (arg3 || arg4 || arg5) +return -EINVAL ; + error =arch_prctl_l1d_flush_set (me, arg2); +break ; +case PR_GET_L1D_FLUSH : +if (arg2 || arg3 || arg4 || arg5) +return -EINVAL ; + error =arch_prctl_l1d_flush_get (me); +break ;default : error = -EINVAL ;break ;
Signature · 1 line
2.17.1
Quoted 100 lines · Balbir Singh
Implement a mechanism to selectively flush the L1D cache. The goal is to allow tasks that are paranoid due to the recent snoop assisted data sampling vulnerabilites, to flush their L1D on being switched out. This protects their data from being snooped or leaked via side channels after the task has context switched out.
There are two scenarios we might want to protect against, a task leaving the CPU with data still in L1D (which is the main concern of this patch), the second scenario is a malicious task coming in (not so well trusted) for which we want to clean up the cache before it starts. Only the case for the former is addressed.
A new
thread_infoflagTIF_SPEC_FLUSH_L1Dis added to track tasks which opt-into L1D flushing. cpu_tlbstate.last_user_mm_spec is used to convert the TIF flags into mm state (per cpu vialast_user_mm_spec) incond_mitigation(), which then used to do decide when to callflush_l1d().Add prctl()'s to opt-in to the L1D cache on context switch out, the existing mechanisms of tracking
prev_mmviacpu_tlbstateis reused to track state of the tasks and to flush the L1D cache. The prctl interface is generic and can be ported over to other architectures.Suggested-by Thomas Gleixner <[email protected]>Signed-off-by Balbir Singh <[email protected]>Reviewed-by Kees Cook <[email protected]>--- arch/x86/include/asm/thread_info.h | 7 ++++- arch/x86/mm/tlb.c | 44 ++++++++++++++++++++++++++++-- include/uapi/linux/prctl.h | 4 +++ kernel/sys.c | 20 ++++++++++++++ 4 files changed, 72 insertions(+), 3 deletions(-)arch/x86/include/asm/thread_info.h +6
-1@@ -84,7 +84,7 @@ struct thread_info {#define TIF_SYSCALL_AUDIT 7/* syscall auditing active */ #define TIF_SECCOMP 8/* secure computing */ #define TIF_SPEC_IB 9/* Indirect branch speculation mitigation */ -#define TIF_SPEC_FORCE_UPDATE 10/* Force speculation MSR update in context switch */ +#define TIF_SPEC_FLUSH_L1D 10/* Flush L1D on mm switches (processes) */ #define TIF_USER_RETURN_NOTIFY 11/* notify kernel of userspace return */ #define TIF_UPROBE 12/* breakpointed or singlestepping */ #define TIF_PATCH_PENDING 13/* pending live patching update */ @@ -96,6 +96,7 @@ struct thread_info {#define TIF_MEMDIE 20/* is terminating due to OOM killer */ #define TIF_POLLING_NRFLAG 21/* idle is polling for TIF_NEED_RESCHED */ #define TIF_IO_BITMAP 22/* uses I/O bitmap */ +#define TIF_SPEC_FORCE_UPDATE 23/* Force speculation MSR update in context switch */ #define TIF_FORCED_TF 24/* true if TF in eflags artificially */ #define TIF_BLOCKSTEP 25/* set when we want DEBUGCTLMSR_BTF */ #define TIF_LAZY_MMU_UPDATES 27/* task is updating the mmu lazily */ @@ -132,6 +133,7 @@ struct thread_info {#define _TIF_ADDR32 (1 << TIF_ADDR32)#define _TIF_X32 (1 << TIF_X32)#define _TIF_FSCHECK (1 << TIF_FSCHECK) +#define _TIF_SPEC_FLUSH_L1D (1 << TIF_SPEC_FLUSH_L1D)/* Work to do before invoking the actual syscall. */ #define _TIF_WORK_SYSCALL_ENTRY \@@ -235,6 +237,9 @@ static inline int arch_within_stack_frames(const void * const stack, current_thread_info()->status & TS_COMPAT)#endif +extern int arch_prctl_l1d_flush_set (struct task_struct *tsk,unsigned long enable); +extern int arch_prctl_l1d_flush_get (struct task_struct *tsk); +extern void arch_task_cache_init (void );extern int arch_dup_task_struct (struct task_struct *dst,struct task_struct *src);extern void arch_release_task_struct (struct task_struct *tsk);arch/x86/mm/tlb.c +4
-2@@ -13,6 +13,7 @@#include <asm/mmu_context.h> #include <asm/nospec-branch.h> #include <asm/cache.h> +#include <asm/cacheflush.h> #include <asm/apic.h> #include <asm/uv/uv.h> @@ -43,11 +44,12 @@*/ /* -* Bits to mangle the TIF_SPEC_IB state into the mm pointer which is +* Bits to mangle the TIF_SPEC_* state into the mm pointer which is * stored in cpu_tlb_state.last_user_mm_spec. */ #define LAST_USER_MM_IBPB 0x1UL -#define LAST_USER_MM_SPEC_MASK (LAST_USER_MM_IBPB) +#define LAST_USER_MM_L1D_FLUSH 0x2UL +#define LAST_USER_MM_SPEC_MASK (LAST_USER_MM_IBPB | LAST_USER_MM_L1D_FLUSH)/* * The x86 feature is called PCID (Process Context IDentifier). It is similar
arch/x86/mm/tlb.c void leave_mm(int cpu) · line 318
Quoted 9 lines
arch/x86/mm/tlb.c +6
-0@@ -308,6 +310,35 @@ void leave_mm(int cpu) }EXPORT_SYMBOL_GPL (leave_mm); +static int enable_l1d_flush_for_task (struct task_struct *tsk) +{ +int ret =l1d_flush_init_once (); + +if (ret <0 ) +return ret;
Am I reading correctly (in the v5 delta) that with l1d_flush_init_once() now testing for Intel CPUs, it means processes on non-Intel x86 CPUs can't use the prctl() feature as a defense-in-depth to potential future L1D cache side-channel flaws?
Why can't the L1D_CACHE_ORDER just get set dynamically based on CPU?
Quoted 4 lines
arch/x86/mm/tlb.c +4
-0+ +set_ti_thread_flag (&tsk->thread_info,TIF_SPEC_FLUSH_L1D ); +return ret; +}
Signature · 1 line
Kees Cook
Quoted 136 lines · Kees Cook
Quoted 134 lines · Balbir Singh
Implement a mechanism to selectively flush the L1D cache. The goal is to
allow tasks that are paranoid due to the recent snoop assisted data sampling
vulnerabilites, to flush their L1D on being switched out. This protects
their data from being snooped or leaked via side channels after the task
has context switched out.There are two scenarios we might want to protect against, a task leaving
the CPU with data still in L1D (which is the main concern of this patch),
the second scenario is a malicious task coming in (not so well trusted)
for which we want to clean up the cache before it starts. Only the case
for the former is addressed.A new
thread_infoflagTIF_SPEC_FLUSH_L1Dis added to track tasks which
opt-into L1D flushing. cpu_tlbstate.last_user_mm_spec is used to convert
the TIF flags into mm state (per cpu vialast_user_mm_spec) incond_mitigation(), which then used to do decide when to callflush_l1d().Add prctl()'s to opt-in to the L1D cache on context switch out, the existing mechanisms of tracking
prev_mmviacpu_tlbstateis reused to track state of the tasks and to flush the L1D cache. The prctl interface is generic and can be ported over to other architectures.Suggested-by Thomas Gleixner <[email protected]>Signed-off-by Balbir Singh <[email protected]>Reviewed-by Kees Cook <[email protected]>--- arch/x86/include/asm/thread_info.h | 7 ++++- arch/x86/mm/tlb.c | 44 ++++++++++++++++++++++++++++-- include/uapi/linux/prctl.h | 4 +++ kernel/sys.c | 20 ++++++++++++++ 4 files changed, 72 insertions(+), 3 deletions(-)arch/x86/include/asm/thread_info.h +0
-0@@ -84,7 +84,7 @@ struct thread_info {#define TIF_SYSCALL_AUDIT 7/* syscall auditing active */ #define TIF_SECCOMP 8/* secure computing */ #define TIF_SPEC_IB 9/* Indirect branch speculation mitigation */
-#defineTIF_SPEC_FORCE_UPDATE10 /* Force speculation
MSR update in context switch */
+#defineTIF_SPEC_FLUSH_L1D10 /* Flush L1D on mm switches
(processes) */
#defineTIF_USER_RETURN_NOTIFY11 /* notify kernel of
userspace return */
#defineTIF_UPROBE12 /* breakpointed or
singlestepping */
#defineTIF_PATCH_PENDING13 /* pending live patching
update */arch/x86/include/asm/thread_info.h +0
-0@@ -96,6 +96,7 @@ struct thread_info {#define TIF_MEMDIE 20/* is terminating due to OOM killer */
#defineTIF_POLLING_NRFLAG21 /* idle is polling forTIF_NEED_RESCHED*/#define TIF_IO_BITMAP 22 /* uses I/O bitmap */ +#define TIF_SPEC_FORCE_UPDATE 23 /* Force speculationMSR update in context switch */
#defineTIF_FORCED_TF24 /* true if TF in
eflags artificially */
#defineTIF_BLOCKSTEP25 /* set when we wantDEBUGCTLMSR_BTF*/
#defineTIF_LAZY_MMU_UPDATES27 /* task is updating the mmu
lazily */arch/x86/include/asm/thread_info.h +4
-0@@ -132,6 +133,7 @@ struct thread_info {#define _TIF_ADDR32 (1 << TIF_ADDR32)#define _TIF_X32 (1 << TIF_X32)#define _TIF_FSCHECK (1 << TIF_FSCHECK) +#define _TIF_SPEC_FLUSH_L1D (1 << TIF_SPEC_FLUSH_L1D)/* Work to do before invoking the actual syscall. */ #define _TIF_WORK_SYSCALL_ENTRY \@@ -235,6 +237,9 @@ static inline int arch_within_stack_frames(const void * const stack, current_thread_info()->status & TS_COMPAT)#endif +extern int arch_prctl_l1d_flush_set (struct task_struct *tsk,unsigned long enable); +extern int arch_prctl_l1d_flush_get (struct task_struct *tsk); +extern void arch_task_cache_init (void );extern int arch_dup_task_struct(struct
task_struct*dst, structtask_struct*src);
extern voidarch_release_task_struct(struct task_struct *tsk);arch/x86/mm/tlb.c +1
-1@@ -13,6 +13,7 @@#include <asm/mmu_context.h> #include <asm/nospec-branch.h> #include <asm/cache.h> +#include <asm/cacheflush.h> #include <asm/apic.h> #include <asm/uv/uv.h> @@ -43,11 +44,12 @@*/ /* -* Bits to mangle the TIF_SPEC_IB state into the mm pointer which is
+ * Bits to mangle the TIF_SPEC_* state into the mm pointer which is
* stored in cpu_tlb_state.last_user_mm_spec.
*/
#defineLAST_USER_MM_IBPB0x1ULarch/x86/mm/tlb.c +2
-1-#define LAST_USER_MM_SPEC_MASK (LAST_USER_MM_IBPB) +#define LAST_USER_MM_L1D_FLUSH 0x2UL +#define LAST_USER_MM_SPEC_MASK (LAST_USER_MM_IBPB |
LAST_USER_MM_L1D_FLUSH)/*
* The x86 feature is called PCID (Process Context IDentifier). It
is similararch/x86/mm/tlb.c +0
-0@@ -308,6 +310,35 @@ void leave_mm(int cpu) }EXPORT_SYMBOL_GPL (leave_mm);
Quoted 13 lines
arch/x86/mm/tlb.c void leave_mm(int cpu) · line 318
Quoted 6 lines
arch/x86/mm/tlb.c +6
-0+static int enable_l1d_flush_for_task (struct task_struct *tsk) +{ +int ret =l1d_flush_init_once (); + +if (ret <0 ) +return ret;Am I reading correctly (in the v5 delta) that with
l1d_flush_init_once()
now testing for Intel CPUs, it means processes on non-Intel x86 CPUs can't use theprctl()feature as a defense-in-depth to potential future
L1D cache side-channel flaws?
Yes, you are right, I wanted to add it here to address Tom's concern, but going forward we could do what you've suggested or even cpu_dev work for the right algorithm per cpu vendor/device.
Ideally we have the cache size information in our x86_cpuinfo structure at the time of discovery, but we overwrite it with the size of the largest cache, so we could definitely fix those bits.
I suggest we move forward this series and add support for other vendors as a follow up. What do you think?
Balbir Singh.
Quoted 10 lines
Why can't the
L1D_CACHE_ORDERjust get set dynamically based on CPU?arch/x86/mm/tlb.c void leave_mm(int cpu) · line 322
Quoted 4 lines
arch/x86/mm/tlb.c +4
-0+ +set_ti_thread_flag (&tsk->thread_info,TIF_SPEC_FLUSH_L1D ); +return ret; +}--
Kees Cook
Quoted 156 lines · Balbir
Quoted 136 lines · Kees Cook
Quoted 134 lines · Balbir Singh
Implement a mechanism to selectively flush the L1D cache. The goal is to
allow tasks that are paranoid due to the recent snoop assisted data sampling
vulnerabilites, to flush their L1D on being switched out. This protects
their data from being snooped or leaked via side channels after the task
has context switched out.There are two scenarios we might want to protect against, a task leaving
the CPU with data still in L1D (which is the main concern of this patch),
the second scenario is a malicious task coming in (not so well trusted)
for which we want to clean up the cache before it starts. Only the case
for the former is addressed.A new
thread_infoflagTIF_SPEC_FLUSH_L1Dis added to track tasks which
opt-into L1D flushing. cpu_tlbstate.last_user_mm_spec is used to convert
the TIF flags into mm state (per cpu vialast_user_mm_spec) incond_mitigation(), which then used to do decide when to callflush_l1d().Add prctl()'s to opt-in to the L1D cache on context switch out, the existing mechanisms of tracking
prev_mmviacpu_tlbstateis reused to track state of the tasks and to flush the L1D cache. The prctl interface is generic and can be ported over to other architectures.Suggested-by Thomas Gleixner <[email protected]>Signed-off-by Balbir Singh <[email protected]>Reviewed-by Kees Cook <[email protected]>--- arch/x86/include/asm/thread_info.h | 7 ++++- arch/x86/mm/tlb.c | 44 ++++++++++++++++++++++++++++-- include/uapi/linux/prctl.h | 4 +++ kernel/sys.c | 20 ++++++++++++++ 4 files changed, 72 insertions(+), 3 deletions(-)arch/x86/include/asm/thread_info.h +0
-0@@ -84,7 +84,7 @@ struct thread_info {#define TIF_SYSCALL_AUDIT 7/* syscall auditing active */ #define TIF_SECCOMP 8/* secure computing */ #define TIF_SPEC_IB 9/* Indirect branch speculation mitigation */
-#defineTIF_SPEC_FORCE_UPDATE10 /* Force speculation
MSR update in context switch */
+#defineTIF_SPEC_FLUSH_L1D10 /* Flush L1D on mm switches
(processes) */
#defineTIF_USER_RETURN_NOTIFY11 /* notify kernel of
userspace return */
#defineTIF_UPROBE12 /* breakpointed or
singlestepping */
#defineTIF_PATCH_PENDING13 /* pending live patching
update */arch/x86/include/asm/thread_info.h +0
-0@@ -96,6 +96,7 @@ struct thread_info {#define TIF_MEMDIE 20/* is terminating due to OOM killer */
#defineTIF_POLLING_NRFLAG21 /* idle is polling forTIF_NEED_RESCHED*/#define TIF_IO_BITMAP 22 /* uses I/O bitmap */ +#define TIF_SPEC_FORCE_UPDATE 23 /* Force speculationMSR update in context switch */
#defineTIF_FORCED_TF24 /* true if TF in
eflags artificially */
#defineTIF_BLOCKSTEP25 /* set when we wantDEBUGCTLMSR_BTF*/
#defineTIF_LAZY_MMU_UPDATES27 /* task is updating the mmu
lazily */arch/x86/include/asm/thread_info.h +4
-0@@ -132,6 +133,7 @@ struct thread_info {#define _TIF_ADDR32 (1 << TIF_ADDR32)#define _TIF_X32 (1 << TIF_X32)#define _TIF_FSCHECK (1 << TIF_FSCHECK) +#define _TIF_SPEC_FLUSH_L1D (1 << TIF_SPEC_FLUSH_L1D)/* Work to do before invoking the actual syscall. */ #define _TIF_WORK_SYSCALL_ENTRY \@@ -235,6 +237,9 @@ static inline int arch_within_stack_frames(const void * const stack, current_thread_info()->status & TS_COMPAT)#endif +extern int arch_prctl_l1d_flush_set (struct task_struct *tsk,unsigned long enable); +extern int arch_prctl_l1d_flush_get (struct task_struct *tsk); +extern void arch_task_cache_init (void );extern int arch_dup_task_struct(struct
task_struct*dst, structtask_struct*src);
extern voidarch_release_task_struct(struct task_struct *tsk);arch/x86/mm/tlb.c +1
-1@@ -13,6 +13,7 @@#include <asm/mmu_context.h> #include <asm/nospec-branch.h> #include <asm/cache.h> +#include <asm/cacheflush.h> #include <asm/apic.h> #include <asm/uv/uv.h> @@ -43,11 +44,12 @@*/ /* -* Bits to mangle the TIF_SPEC_IB state into the mm pointer which is
+ * Bits to mangle the TIF_SPEC_* state into the mm pointer which is
* stored in cpu_tlb_state.last_user_mm_spec.
*/
#defineLAST_USER_MM_IBPB0x1ULarch/x86/mm/tlb.c +2
-1-#define LAST_USER_MM_SPEC_MASK (LAST_USER_MM_IBPB) +#define LAST_USER_MM_L1D_FLUSH 0x2UL +#define LAST_USER_MM_SPEC_MASK (LAST_USER_MM_IBPB |
LAST_USER_MM_L1D_FLUSH)/*
* The x86 feature is called PCID (Process Context IDentifier). It
is similararch/x86/mm/tlb.c +0
-0@@ -308,6 +310,35 @@ void leave_mm(int cpu) }EXPORT_SYMBOL_GPL (leave_mm);Quoted 13 lines
arch/x86/mm/tlb.c void leave_mm(int cpu) · line 318
Quoted 6 lines
arch/x86/mm/tlb.c +6
-0+static int enable_l1d_flush_for_task (struct task_struct *tsk) +{ +int ret =l1d_flush_init_once (); + +if (ret <0 ) +return ret;Am I reading correctly (in the v5 delta) that with
l1d_flush_init_once()
now testing for Intel CPUs, it means processes on non-Intel x86 CPUs can't use theprctl()feature as a defense-in-depth to potential future
L1D cache side-channel flaws?Yes, you are right, I wanted to add it here to address Tom's concern, but going forward we could do what you've suggested or even
cpu_devwork for the right algorithm per cpu vendor/device.
Quoted 6 lines
Ideally we have the cache size information in our
x86_cpuinfostructure at the time of discovery, but we overwrite it with the size of the largest cache, so we could definitely fix those bits.I suggest we move forward this series and add support for other vendors as a follow up. What do you think?
Okay, sounds good.
-Kees
Quoted 14 lines
Balbir Singh.
Quoted 10 lines
Why can't the
L1D_CACHE_ORDERjust get set dynamically based on CPU?arch/x86/mm/tlb.c void leave_mm(int cpu) · line 322
Quoted 4 lines
arch/x86/mm/tlb.c +4
-0+ +set_ti_thread_flag (&tsk->thread_info,TIF_SPEC_FLUSH_L1D ); +return ret; +}--
Kees Cook
Signature · 1 line
Kees Cook
PATCH v5 6/6 Documentation: Add L1D flushing Documentation
In reply to Balbir SinghAdd documentation of l1d flushing, explain the need for the feature and how it can be used.
--- Documentation/admin-guide/hw-vuln/index.rst | 1 + .../admin-guide/hw-vuln/l1d_flush.rst | 40 +++++++++++++++++++ 2 files changed, 41 insertions(+) create mode 100644 Documentation/admin-guide/hw-vuln/l1d_flush.rst
Documentation/admin-guide/hw-vuln/index.rst +1 -0
@@ -14,3 +14,4 @@ are configurable at compile, boot or run time. mds tsx_async_abort multihit.rst + l1d_flush
Documentation/admin-guide/hw-vuln/l1d_flush.rst new file +40 -0
@@ -0,0 +1,40 @@ +L1D Flushing for the paranoid +============================= + +With an increasing number of vulnerabilities being reported around data +leaks from L1D, a new user space mechanism to flush the L1D cache on +context switch is added to the kernel. This should help address +CVE-2020-0550 and for paranoid applications, keep them safe from any +yet to be discovered vulnerabilities, related to leaks from the L1D +cache. + +Tasks can opt in to this mechanism by using a prctl (implemented only +for x86 at the moment). + +Related CVES +------------ +At the present moment, the following CVEs can be addressed by this +mechanism + + ============= ======================== ================== + CVE-2020-0550 Improper Data Forwarding OS related aspects + ============= ======================== ================== + +Usage Guidelines +---------------- +Applications can call ``prctl(2)`` with one of these two arguments + +1. PR_SET_L1D_FLUSH - flush the L1D cache on context switch (out) +2. PR_GET_L1D_FLUSH - get the current state of the L1D cache flush, returns 1 + if set and 0 if not set. + +**NOTE**: The feature is disabled by default, applications to need to specifically +opt into the feature to enable it. + +Mitigation +---------- +When PR_SET_L1D_FLUSH is enabled for a task, on switching tasks (when +the address space changes), a flush of the L1D cache is performed for +the task when it leaves the CPU. If the underlying CPU supports L1D +flushing in hardware, the hardware mechanism is used, otherwise a software +fallback, similar to the mechanism used by L1TF is used.
Signature · 1 line
2.17.1