Return-path: Received: from mga04.intel.com ([192.55.52.120]) by Galois.linutronix.de with esmtps (TLS1.2:DHE_RSA_AES_256_CBC_SHA256:256) (Exim 4.80) (envelope-from ) id 1fEMoF-00061l-OR for speck@linutronix.de; Fri, 04 May 2018 00:34:44 +0200 From: Dave Hansen Subject: [MODERATED] [PATCH 5/5] SSB extra 4 Date: Thu, 3 May 2018 15:29:48 -0700 Message-Id: =?utf-8?q?=3C690644befb780c7b700274574d270620a17e6ed8=2E152538?= =?utf-8?q?3411=2Egit=2Edave=2Ehansen=40intel=2Ecom=3E?= In-Reply-To: References: In-Reply-To: References: Content-Type: text/plain; charset="utf-8" Content-Transfer-Encoding: 7bit MIME-Version: 1.0 To: speck@linutronix.de Cc: Dave Hansen List-ID: From: Dave Hansen Enable the SPEC_CTRL_RDS feature when running BPF code. Underneath x86_calculate_kernel_spec_ctrl(), we now check the per-cpu bpf_prog_ran counter. If the counter is elevated, we need to set the SPEC_CTRL_RDS bit. We also add MSR writes (via x86_sync_spec_ctrl()) to: cpu_enter_reduced_memory_speculation() and cpu_leave_reduced_memory_speculation() I'm not super happy that x86_sync_spec_ctrl() does an unconditional MSR write. But, they should be infrequent since they only happen twice per timeout period. Signed-off-by: Dave Hansen Cc: Andi Kleen Cc: Tim Chen --- arch/x86/Kconfig | 4 ++++ arch/x86/include/asm/rmspec.h | 24 ++++++++++++++++++++++++ arch/x86/include/asm/spec-ctrl.h | 3 +++ arch/x86/kernel/cpu/bugs.c | 37 ++++++++++++++++++++++++++++++++++++- include/linux/filter.h | 4 +--- include/linux/nospec.h | 2 ++ 6 files changed, 70 insertions(+), 4 deletions(-) create mode 100644 arch/x86/include/asm/rmspec.h diff --git a/arch/x86/Kconfig b/arch/x86/Kconfig index c07f492..aa023d0 100644 --- a/arch/x86/Kconfig +++ b/arch/x86/Kconfig @@ -58,6 +58,7 @@ config X86 select ARCH_HAS_KCOV if X86_64 select ARCH_HAS_MEMBARRIER_SYNC_CORE select ARCH_HAS_PMEM_API if X86_64 + select ARCH_HAS_REDUCED_MEMORY_SPECULATION select ARCH_HAS_REFCOUNT select ARCH_HAS_UACCESS_FLUSHCACHE if X86_64 select ARCH_HAS_SET_MEMORY @@ -268,6 +269,9 @@ config RWSEM_XCHGADD_ALGORITHM config GENERIC_CALIBRATE_DELAY def_bool y +config ARCH_HAS_REDUCED_MEMORY_SPECULATION + def_bool y + config ARCH_HAS_CPU_RELAX def_bool y diff --git a/arch/x86/include/asm/rmspec.h b/arch/x86/include/asm/rmspec.h new file mode 100644 index 0000000..cac0189 --- /dev/null +++ b/arch/x86/include/asm/rmspec.h @@ -0,0 +1,24 @@ +#ifndef _LINUX_RMSPEC_H +#define _LINUX_RMSPEC_H +#include +#include + +/* + * We call these when we *know* the CPU can go in/out of its + * "safer" reduced memory speculation mode. + * + * For BPF, x86_sync_spec_ctrl() reads the per-cpu BPF state + * variable and figures out the MSR value by itself. Thus, + * we do not need to pass the "direction". + */ +static inline void cpu_enter_reduced_memory_speculation(void) +{ + x86_sync_spec_ctrl(); +} + +static inline void cpu_leave_reduced_memory_speculation(void) +{ + x86_sync_spec_ctrl(); +} + +#endif /* _LINUX_RMSPEC_H */ diff --git a/arch/x86/include/asm/spec-ctrl.h b/arch/x86/include/asm/spec-ctrl.h index 607236a..881c353 100644 --- a/arch/x86/include/asm/spec-ctrl.h +++ b/arch/x86/include/asm/spec-ctrl.h @@ -15,6 +15,9 @@ extern void x86_set_guest_spec_ctrl(u64); extern void x86_restore_host_spec_ctrl(u64); +/* Write a new SPEC_CTRL MSR based on current kernel state: */ +extern void x86_sync_spec_ctrl(void); + /* AMD specific Speculative Store Bypass MSR data */ extern u64 x86_amd_ls_cfg_base; extern u64 x86_amd_ls_cfg_rds_mask; diff --git a/arch/x86/kernel/cpu/bugs.c b/arch/x86/kernel/cpu/bugs.c index 0e17379..947ae8a 100644 --- a/arch/x86/kernel/cpu/bugs.c +++ b/arch/x86/kernel/cpu/bugs.c @@ -11,6 +11,7 @@ #include #include #include +#include #include #include #include @@ -148,10 +149,23 @@ EXPORT_SYMBOL_GPL(x86_get_default_spec_ctrl); static inline u64 intel_rds_mask(void) { + u64 mask; + if (boot_cpu_data.x86_vendor != X86_VENDOR_INTEL) return 0; - return rds_tif_to_spec_ctrl(current_thread_info()->flags); + mask = rds_tif_to_spec_ctrl(current_thread_info()->flags); + + /* + * BPF programs can be exploited to attack the kernel. + * Leave the RDS bit on when we recently ran one. This + * bit gets cleared after a BFP program has not run on + * the CPU for a while. + */ + if (get_cpu_var(bpf_prog_ran)) + mask |= SPEC_CTRL_RDS; + + return mask; } /* @@ -192,6 +206,27 @@ void x86_restore_host_spec_ctrl(u64 current_spec_ctrl) } EXPORT_SYMBOL_GPL(x86_restore_host_spec_ctrl); +/* + * A condition that may affect the SPEC_CTRL MSR has changed. + * Recalculate a new value for this CPU and set it. + * + * It is not easy to optimize the wrmsrl() away unless the + * callers have a full understanding of all the conditions + * that affect the output of x86_calculate_kernel_spec_ctrl(). + * + * Try not to call this too often. + */ +void x86_sync_spec_ctrl(void) +{ + u64 new_spec_ctrl = x86_calculate_kernel_spec_ctrl(); + + if (!boot_cpu_has(X86_FEATURE_IBRS)) + return; + + wrmsrl(MSR_IA32_SPEC_CTRL, new_spec_ctrl); +} +EXPORT_SYMBOL_GPL(x86_sync_spec_ctrl); + static void x86_amd_rds_enable(void) { u64 msrval = x86_amd_ls_cfg_base | x86_amd_ls_cfg_rds_mask; diff --git a/include/linux/filter.h b/include/linux/filter.h index ad63d9a..7e51184 100644 --- a/include/linux/filter.h +++ b/include/linux/filter.h @@ -495,13 +495,11 @@ DECLARE_PER_CPU(unsigned int, bpf_prog_ran); static inline void bpf_enter_prog(const struct bpf_prog *fp) { - int *count = &get_cpu_var(bpf_prog_ran); - (*count)++; /* * Upon the first entry to BPF code, we need to reduce * memory speculation to mitigate attacks targeting it. */ - if (*count == 1) + if (this_cpu_inc_return(bpf_prog_ran) == 1) cpu_enter_reduced_memory_speculation(); } diff --git a/include/linux/nospec.h b/include/linux/nospec.h index 037ed8e..26873ea 100644 --- a/include/linux/nospec.h +++ b/include/linux/nospec.h @@ -67,6 +67,8 @@ static inline void cpu_enter_reduced_memory_speculation(void) static inline void cpu_leave_reduced_memory_speculation(void) { } +#else +#include #endif #endif /* _LINUX_NOSPEC_H */ -- 2.9.5