Process-local memory allocations
Subsystems (5)
cover Process-local memory allocations
In a world with processor information leak vulnerabilities, having a treasure trove of information available for leaking in the global kernel address space is starting to be a liability. The biggest offender is the linear mapping of all physical memory and there are already efforts (XPFO) to start addressing this. In this patch series, I'd like to propose breaking up the kernel address space further and introduce process-local mappings in the kernel.
The rationale is that there are allocations in the kernel containing data that should only be accessible when the kernel is executing in the context of a specific process. A prime example is KVM vCPU state. This patch series introduces process-local memory in the kernel address space by claiming a PGD entry for this specific purpose. Then it converts KVM on x86 to use these new primitives to store GPR and FPU registers of vCPUs. KVM is a good testing ground, because it makes sure userspace can only interact with a VM from a single process.
Process-local allocations in the kernel can be part of a robust L1TF mitigation strategy that works even with SMT enabled. The specific goal here is to make it harder for a random thread using cache load gadget (usually a bounds check of a system call argument plus array access suffices) to prefetch interesting data into the L1 cache and use L1TF to leak this data.
The patch set applies to kvm/next [1]. Feedback is very welcome, both about the general approach and the actual implementation. As far as testing goes, the KVM unit tests seem happy on Intel. AMD is only compile tested at the moment.
[1] git://git.kernel.org/pub/scm/virt/kvm/kvm.git
Julian Stecklina (6):
kvm, vmx: move CR2 context switch out of assembly path kvm, vmx: move register clearing out of assembly path
mm, x86: make __kernel_map_pages always available x86/speculation, mm: add process local virtual memory region x86/speculation, kvm: move guest FPU state into process local memory x86/speculation, kvm: move gprs to process local storage
arch/x86/Kconfig | 1 + arch/x86/include/asm/kvm_host.h | 13 +- arch/x86/include/asm/pgtable_64_types.h | 6 + arch/x86/include/asm/proclocal.h | 44 ++++ arch/x86/kvm/kvm_cache_regs.h | 4 +- arch/x86/kvm/svm.c | 132 +++++++----- arch/x86/kvm/vmx.c | 203 ++++++++++-------- arch/x86/kvm/x86.c | 45 ++-- arch/x86/mm/Makefile | 2 + arch/x86/mm/dump_pagetables.c | 3 + arch/x86/mm/fault.c | 14 ++ arch/x86/mm/pageattr.c | 3 +- arch/x86/mm/proclocal.c | 269 ++++++++++++++++++++++++ include/linux/mm.h | 3 +- include/linux/mm_types.h | 7 + security/Kconfig | 16 ++ 16 files changed, 596 insertions(+), 169 deletions(-) create mode 100644 arch/x86/include/asm/proclocal.h create mode 100644 arch/x86/mm/proclocal.c
Signature · 1 line
2.17.1
RFC PATCH 1/6 kvm, vmx: move CR2 context switch out of assembly path
The VM entry/exit path is a giant inline assembly statement. Simplify it by doing CR2 context switching in plain C. Move CR2 restore behind IBRS clearing, so we reduce the amount of code we execute with IBRS on.
Using {read,write}_cr2() means KVM will use pv_mmu_ops instead of open coding native_{read,write}_cr2(). The CR2 code has been done in assembly since KVM's genesis[1], which predates the addition of the paravirt ops[2], i.e. KVM isn't deliberately avoiding the paravirt ops.
[1] Commit 6aa8b732ca01 ("[PATCH] kvm: userspace interface")
[2] Commit d3561b7fa0fb ("[PATCH] paravirt: header and stubs for paravirtualisation")
--- arch/x86/kvm/vmx.c | 15 +++++---------- 1 file changed, 5 insertions(+), 10 deletions(-)
arch/x86/kvm/vmx.c +5 -10
@@ -11212,6 +11212,9 @@ static void __noclone vmx_vcpu_run(struct kvm_vcpu *vcpu) evmcs_rsp =static_branch_unlikely (&enable_evmcs) ? (unsigned long )¤t_evmcs->host_rsp :0 ; +if (read_cr2 () != vcpu->arch.cr2) +write_cr2 (vcpu->arch.cr2); +if (static_branch_unlikely (&vmx_l1d_should_flush))vmx_l1d_flush (vcpu);
@@ -11231,13 +11234,6 @@ static void __noclone vmx_vcpu_run(struct kvm_vcpu *vcpu)"2: \n\t" __ex("vmwrite %%" _ASM_SP ", %%" _ASM_DX) "\n\t""1: \n\t" -/* Reload cr2 if changed */ -"mov %c[cr2](%0), %%" _ASM_AX" \n\t" -"mov %%cr2, %%" _ASM_DX" \n\t" -"cmp %%" _ASM_AX", %%" _ASM_DX" \n\t" -"je 3f \n\t" -"mov %%" _ASM_AX", %%cr2 \n\t" -"3: \n\t" /* Check if vmlaunch of vmresume is needed */ "cmpl $0, %c[launched](%0) \n\t" /* Load guest registers. Don't clobber flags. */
@@ -11298,8 +11294,6 @@ static void __noclone vmx_vcpu_run(struct kvm_vcpu *vcpu)"xor %%r14d, %%r14d \n\t" "xor %%r15d, %%r15d \n\t"#endif -"mov %%cr2, %%" _ASM_AX" \n\t" -"mov %%" _ASM_AX", %c[cr2](%0) \n\t" "xor %%eax, %%eax \n\t" "xor %%ebx, %%ebx \n\t"
@@ -11331,7 +11325,6 @@ static void __noclone vmx_vcpu_run(struct kvm_vcpu *vcpu) [r14]"i"(offsetof(struct vcpu_vmx, vcpu.arch.regs[VCPU_REGS_R14])), [r15]"i"(offsetof(struct vcpu_vmx, vcpu.arch.regs[VCPU_REGS_R15])), #endif - [cr2]"i"(offsetof(struct vcpu_vmx, vcpu.arch.cr2)), [wordsize]"i"(sizeof(ulong)) : "cc", "memory"#ifdef CONFIG_X86_64
@@ -11365,6 +11358,8 @@ static void __noclone vmx_vcpu_run(struct kvm_vcpu *vcpu)/* Eliminate branch target predictions from guest mode */ vmexit_fill_RSB (); + vcpu->arch.cr2 =read_cr2 (); +/* All fields are clean at this point */ if (static_branch_unlikely (&enable_evmcs)) current_evmcs->hv_clean_fields |=
Signature · 1 line
2.17.1
RFC PATCH 2/6 kvm, vmx: move register clearing out of assembly path
In reply to Julian StecklinaSplit the security related register clearing out of the large inline assembly VM entry path. This results in two slightly less complicated inline assembly statements, where it is clearer what each one does.
--- arch/x86/kvm/vmx.c | 46 +++++++++++++++++++++++++++++----------------- 1 file changed, 29 insertions(+), 17 deletions(-)
arch/x86/kvm/vmx.c +29 -17
@@ -11281,24 +11281,7 @@ static void __noclone vmx_vcpu_run(struct kvm_vcpu *vcpu)"mov %%r13, %c[r13](%0) \n\t" "mov %%r14, %c[r14](%0) \n\t" "mov %%r15, %c[r15](%0) \n\t" -/* -* Clear host registers marked as clobbered to prevent -* speculative use. -*/ -"xor %%r8d, %%r8d \n\t" -"xor %%r9d, %%r9d \n\t" -"xor %%r10d, %%r10d \n\t" -"xor %%r11d, %%r11d \n\t" -"xor %%r12d, %%r12d \n\t" -"xor %%r13d, %%r13d \n\t" -"xor %%r14d, %%r14d \n\t" - "xor %%r15d, %%r15d \n\t"#endif - -"xor %%eax, %%eax \n\t" -"xor %%ebx, %%ebx \n\t" -"xor %%esi, %%esi \n\t" -"xor %%edi, %%edi \n\t" "pop %%" _ASM_BP"; pop %%" _ASM_DX" \n\t" ".pushsection .rodata \n\t" ".global vmx_return \n\t"
@@ -11335,6 +11318,35 @@ static void __noclone vmx_vcpu_run(struct kvm_vcpu *vcpu)#endif ); +/* +* Explicitly clear (in addition to marking them as clobbered) all GPRs +* that have not been loaded with host state to prevent speculatively +* using the guest's values. +*/ + asmvolatile ( +"xor %%eax, %%eax \n\t" +"xor %%ebx, %%ebx \n\t" +"xor %%esi, %%esi \n\t" +"xor %%edi, %%edi \n\t" +#ifdef CONFIG_X86_64 +"xor %%r8d, %%r8d \n\t" +"xor %%r9d, %%r9d \n\t" +"xor %%r10d, %%r10d \n\t" +"xor %%r11d, %%r11d \n\t" +"xor %%r12d, %%r12d \n\t" +"xor %%r13d, %%r13d \n\t" +"xor %%r14d, %%r14d \n\t" +"xor %%r15d, %%r15d \n\t" +#endif + :::"cc" +#ifdef CONFIG_X86_64 + ,"rax" ,"rbx" ,"rsi" ,"rdi" + ,"r8" ,"r9" ,"r10" ,"r11" ,"r12" ,"r13" ,"r14" ,"r15" +#else + ,"eax" ,"ebx" ,"esi" ,"edi" +#endif + ); +/* * We do not use IBRS in the kernel. If this vCPU has used the * SPEC_CTRL MSR it may have left it on; save the value and
Signature · 1 line
2.17.1
RFC PATCH 3/6 mm, x86: make __kernel_map_pages always available
In reply to Julian Stecklina__kernel_map_pages is currently only enabled when CONFIG_DEBUG_PAGEALLOC is defined. Enable it unconditionally instead.
--- arch/x86/mm/pageattr.c | 3 +-- include/linux/mm.h | 3 ++- 2 files changed, 3 insertions(+), 3 deletions(-)
arch/x86/mm/pageattr.c +1 -2
@@ -2025,8 +2025,6 @@ int set_pages_rw(struct page *page, int numpages)return set_memory_rw (addr, numpages); } -#ifdef CONFIG_DEBUG_PAGEALLOC -static int __set_pages_p (struct page *page,int numpages) {unsigned long tempaddr = (unsigned long )page_address (page);
@@ -2093,6 +2091,7 @@ void __kernel_map_pages(struct page *page, int numpages, int enable)arch_flush_lazy_mmu_mode (); } +#ifdef CONFIG_DEBUG_PAGEALLOC #ifdef CONFIG_HIBERNATION bool kernel_page_present (struct page *page)
include/linux/mm.h +2 -1
@@ -2589,9 +2589,10 @@ static inline void kernel_poison_pages(struct page *page, int numpages, int enable) { }#endif +extern void __kernel_map_pages (struct page *page,int numpages,int enable); +#ifdef CONFIG_DEBUG_PAGEALLOC extern bool _debug_pagealloc_enabled; -extern void __kernel_map_pages (struct page *page,int numpages,int enable);static inline bool debug_pagealloc_enabled (void ) {
Signature · 1 line
2.17.1
RFC PATCH 4/6 x86/speculation, mm: add process local virtual memory region
In reply to Julian StecklinaThe Linux kernel has a global address space that is the same for any kernel code. This address space becomes a liability in a world with processor information leak vulnerabilities, such as L1TF. With the right cache load gadget, an attacker-controlled hyperthread pair can leak arbitrary data via L1TF. The upstream Linux kernel currently suggests disabling hyperthread, but this comes with a large performance hit for a wide range of workloads.
An alternative mitigation is to not make certain data in the kernel globally visible, but only when the kernel executes in the context of the process where this data belongs to.
This patch adds the initial plumbing for allocating process-local memory. By grabbing one entry in the PML4 of each set of page tables and start treating it as process-local memory. We currently only support 2MB of process-local allocations, but this is an arbitrary limitation and can be lifted by working on the page table allocation code.
While memory is used for process-local allocations, it is unmapped from the linear mapping of physical memory.
The code has some limitations that are spelled out in arch/x86/mm/proclocal.c.
--- arch/x86/Kconfig | 1 + arch/x86/include/asm/pgtable_64_types.h | 6 + arch/x86/include/asm/proclocal.h | 44 ++++ arch/x86/mm/Makefile | 2 + arch/x86/mm/dump_pagetables.c | 3 + arch/x86/mm/fault.c | 14 ++ arch/x86/mm/proclocal.c | 269 ++++++++++++++++++++++++ include/linux/mm_types.h | 7 + security/Kconfig | 16 ++ 9 files changed, 362 insertions(+) create mode 100644 arch/x86/include/asm/proclocal.h create mode 100644 arch/x86/mm/proclocal.c
arch/x86/Kconfig +1 -0
@@ -32,6 +32,7 @@ config X86_64 select SWIOTLB select X86_DEV_DMA_OPS select ARCH_HAS_SYSCALL_WRAPPER + select ARCH_SUPPORTS_PROCLOCAL # # Arch settings
arch/x86/include/asm/pgtable_64_types.h +6 -0
@@ -138,6 +138,12 @@ extern unsigned int ptrs_per_p4d;#define VMALLOC_END (VMALLOC_START + (VMALLOC_SIZE_TB << 40) - 1) +#ifdef CONFIG_PROCLOCAL +/* TODO: Make this generic instead of hardcoded */ +#define PROCLOCAL_START _AC(0xffffeb0000000000, UL) +#define PROCLOCAL_END _AC(0xffffebffffffffff, UL) +#endif +#define MODULES_VADDR (__START_KERNEL_map + KERNEL_IMAGE_SIZE)/* The module sections ends with the start of the fixmap */ #define MODULES_END _AC(0xffffffffff000000, UL)
arch/x86/include/asm/proclocal.h new file +44 -0
44 lines not on this page · raw message
arch/x86/mm/Makefile +2 -0
@@ -53,3 +53,5 @@ obj-$(CONFIG_PAGE_TABLE_ISOLATION) += pti.o obj-$(CONFIG_AMD_MEM_ENCRYPT) += mem_encrypt.o obj-$(CONFIG_AMD_MEM_ENCRYPT) += mem_encrypt_identity.o obj-$(CONFIG_AMD_MEM_ENCRYPT) += mem_encrypt_boot.o + +obj-$(CONFIG_PROCLOCAL) += proclocal.o
arch/x86/mm/dump_pagetables.c +3 -0
@@ -59,6 +59,7 @@ enum address_markers_idx {#endif VMALLOC_START_NR ,VMEMMAP_START_NR , +PROCLOCAL_START_NR , #ifdef CONFIG_KASANKASAN_SHADOW_START_NR , KASAN_SHADOW_END_NR,
@@ -86,6 +87,7 @@ static struct addr_marker address_markers[] = { [LOW_KERNEL_NR] = { 0UL, "Low Kernel Mapping" }, [VMALLOC_START_NR] = { 0UL, "vmalloc() Area" }, [VMEMMAP_START_NR] = { 0UL, "Vmemmap" }, + [PROCLOCAL_START_NR ] = {0UL ,"Process local" }, #ifdef CONFIG_KASAN/* * These fields get initialized with the (dynamic)
@@ -606,6 +608,7 @@ static int __init pt_dump_init(void) address_markers[KASAN_SHADOW_START_NR ].start_address =KASAN_SHADOW_START ; address_markers[KASAN_SHADOW_END_NR ].start_address =KASAN_SHADOW_END ;#endif + address_markers[PROCLOCAL_START_NR ].start_address =PROCLOCAL_START ;#endif #ifdef CONFIG_X86_32 address_markers[VMALLOC_START_NR ].start_address =VMALLOC_START ;
arch/x86/mm/fault.c +14 -0
@@ -1185,6 +1185,15 @@ static int fault_in_kernel_space(unsigned long address)return address >=TASK_SIZE_MAX ; } +static int fault_in_process_local (unsigned long address) +{ +#ifdef CONFIG_PROCLOCAL +return address >=PROCLOCAL_START && address <=PROCLOCAL_END ; +#else +return false; +#endif +} +static inline bool smap_violation (int error_code,struct pt_regs *regs) {if (!IS_ENABLED (CONFIG_X86_SMAP ))
@@ -1240,6 +1249,11 @@ __do_page_fault(struct pt_regs *regs, unsigned long error_code,* protection error (error_code & 9) == 0. */ if (unlikely (fault_in_kernel_space (address))) { + +if (unlikely (fault_in_process_local (address))) { +BUG (); + } +if (!(error_code & (X86_PF_RSVD |X86_PF_USER |X86_PF_PROT ))) {if (vmalloc_fault (address) >=0 )return ;
arch/x86/mm/proclocal.c new file +269 -0
269 lines not on this page · raw message
include/linux/mm_types.h +7 -0
@@ -491,6 +491,13 @@ struct mm_struct {/* HMM needs to track a few things per mm */ struct hmm *hmm;#endif + +#ifdef CONFIG_PROCLOCAL +/* Number of pages still in use */ +size_t proclocal_in_use_pages; + +unsigned long *proclocal_bitmap; +#endif } __randomize_layout;/*
security/Kconfig +16 -0
@@ -6,6 +6,22 @@ menu "Security options" source security/keys/Kconfig +config ARCH_SUPPORTS_PROCLOCAL + bool + +config PROCLOCAL + bool "Support process-local allocations in the kernel" + depends on ARCH_SUPPORTS_PROCLOCAL + default n + help + This feature allows subsystems in the kernel to allocate memory that + is only visible in the context of a specific process. This hardens the + kernel against information leak vulnerabilities. + + There is a slight performance impact when this option is enabled. + + If you are unsure how to answer this question, answer N. + config SECURITY_DMESG_RESTRICT bool "Restrict unprivileged access to the kernel syslog" default n
Signature · 1 line
2.17.1
RFC PATCH 5/6 x86/speculation, kvm: move guest FPU state into process local memory
In reply to Julian StecklinaFPU registers contain guest data and must be protected from information leak vulnerabilities in the kernel.
FPU register state for vCPUs are allocated from the globally-visible kernel heap. Change this to use process-local memory instead and thus prevent access (or prefetching) in any other context in the kernel.
--- arch/x86/include/asm/kvm_host.h | 10 +++++++- arch/x86/kvm/x86.c | 42 ++++++++++++++++++++++----------- 2 files changed, 37 insertions(+), 15 deletions(-)
arch/x86/include/asm/kvm_host.h +9 -1
@@ -36,6 +36,7 @@#include <asm/asm.h> #include <asm/kvm_page_track.h> #include <asm/hyperv-tlfs.h> +#include <asm/proclocal.h> #define KVM_MAX_VCPUS 288#define KVM_SOFT_MAX_VCPUS 240
@@ -530,7 +531,13 @@ struct kvm_vcpu_hv {cpumask_t tlb_flush; }; +struct kvm_vcpu_arch_hidden { +struct fpu guest_fpu; +}; +struct kvm_vcpu_arch { +struct proclocal hidden; +/* * rip and regs accesses must go through * kvm_{register,rip}_{read,write} functions.
@@ -611,7 +618,6 @@ struct kvm_vcpu_arch {* host PRKU bits. */ struct fpu user_fpu; -struct fpu guest_fpu;u64 xcr0;u64 guest_supported_xcr0;
@@ -1580,4 +1586,6 @@ static inline int kvm_cpu_get_apicid(int mps_cpu)#define put_smstate (type, buf, offset, val) \ *(type *)((buf) + (offset) - 0x7e00) = val +struct kvm_vcpu_arch_hidden *kvm_arch_vcpu_hidden_get (struct kvm_vcpu *vcpu); +#endif /* _ASM_X86_KVM_HOST_H */
arch/x86/kvm/x86.c +28 -14
147 lines not on this page · raw message
Signature · 1 line
2.17.1
RFC PATCH 6/6 x86/speculation, kvm: move gprs to process local storage
In reply to Julian StecklinaGeneral-purpose registers (GPRs) contain guest data and must be protected from information leak vulnerabilities in the kernel.
Move GPRs into process local memory and change the VMX and SVM world switch and related code accordingly.
Note: Only Intel VMX support is tested.
--- arch/x86/include/asm/kvm_host.h | 11 +-- arch/x86/kvm/kvm_cache_regs.h | 4 +- arch/x86/kvm/svm.c | 132 ++++++++++++++++------------- arch/x86/kvm/vmx.c | 142 ++++++++++++++++++-------------- arch/x86/kvm/x86.c | 3 +- 5 files changed, 164 insertions(+), 128 deletions(-)
arch/x86/include/asm/kvm_host.h +6 -5
@@ -532,17 +532,18 @@ struct kvm_vcpu_hv { };struct kvm_vcpu_arch_hidden { +/* +* rip and regs accesses must go through +* kvm_{register,rip}_{read,write} functions. +*/ +unsigned long regs[NR_VCPU_REGS ]; +struct fpu guest_fpu; };struct kvm_vcpu_arch {struct proclocal hidden; -/* -* rip and regs accesses must go through -* kvm_{register,rip}_{read,write} functions. -*/ -unsigned long regs[NR_VCPU_REGS ];u32 regs_avail;u32 regs_dirty;
arch/x86/kvm/kvm_cache_regs.h +2 -2
@@ -13,14 +13,14 @@ static inline unsigned long kvm_register_read(struct kvm_vcpu *vcpu,if (!test_bit (reg, (unsigned long *)&vcpu->arch.regs_avail)) kvm_x86_ops->cache_reg (vcpu, reg); -return vcpu->arch.regs[reg]; +return kvm_arch_vcpu_hidden_get (vcpu)->regs[reg]; }static inline void kvm_register_write (struct kvm_vcpu *vcpu,enum kvm_reg reg,unsigned long val) { - vcpu->arch.regs[reg] = val; +kvm_arch_vcpu_hidden_get (vcpu)->regs[reg] = val;__set_bit (reg, (unsigned long *)&vcpu->arch.regs_dirty);__set_bit (reg, (unsigned long *)&vcpu->arch.regs_avail); }
arch/x86/kvm/svm.c +75 -57
196 lines not on this page · raw message
arch/x86/kvm/vmx.c +79 -63
237 lines not on this page · raw message
arch/x86/kvm/x86.c +2 -1
@@ -8726,7 +8726,8 @@ void kvm_vcpu_reset(struct kvm_vcpu *vcpu, bool init_event) vcpu->arch.xcr0 =XFEATURE_MASK_FP ; } -memset (vcpu->arch.regs,0 ,sizeof (vcpu->arch.regs)); +memset (kvm_arch_vcpu_hidden_get (vcpu)->regs,0 , +sizeof (kvm_arch_vcpu_hidden_get (vcpu)->regs)); vcpu->arch.regs_avail = ~0 ; vcpu->arch.regs_dirty = ~0 ;
Signature · 1 line
2.17.1