Isolate pKVM & host
cover Isolate pKVM & host
This patch set is part-3 of this RFC patches. It introduces page table management for pKVM on Intel platform, and based on it, build pKVM's MMU page table and host EPT page table.
pKVM needs isolated address space, so it shall have its own MMU page table. At the same time, host VM shall not be able to access pKVM's memory, host EPT page table shall be setup for it.
pKVM requires its own memory pool and memory allocation to setup above MMU & EPT page table, it leverages the memory reservation mechanism and buddy page allocator from ARM solution.
The setup of MMU/EPT page table is triggered by new added hypercall init-finalise after host OS got deprivileged.
After MMU/EPT setup, pKVM can access all system memory, while host VM can access most of system memory except ones owned by pKVM.
For MMIO access, host EPT only prepared mapping below 4G, for high address MMIO access from host VM, needs to build during runtime host EPT violation.
Chuanxiao Dong (3):
pkvm: x86: Introduce general page table management framework
pkvm: x86: Introduce find_mem_range API
pkvm: x86: Dynamically handle host MMIO EPT violation
Jason Chen CJ (19):
pkvm: x86: Define hypervisor runtime VA/PA APIs
pkvm: x86: Add arch specific spinlock
pkvm: x86: Add memset lib
pkvm: x86: Add buddy page allocator
pkvm: x86: Generate pkvm_constants.h for pKVM initialization
pkvm: x86: Calculate total reserve page numbers
pkvm: x86: Reserve memory for pKVM
pkvm: x86: Early alloc from reserved memory
pkvm: x86: Initialize MMU/EPT configuration
pkvm: x86: Add early allocator based mm_ops
pkvm: x86: Define linker script alias for kernel-proper symbol
pkvm: x86: Introduce MMU pgtable support
pkvm: x86: Add global pkvm_hyp pointer
pkvm: x86: Add init-finalise hypercall
pkvm: x86: Create MMU pgtable in init-finalise hypercall
pkvm: x86: Add vmemmap and switch to buddy page allocator
pkvm: x86: Introduce host EPT pgtable support
pkvm: x86: Create host EPT pgtable in init-finalise hypercall
pkvm: x86: Add pgtable API pkvm_pgtable_lookup
arch/x86/include/asm/kvm_pkvm.h | 134 ++++++ arch/x86/include/asm/pkvm_image.h | 6 + arch/x86/include/asm/pkvm_image_vars.h | 18 + arch/x86/include/asm/pkvm_spinlock.h | 73 +++ arch/x86/kernel/setup.c | 3 + arch/x86/kernel/vmlinux.lds.S | 4 + arch/x86/kvm/.gitignore | 1 + arch/x86/kvm/vmx/pkvm/Makefile | 20 + arch/x86/kvm/vmx/pkvm/hyp/Makefile | 9 +- arch/x86/kvm/vmx/pkvm/hyp/early_alloc.c | 73 +++ arch/x86/kvm/vmx/pkvm/hyp/early_alloc.h | 15 + arch/x86/kvm/vmx/pkvm/hyp/ept.c | 223 +++++++++ arch/x86/kvm/vmx/pkvm/hyp/ept.h | 21 + arch/x86/kvm/vmx/pkvm/hyp/init_finalise.c | 296 ++++++++++++ arch/x86/kvm/vmx/pkvm/hyp/lib/memset_64.S | 24 + arch/x86/kvm/vmx/pkvm/hyp/memory.c | 65 +++ arch/x86/kvm/vmx/pkvm/hyp/memory.h | 23 + arch/x86/kvm/vmx/pkvm/hyp/mmu.c | 230 +++++++++ arch/x86/kvm/vmx/pkvm/hyp/mmu.h | 19 + arch/x86/kvm/vmx/pkvm/hyp/pgtable.c | 560 ++++++++++++++++++++++ arch/x86/kvm/vmx/pkvm/hyp/pgtable.h | 81 ++++ arch/x86/kvm/vmx/pkvm/hyp/pkvm.c | 8 + arch/x86/kvm/vmx/pkvm/hyp/pkvm_hyp.h | 10 + arch/x86/kvm/vmx/pkvm/hyp/vmexit.c | 36 ++ arch/x86/kvm/vmx/pkvm/hyp/vmx.h | 48 ++ arch/x86/kvm/vmx/pkvm/hyp/vmx_ops.h | 5 +- arch/x86/kvm/vmx/pkvm/include/pkvm.h | 49 ++ arch/x86/kvm/vmx/pkvm/pkvm_constants.c | 15 + arch/x86/kvm/vmx/pkvm/pkvm_host.c | 220 +++++++-- virt/kvm/pkvm/gfp.h | 1 + virt/kvm/pkvm/pkvm.c | 1 + 31 files changed, 2245 insertions(+), 46 deletions(-) create mode 100644 arch/x86/include/asm/kvm_pkvm.h create mode 100644 arch/x86/include/asm/pkvm_image_vars.h create mode 100644 arch/x86/include/asm/pkvm_spinlock.h create mode 100644 arch/x86/kvm/vmx/pkvm/hyp/early_alloc.c create mode 100644 arch/x86/kvm/vmx/pkvm/hyp/early_alloc.h create mode 100644 arch/x86/kvm/vmx/pkvm/hyp/ept.c create mode 100644 arch/x86/kvm/vmx/pkvm/hyp/ept.h create mode 100644 arch/x86/kvm/vmx/pkvm/hyp/init_finalise.c create mode 100644 arch/x86/kvm/vmx/pkvm/hyp/lib/memset_64.S create mode 100644 arch/x86/kvm/vmx/pkvm/hyp/memory.c create mode 100644 arch/x86/kvm/vmx/pkvm/hyp/memory.h create mode 100644 arch/x86/kvm/vmx/pkvm/hyp/mmu.c create mode 100644 arch/x86/kvm/vmx/pkvm/hyp/mmu.h create mode 100644 arch/x86/kvm/vmx/pkvm/hyp/pgtable.c create mode 100644 arch/x86/kvm/vmx/pkvm/hyp/pgtable.h create mode 100644 arch/x86/kvm/vmx/pkvm/hyp/pkvm.c create mode 100644 arch/x86/kvm/vmx/pkvm/hyp/pkvm_hyp.h create mode 100644 arch/x86/kvm/vmx/pkvm/hyp/vmx.h create mode 100644 arch/x86/kvm/vmx/pkvm/pkvm_constants.c
Signature · 1 line
2.25.1
RFC PATCH 1/22 pkvm: x86: Define hypervisor runtime VA/PA APIs
pKVM has its own MMU page table, to maintain the mapping between VA & PA under hypervisor runtime (VMX root mode).
Setup the translation between VA & PA for hypervisor runtime:
keep same direct mapping as Linux kernel based on __page_base_offset. also keep same pkvm symbol mapping as Linux kernel based on __symbol_base_offset.
direct mapping translation:
- pkvm_phys_to_virt: phys + __page_base_offset
- pkvm_virt_to_phys: virt - __page_base_offset symbol mapping translation:
- pkvm_virt_to_symbol_phys: virt - __symbol_base_offset
The following patches will setup pKVM's MMU page table based on above translation.
--- arch/x86/include/asm/kvm_pkvm.h | 18 ++++++++++++++++++ arch/x86/kvm/vmx/pkvm/hyp/Makefile | 2 +- arch/x86/kvm/vmx/pkvm/hyp/memory.c | 24 ++++++++++++++++++++++++ arch/x86/kvm/vmx/pkvm/hyp/memory.h | 13 +++++++++++++ arch/x86/kvm/vmx/pkvm/hyp/vmx_ops.h | 5 +++-- arch/x86/kvm/vmx/pkvm/include/pkvm.h | 5 +++++ arch/x86/kvm/vmx/pkvm/pkvm_host.c | 19 +++++++++++++++++++ 7 files changed, 83 insertions(+), 3 deletions(-)
arch/x86/include/asm/kvm_pkvm.h new file +18 -0
@@ -0,0 +1,18 @@ +// SPDX-License-Identifier: GPL-2.0 +/* +* Copyright (C) 2022 Intel Corporation +*/ + +#ifndef _ASM_X86_KVM_PKVM_H +#define _ASM_X86_KVM_PKVM_H + +#ifdef CONFIG_PKVM_INTEL + +void *pkvm_phys_to_virt (unsigned long phys); +unsigned long pkvm_virt_to_phys (void *virt); + +#define __pkvm_pa (virt) pkvm_virt_to_phys((void *)(virt)) +#define __pkvm_va (phys) pkvm_phys_to_virt((unsigned long)(phys)) +#endif + +#endif
arch/x86/kvm/vmx/pkvm/hyp/Makefile +1 -1
@@ -8,7 +8,7 @@ ccflags-y += -D__PKVM_HYP__ lib-dir := lib -pkvm-hyp-y := vmx_asm.o vmexit.o +pkvm-hyp-y := vmx_asm.o vmexit.o memory.o pkvm-hyp-$(CONFIG_RETPOLINE) += $(lib-dir)/retpoline.o pkvm-hyp-$(CONFIG_DEBUG_LIST) += $(lib-dir)/list_debug.o
arch/x86/kvm/vmx/pkvm/hyp/memory.c new file +24 -0
@@ -0,0 +1,24 @@ +// SPDX-License-Identifier: GPL-2.0 +/* +* Copyright (C) 2022 Intel Corporation +*/ + +#include <linux/types.h> + +unsigned long __page_base_offset; +unsigned long __symbol_base_offset; + +void *pkvm_phys_to_virt (unsigned long phys) +{ +return (void *)__page_base_offset + phys; +} + +unsigned long pkvm_virt_to_phys (void *virt) +{ +return (unsigned long )virt - __page_base_offset; +} + +unsigned long pkvm_virt_to_symbol_phys (void *virt) +{ +return (unsigned long )virt - __symbol_base_offset; +}
arch/x86/kvm/vmx/pkvm/hyp/memory.h new file +13 -0
@@ -0,0 +1,13 @@ +// SPDX-License-Identifier: GPL-2.0 +/* +* Copyright (C) 2022 Intel Corporation +*/ +#ifndef _PKVM_MEMORY_H_ +#define _PKVM_MEMORY_H_ + +#include <asm/kvm_pkvm.h> + +unsigned long pkvm_virt_to_symbol_phys (void *virt); +#define __pkvm_pa_symbol (x) pkvm_virt_to_symbol_phys((void *)x) + +#endif
arch/x86/kvm/vmx/pkvm/hyp/vmx_ops.h +3 -2
@@ -5,6 +5,7 @@#ifndef _PKVM_VMX_OPS_H_#define _PKVM_VMX_OPS_H_ +#include "memory.h" #include "debug.h" #ifdef asm_volatile_goto
@@ -170,14 +171,14 @@ static __always_inline void vmcs_set_bits(unsigned long field, u32 mask)static inline void vmcs_clear (struct vmcs *vmcs) { -u64 phys_addr =__pa (vmcs); +u64 phys_addr =__pkvm_pa (vmcs);vmx_asm1 (vmclear,"m" (phys_addr), vmcs, phys_addr); }static inline void vmcs_load (struct vmcs *vmcs) { -u64 phys_addr =__pa (vmcs); +u64 phys_addr =__pkvm_pa (vmcs);vmx_asm1 (vmptrld,"m" (phys_addr), vmcs, phys_addr); }
arch/x86/kvm/vmx/pkvm/include/pkvm.h +5 -0
@@ -48,6 +48,11 @@ struct pkvm_hyp {#define PKVM_PCPU_PAGES (ALIGN(sizeof(struct pkvm_pcpu), PAGE_SIZE) >> PAGE_SHIFT)#define PKVM_HOST_VCPU_PAGES (ALIGN(sizeof(struct pkvm_host_vcpu), PAGE_SIZE) >> PAGE_SHIFT) +extern char __pkvm_text_start[], __pkvm_text_end[]; + +extern unsigned long pkvm_sym (__page_base_offset ); +extern unsigned long pkvm_sym (__symbol_base_offset ); +PKVM_DECLARE (void,__pkvm_vmx_vmexit (void)); PKVM_DECLARE(int, pkvm_main(struct kvm_vcpu *vcpu));
arch/x86/kvm/vmx/pkvm/pkvm_host.c +19 -0
@@ -440,6 +440,21 @@ static __init int pkvm_host_check_and_setup_vmx_cap(struct pkvm_hyp *pkvm)return ret; } +static __init int pkvm_init_mmu (void ) +{ +/* +* __page_base_offset stores the offset for pkvm +* to translate VA to a PA. +* +* __symbol_base_offset stores the offset for pkvm +* to translate its symbole's VA to a PA. +*/ +pkvm_sym (__page_base_offset) = (unsigned long )__va (0 ); +pkvm_sym (__symbol_base_offset) = (unsigned long )__pkvm_text_start -__pa_symbol (__pkvm_text_start); + +return 0 ; +} +static __init void init_gdt (struct pkvm_pcpu *pcpu) { pcpu->gdt_page = pkvm_gdt_page;
@@ -693,6 +708,10 @@ __init int pkvm_init(void)if (ret) goto out_free_pkvm; + ret =pkvm_init_mmu (); +if (ret) + goto out_free_pkvm; +for_each_possible_cpu (cpu) { ret =pkvm_setup_pcpu (pkvm, cpu);if (ret)
Signature · 1 line
2.25.1
RFC PATCH 2/22 pkvm: x86: Add arch specific spinlock
In reply to Jason Chen CJpKVM on Intel platform need its own spinlock implementation. Add arch specific spinlock APIs for it.
--- arch/x86/include/asm/pkvm_spinlock.h | 64 ++++++++++++++++++++++++++++ 1 file changed, 64 insertions(+)
arch/x86/include/asm/pkvm_spinlock.h new file +64 -0
@@ -0,0 +1,64 @@ +// SPDX-License-Identifier: BSD-3-Clause OR GPL-2.0 +/* +* Copyright (C) 2018-2022 Intel Corporation +* +* pkvm runs in a self-contained environment +* and requires a self-contained spinlock implementation +* which doesn't rely on any other external symbols. +* +* This is arch specific implementation +*/ +#ifndef _X86_ASM_PKVM_SPINLOCK_H +#define _X86_ASM_PKVM_SPINLOCK_H + +#include <linux/types.h> + +typedef union pkvm_spinlock { +u64 __val; +struct { +u32 head; +u32 tail; + }; +}pkvm_spinlock_t ; + +#define __PKVM_SPINLOCK_UNLOCKED \ + ((pkvm_spinlock_t){ .__val = 0 }) + +#define pkvm_spin_lock_init (l) \ +do { \ + *(l) = __PKVM_SPINLOCK_UNLOCKED; \ +} while (0); + +static inline void pkvm_spin_lock (pkvm_spinlock_t *lock) +{ +/* The lock function atomically increments and exchanges the head +* counter of the queue. If the old head of the queue is equal to the +* tail, we have locked the spinlock. Otherwise we have to wait. +*/ + + asmvolatile (" movl $0x1,%%eax\n" +" lock xaddl %%eax,%[head]\n" +" cmpl %%eax,%[tail]\n" +" jz 1f\n" +"2: pause\n" +" cmpl %%eax,%[tail]\n" +" jnz 2b\n" +"1:\n" + : + : + [head]"m" (lock->head), + [tail]"m" (lock->tail) + :"cc" ,"memory" ,"eax" ); +} + +static inline void pkvm_spin_unlock (pkvm_spinlock_t *lock) +{ +/* Increment tail of queue */ + asmvolatile (" lock incl %[tail]\n" + : + : [tail]"m" (lock->tail) + :"cc" ,"memory" ); + +} + +#endif
Signature · 1 line
2.25.1
RFC PATCH 3/22 pkvm: x86: Add memset lib
In reply to Jason Chen CJpKVM need its own memset library for the page allocator.
It cannot directly use arch/x86/lib/memset_64.S as memset_64.S is based on ALTERNATIVE section which pKVM does not support yet.
--- arch/x86/kvm/vmx/pkvm/hyp/Makefile | 1 + arch/x86/kvm/vmx/pkvm/hyp/lib/memset_64.S | 24 +++++++++++++++++++++++ 2 files changed, 25 insertions(+)
arch/x86/kvm/vmx/pkvm/hyp/Makefile +1 -0
@@ -10,6 +10,7 @@ lib-dir := lib pkvm-hyp-y := vmx_asm.o vmexit.o memory.o +pkvm-hyp-y += $(lib-dir)/memset_64.o pkvm-hyp-$(CONFIG_RETPOLINE) += $(lib-dir)/retpoline.o pkvm-hyp-$(CONFIG_DEBUG_LIST) += $(lib-dir)/list_debug.o
arch/x86/kvm/vmx/pkvm/hyp/lib/memset_64.S new file +24 -0
@@ -0,0 +1,24 @@ +/* SPDX-License-Identifier: GPL-2.0 */ +/* Copyright 2002 Andi Kleen, SuSE Labs */ + +#include <linux/linkage.h> + +/* + * ISO C memset - set a memory block to a byte value. This function uses + * enhanced rep stosb to override the fast string function. + * The code is simpler and shorter than the fast string function as well. + * + * rdi destination + * rsi value (char) + * rdx count (bytes) + * + * rax original destination + */ +SYM_FUNC_START(memset) + movq %rdi,%r9 + movb %sil,%al + movq %rdx,%rcx + rep stosb + movq %r9,%rax + RET +SYM_FUNC_END(memset)
Signature · 1 line
2.25.1
RFC PATCH 4/22 pkvm: x86: Add buddy page allocator
In reply to Jason Chen CJCompile in buddy page allocator for pKVM on Intel platform, to support its own page allocation.
--- arch/x86/include/asm/kvm_pkvm.h | 5 +++++ arch/x86/include/asm/pkvm_spinlock.h | 9 +++++++++ arch/x86/kvm/vmx/pkvm/hyp/Makefile | 5 +++++ virt/kvm/pkvm/gfp.h | 1 + 4 files changed, 20 insertions(+)
arch/x86/include/asm/kvm_pkvm.h +5 -0
@@ -13,6 +13,11 @@ unsigned long pkvm_virt_to_phys(void *virt);#define __pkvm_pa (virt) pkvm_virt_to_phys((void *)(virt))#define __pkvm_va (phys) pkvm_phys_to_virt((unsigned long)(phys)) + +/*TODO: unify the API name: __pkvm vs. __hyp? */ +#define __hyp_pa __pkvm_pa +#define __hyp_va __pkvm_va +#endif #endif
arch/x86/include/asm/pkvm_spinlock.h +9 -0
@@ -61,4 +61,13 @@ static inline void pkvm_spin_unlock(pkvm_spinlock_t *lock) } +/* +* Redefine for virt/kvm/pkvm/page_alloc.c usage +* TODO: unify the API name: pkvm_ vs. hyp_ ? +*/ +#define hyp_spinlock_t pkvm_spinlock_t +#define hyp_spin_lock_init pkvm_spin_lock_init +#define hyp_spin_lock pkvm_spin_lock +#define hyp_spin_unlock pkvm_spin_unlock +#endif
arch/x86/kvm/vmx/pkvm/hyp/Makefile +5 -0
@@ -1,5 +1,8 @@ # SPDX-License-Identifier: GPL-2.0 +KVM_PKVM = virt/kvm/pkvm + +ccflags-y += -I $(srctree)/$(KVM_PKVM)/ ccflags-y += -I $(srctree)/arch/x86/kvm ccflags-y += -I $(srctree)/arch/x86/kvm/vmx/pkvm/include ccflags-y += -fno-stack-protector
@@ -7,12 +10,14 @@ ccflags-y += -D__DISABLE_EXPORTS ccflags-y += -D__PKVM_HYP__ lib-dir := lib +virt-dir := ../../../../../../$(KVM_PKVM) pkvm-hyp-y := vmx_asm.o vmexit.o memory.o pkvm-hyp-y += $(lib-dir)/memset_64.o pkvm-hyp-$(CONFIG_RETPOLINE) += $(lib-dir)/retpoline.o pkvm-hyp-$(CONFIG_DEBUG_LIST) += $(lib-dir)/list_debug.o +pkvm-hyp-y += $(virt-dir)/page_alloc.o pkvm-obj := $(patsubst %.o,%.pkvm.o,$(pkvm-hyp-y)) obj-$(CONFIG_PKVM_INTEL) += pkvm.o
virt/kvm/pkvm/gfp.h +1 -0
@@ -2,6 +2,7 @@#ifndef __PKVM_GFP_H#define __PKVM_GFP_H +#include <linux/mmzone.h> #include <linux/list.h> #include <buddy_memory.h>
Signature · 1 line
2.25.1
RFC PATCH 5/22 pkvm: x86: Generate pkvm_constants.h for pKVM initialization
In reply to Jason Chen CJThe pKVM runtime data structure is private in VMX root mode, it shall not be exposed to host Linux. Meanwhile during pKVM initialization, host need to allocate memory for pKVM based on such runtime data structure.
Generate pkvm_constants.h to provide constants such as structure size to the pKVM init without dragging in the definitions themselves.
--- arch/x86/kvm/.gitignore | 1 + arch/x86/kvm/vmx/pkvm/Makefile | 17 +++++++++++++++++ arch/x86/kvm/vmx/pkvm/pkvm_constants.c | 15 +++++++++++++++ 3 files changed, 33 insertions(+)
arch/x86/kvm/.gitignore +1 -0
@@ -1,2 +1,3 @@ /kvm-asm-offsets.s /kvm-asm-offsets.h +pkvm_constants.h
arch/x86/kvm/vmx/pkvm/Makefile +17 -0
@@ -7,3 +7,20 @@ pkvm-obj := pkvm_host.o obj-$(CONFIG_PKVM_INTEL) += $(pkvm-obj) obj-$(CONFIG_PKVM_INTEL) += hyp/ + +always-y := pkvm_constants.h pkvm-constants.s + +define rule_gen_hyp_constants + $(call filechk,offsets,__PKVM_CONSTANTS_H__) +endef + +CFLAGS_pkvm-constants.o = -I $(src)/include +CFLAGS_pkvm-constants.o += -I $(srctree)/virt/kvm/pkvm +$(obj)/pkvm-constants.s: $(src)/pkvm_constants.c FORCE + $(call if_changed_dep,cc_s_c) + +$(obj)/pkvm_constants.h: $(obj)/pkvm-constants.s FORCE + $(call if_changed_rule,gen_hyp_constants) + +obj-intel-pkvm := $(addprefix $(obj)/, $(pkvm-obj)) +$(obj-intel-pkvm): $(obj)/pkvm_constants.h
arch/x86/kvm/vmx/pkvm/pkvm_constants.c new file +15 -0
@@ -0,0 +1,15 @@ +// SPDX-License-Identifier: GPL-2.0 +/* +* Copyright (C) 2022 Intel Corporation +*/ + +#include <linux/kbuild.h> +#include <linux/bug.h> +#include <vdso/limits.h> +#include <buddy_memory.h> + +int main (void ) +{ +DEFINE (PKVM_VMEMMAP_ENTRY_SIZE ,sizeof (struct hyp_page )); +return 0 ; +}
Signature · 1 line
2.25.1
RFC PATCH 6/22 pkvm: x86: Calculate total reserve page numbers
In reply to Jason Chen CJpKVM need to reserve its own memory for its data or runtime allocation, such as pkvm data structure, page allocator data structure (vmemmap), VMX pages, and MMU/EPT page table pages.
Implement hyp_total_reserve_pages to calculate total reserve page numbers needed by pKVM.
Virtual memory map (vmemmap) for page allocator is calculated based on system memory size.
3 extra pages for VMX - vmcs area to support VMXON, vmcs page to support run host vcpu, and msr_bitmap to support msr emulation.
Consider the worst case for page table allocation pages. For host EPT, allow extra 1 GiB for MMIO mappings.
--- arch/x86/include/asm/kvm_pkvm.h | 94 ++++++++++++++++++++++++++++ arch/x86/kvm/vmx/pkvm/hyp/memory.c | 4 ++ arch/x86/kvm/vmx/pkvm/include/pkvm.h | 2 + arch/x86/kvm/vmx/pkvm/pkvm_host.c | 14 +++++ 4 files changed, 114 insertions(+)
arch/x86/include/asm/kvm_pkvm.h +94 -0
@@ -8,6 +8,15 @@#ifdef CONFIG_PKVM_INTEL +#include <linux/memblock.h> +#include <asm/pkvm_image.h> + +#define HYP_MEMBLOCK_REGIONS 128 +#define PKVM_PGTABLE_MAX_LEVELS 5U + +extern struct memblock_region pkvm_sym (hyp_memory )[]; +extern unsigned int pkvm_sym (hyp_memblock_nr ); +void *pkvm_phys_to_virt (unsigned long phys);unsigned long pkvm_virt_to_phys (void *virt);
@@ -18,6 +27,91 @@ unsigned long pkvm_virt_to_phys(void *virt);#define __hyp_pa __pkvm_pa#define __hyp_va __pkvm_va +static inline unsigned long __pkvm_pgtable_max_pages (unsigned long nr_pages) +{ +unsigned long total =0 , i; + +/* Provision the worst case */ +for (i =0 ; i <PKVM_PGTABLE_MAX_LEVELS ; i++) { + nr_pages =DIV_ROUND_UP (nr_pages,PTRS_PER_PTE ); + total += nr_pages; + } + +return total; +} + +static inline unsigned long __pkvm_pgtable_total_pages (void ) +{ +unsigned long total =0 , i; + +for (i =0 ; i <pkvm_sym (hyp_memblock_nr); i++) { +struct memblock_region *reg = &pkvm_sym (hyp_memory)[i]; + + total +=__pkvm_pgtable_max_pages (reg->size >>PAGE_SHIFT ); + } + +return total; +} + +static inline unsigned long host_ept_pgtable_pages (void ) +{ +unsigned long res; + +/* +* Include an extra 16 pages to safely upper-bound the worst case of +* concatenated pgds. +*/ + res =__pkvm_pgtable_total_pages () +16 ; + +/* Allow 1 GiB for MMIO mappings */ + res +=__pkvm_pgtable_max_pages (SZ_1G >>PAGE_SHIFT ); + +return res; +} + +static inline unsigned long pkvm_mmu_pgtable_pages (void ) +{ +unsigned long res; + + res =__pkvm_pgtable_total_pages (); + +return res; +} + +static inline unsigned long pkvm_vmemmap_memblock_size (struct memblock_region *reg, +size_t vmemmap_entry_size) +{ +unsigned long nr_pages = reg->size >>PAGE_SHIFT ; +unsigned long start, end; + +/* Translate the pfn to the vmemmap entry */ + start = (reg->base >>PAGE_SHIFT ) * vmemmap_entry_size; + end = start + nr_pages * vmemmap_entry_size; + start =ALIGN_DOWN (start,PAGE_SIZE ); + end =ALIGN (end,PAGE_SIZE ); + +return end - start; +} + +static inline unsigned long pkvm_vmemmap_pages (size_t vmemmap_entry_size) +{ +unsigned long total_size =0 , i; + +for (i =0 ; i <pkvm_sym (hyp_memblock_nr); i++) { + total_size +=pkvm_vmemmap_memblock_size (&pkvm_sym (hyp_memory)[i], + vmemmap_entry_size); + } + +return total_size >>PAGE_SHIFT ; +} + +static inline unsigned long pkvm_data_struct_pages (unsigned long global_pgs, +unsigned long percpu_pgs,int num_cpus) +{ +return (percpu_pgs * num_cpus + global_pgs); +} + +u64 hyp_total_reserve_pages (void );#endif #endif
arch/x86/kvm/vmx/pkvm/hyp/memory.c +4 -0
@@ -4,10 +4,14 @@*/ #include <linux/types.h> +#include <asm/kvm_pkvm.h> unsigned long __page_base_offset;unsigned long __symbol_base_offset; +unsigned int hyp_memblock_nr; +struct memblock_region hyp_memory[HYP_MEMBLOCK_REGIONS ]; +void *pkvm_phys_to_virt (unsigned long phys) {return (void *)__page_base_offset + phys;
arch/x86/kvm/vmx/pkvm/include/pkvm.h +2 -0
@@ -47,6 +47,8 @@ struct pkvm_hyp {#define PKVM_PAGES (ALIGN(sizeof(struct pkvm_hyp), PAGE_SIZE) >> PAGE_SHIFT)#define PKVM_PCPU_PAGES (ALIGN(sizeof(struct pkvm_pcpu), PAGE_SIZE) >> PAGE_SHIFT)#define PKVM_HOST_VCPU_PAGES (ALIGN(sizeof(struct pkvm_host_vcpu), PAGE_SIZE) >> PAGE_SHIFT) +#define PKVM_VMCS_PAGES 3/*vmxarea+vmcs+msr_bitmap*/ +#define PKVM_PERCPU_PAGES (PKVM_PCPU_PAGES + PKVM_HOST_VCPU_PAGES + PKVM_VMCS_PAGES)extern char __pkvm_text_start[], __pkvm_text_end[];
arch/x86/kvm/vmx/pkvm/pkvm_host.c +14 -0
@@ -6,8 +6,10 @@#include <linux/kernel.h> #include <linux/module.h> #include <asm/trapnr.h> +#include <asm/kvm_pkvm.h> #include <pkvm.h> +#include "pkvm_constants.h" MODULE_LICENSE ("GPL" );
@@ -29,6 +31,18 @@ static struct gdt_page pkvm_gdt_page = { }, }; +u64 hyp_total_reserve_pages (void ) +{ +u64 total; + + total =pkvm_data_struct_pages (PKVM_PAGES ,PKVM_PERCPU_PAGES ,num_possible_cpus ()); + total +=pkvm_vmemmap_pages (PKVM_VMEMMAP_ENTRY_SIZE ); + total +=pkvm_mmu_pgtable_pages (); + total +=host_ept_pgtable_pages (); + +return total; +} +static void *pkvm_early_alloc_contig (int pages) {return alloc_pages_exact (pages <<PAGE_SHIFT ,GFP_KERNEL | __GFP_ZERO);
Signature · 1 line
2.25.1
RFC PATCH 7/22 pkvm: x86: Reserve memory for pKVM
In reply to Jason Chen CJReserve required memory for pKVM on Intel platform before de-privilege.
It takes use of general API kvm_hyp_reserve() to do the reservation, which get pkvm reserve memory size (hyp_mem_size) from API hyp_total_reserve_pages() then record reserved physical memory base to hyp_mem_base. At the same time, register system memory block layout info to hyp_memory[] array.
--- arch/x86/include/asm/kvm_pkvm.h | 14 ++++++++++++++ arch/x86/kernel/setup.c | 3 +++ arch/x86/kvm/vmx/pkvm/Makefile | 3 +++ virt/kvm/pkvm/pkvm.c | 1 + 4 files changed, 21 insertions(+)
arch/x86/include/asm/kvm_pkvm.h +14 -0
@@ -27,6 +27,11 @@ unsigned long pkvm_virt_to_phys(void *virt);#define __hyp_pa __pkvm_pa#define __hyp_va __pkvm_va +extern phys_addr_t hyp_mem_base; +extern phys_addr_t hyp_mem_size; + +void __init kvm_hyp_reserve (void ); +static inline unsigned long __pkvm_pgtable_max_pages (unsigned long nr_pages) {unsigned long total =0 , i;
@@ -111,7 +116,16 @@ static inline unsigned long pkvm_data_struct_pages(unsigned long global_pgs,return (percpu_pgs * num_cpus + global_pgs); } +static inline int hyp_pre_reserve_check (void ) +{ +/* no necessary check yet*/ +return 0 ; +} +u64 hyp_total_reserve_pages (void ); + +#else +static inline void kvm_hyp_reserve (void ) {}#endif #endif
arch/x86/kernel/setup.c +3 -0
@@ -39,6 +39,7 @@#include <asm/cpu.h> #include <asm/efi.h> #include <asm/gart.h> +#include <asm/kvm_pkvm.h> #include <asm/hypervisor.h> #include <asm/io_apic.h> #include <asm/kasan.h>
@@ -1322,6 +1323,8 @@ void __init setup_arch(char **cmdline_p)#endif unwind_init (); + +kvm_hyp_reserve (); }#ifdef CONFIG_X86_32
arch/x86/kvm/vmx/pkvm/Makefile +3 -0
@@ -1,10 +1,13 @@ # SPDX-License-Identifier: GPL-2.0 +KVM_PKVM ?= ../../../../../virt/kvm/pkvm ccflags-y += -I $(srctree)/arch/x86/kvm ccflags-y += -I $(srctree)/arch/x86/kvm/vmx/pkvm/include pkvm-obj := pkvm_host.o +pkvm-obj += $(KVM_PKVM)/pkvm.o + obj-$(CONFIG_PKVM_INTEL) += $(pkvm-obj) obj-$(CONFIG_PKVM_INTEL) += hyp/
virt/kvm/pkvm/pkvm.c +1 -0
@@ -4,6 +4,7 @@* Author: Quentin Perret <[email protected]> */ +#include <linux/kvm_host.h> #include <linux/memblock.h> #include <linux/sort.h>
Signature · 1 line
2.25.1
RFC PATCH 8/22 pkvm: x86: Early alloc from reserved memory
In reply to Jason Chen CJpKVM's data strcuture shall be allocated from its own memory, to ensure it can be isolated from host Linux.
Add early page allocator which fetch pages from reserved memory of pKVM. For pkvm_early_alloc_config(), replace alloc_pages_exact() with the new added early page allocator. And as the allocation is from reserved memory, pkvm_init failure make reserved memory useless, there is no need to free such allocated pages anymore.
TODO: return reserved memory to host Linux if pkvm_init fails.
--- arch/x86/kvm/vmx/pkvm/hyp/Makefile | 2 +- arch/x86/kvm/vmx/pkvm/hyp/early_alloc.c | 47 +++++++++++++++++ arch/x86/kvm/vmx/pkvm/hyp/early_alloc.h | 12 +++++ arch/x86/kvm/vmx/pkvm/include/pkvm.h | 4 ++ arch/x86/kvm/vmx/pkvm/pkvm_host.c | 69 ++++++++----------------- 5 files changed, 86 insertions(+), 48 deletions(-)
arch/x86/kvm/vmx/pkvm/hyp/Makefile +1 -1
@@ -12,7 +12,7 @@ ccflags-y += -D__PKVM_HYP__ lib-dir := lib virt-dir := ../../../../../../$(KVM_PKVM) -pkvm-hyp-y := vmx_asm.o vmexit.o memory.o +pkvm-hyp-y := vmx_asm.o vmexit.o memory.o early_alloc.o pkvm-hyp-y += $(lib-dir)/memset_64.o pkvm-hyp-$(CONFIG_RETPOLINE) += $(lib-dir)/retpoline.o
arch/x86/kvm/vmx/pkvm/hyp/early_alloc.c new file +47 -0
47 lines not on this page · raw message
arch/x86/kvm/vmx/pkvm/hyp/early_alloc.h new file +12 -0
@@ -0,0 +1,12 @@ +// SPDX-License-Identifier: GPL-2.0 +/* +* Copyright (C) 2022 Intel Corporation +*/ +#ifndef __PKVM_EARLY_ALLOC_H +#define __PKVM_EARLY_ALLOC_H + +void *pkvm_early_alloc_contig (unsigned int nr_pages); +void *pkvm_early_alloc_page (void ); +void pkvm_early_alloc_init (void *virt,unsigned long size); + +#endif
arch/x86/kvm/vmx/pkvm/include/pkvm.h +4 -0
@@ -58,4 +58,8 @@ extern unsigned long pkvm_sym(__symbol_base_offset);PKVM_DECLARE (void,__pkvm_vmx_vmexit (void)); PKVM_DECLARE(int, pkvm_main(struct kvm_vcpu *vcpu)); +PKVM_DECLARE(void *, pkvm_early_alloc_contig(unsigned int nr_pages)); +PKVM_DECLARE(void *, pkvm_early_alloc_page(void)); +PKVM_DECLARE(void, pkvm_early_alloc_init(void *virt, unsigned long size)); +#endif
arch/x86/kvm/vmx/pkvm/pkvm_host.c +22 -47
149 lines not on this page · raw message
Signature · 1 line
2.25.1
RFC PATCH 9/22 pkvm: x86: Introduce general page table management framework
In reply to Jason Chen CJFrom: Chuanxiao Dong <[email protected]>
pKVM need to setup its own MMU page table and host VM's EPT for the isolation.
Introduce a general framework to do page table operations (e.g., walk page table in pKVM), which used to further support creating MMU & host EPT page table in VMX root mode.
pkvm_pgtable is defined to represent different page tables. Each pkvm_pgtable have two major ops: the pkvm_pgtable_ops is introduced to specify the page table operations (like set a specific page table entry or check if page table entry is present etc.); meantime the pkvm_mm_ops is introduced to define the page table related memory management (like page allocation, TLB flush, or phys to virt address translation etc.).
pkvm_pgtable_walker is defined to assist the page table walk, which provides the callbacks the caller would like to use during walking the table. Through it, the different page table actions can be archived, in this patch, map/unmap a range of memory or destroy for a specific page table is supported.
The map populates or overwrites target range page table entries (PTEs) for a specific page table, and setup corresponding PTE according to the desired mapping page size. If overwriting an existing large PTE, while the mapping range is small than the range represented by this large PTE, do the split for existing large PTE. Overwriting a present PTE leads to the TLB flushing at the end of the page walk.
The unmap clears target range PTEs for a specific page table. If it needs to clear part of a large PTE, do the split as well. Unmap leads to the TLB flushing at the end of the page walk as well.
The destroy does similar thing as unmap, the difference is it do unmap for the whole range of a specific page table.
During pgtable unmap or free, a page table page could be freed. From security consideration, such freed page shall not be used for other purpose before the TLB flushing happen. Put all the to-be-freed page table pages into a list and free them one by one after done the TLB flushing.
TODO: anything can be combined with PKVM on ARM?
--- arch/x86/kvm/vmx/pkvm/hyp/Makefile | 2 +- arch/x86/kvm/vmx/pkvm/hyp/pgtable.c | 488 +++++++++++++++++++++++++++ arch/x86/kvm/vmx/pkvm/hyp/pgtable.h | 76 +++++ arch/x86/kvm/vmx/pkvm/include/pkvm.h | 6 + 4 files changed, 571 insertions(+), 1 deletion(-)
arch/x86/kvm/vmx/pkvm/hyp/Makefile +1 -1
@@ -12,7 +12,7 @@ ccflags-y += -D__PKVM_HYP__ lib-dir := lib virt-dir := ../../../../../../$(KVM_PKVM) -pkvm-hyp-y := vmx_asm.o vmexit.o memory.o early_alloc.o +pkvm-hyp-y := vmx_asm.o vmexit.o memory.o early_alloc.o pgtable.o pkvm-hyp-y += $(lib-dir)/memset_64.o pkvm-hyp-$(CONFIG_RETPOLINE) += $(lib-dir)/retpoline.o
arch/x86/kvm/vmx/pkvm/hyp/pgtable.c new file +488 -0
488 lines not on this page · raw message
arch/x86/kvm/vmx/pkvm/hyp/pgtable.h new file +76 -0
76 lines not on this page · raw message
arch/x86/kvm/vmx/pkvm/include/pkvm.h +6 -0
@@ -11,6 +11,12 @@#define STACK_SIZE SZ_16K +struct pkvm_pgtable_cap { +int level; +int allowed_pgsz; +u64 table_prot; +}; +struct idt_page {gate_desc idt[IDT_ENTRIES ]; }__aligned(PAGE_SIZE) ;
Signature · 1 line
2.25.1
RFC PATCH 10/22 pkvm: x86: Initialize MMU/EPT configuration
In reply to Jason Chen CJInitializes MMU/EPT pgtable cap for pKVM, which is used for later MMU/EPT pgtable creation.
--- arch/x86/kvm/vmx/pkvm/include/pkvm.h | 3 +++ arch/x86/kvm/vmx/pkvm/pkvm_host.c | 21 +++++++++++++++++++-- 2 files changed, 22 insertions(+), 2 deletions(-)
arch/x86/kvm/vmx/pkvm/include/pkvm.h +3 -0
@@ -45,6 +45,9 @@ struct pkvm_hyp {struct vmx_capability vmx_cap;struct vmcs_config vmcs_config; +struct pkvm_pgtable_cap mmu_cap; +struct pkvm_pgtable_cap ept_cap; +struct pkvm_pcpu *pcpus[CONFIG_NR_CPUS ];struct pkvm_host_vm host_vm;
arch/x86/kvm/vmx/pkvm/pkvm_host.c +19 -2
@@ -8,6 +8,8 @@#include <asm/trapnr.h> #include <asm/kvm_pkvm.h> +#include <mmu.h> +#include <mmu/spte.h> #include <pkvm.h> #include "pkvm_constants.h"
@@ -433,8 +435,23 @@ static __init int pkvm_host_check_and_setup_vmx_cap(struct pkvm_hyp *pkvm)return ret; } -static __init int pkvm_init_mmu (void ) +static __init int pkvm_init_mmu (struct pkvm_hyp *pkvm) { +int pgsz_mask = (1 <<PG_LEVEL_2M ) | (1 <<PG_LEVEL_4K ); + +if (boot_cpu_has (X86_FEATURE_GBPAGES )) + pgsz_mask |=1 <<PG_LEVEL_1G ; + +/* record mmu pgtable cap for later mmu pgtable build */ + pkvm->mmu_cap.level =pgtable_l5_enabled () ?5 :4 ; + pkvm->mmu_cap.allowed_pgsz = pgsz_mask; + pkvm->mmu_cap.table_prot = (u64 )_KERNPG_TABLE_NOENC; + +/* record ept pgtable cap for later ept pgtable build */ + pkvm->ept_cap.level = pkvm->vmx_cap.ept &VMX_EPT_PAGE_WALK_4_BIT ?4 :5 ; + pkvm->ept_cap.allowed_pgsz = pgsz_mask; + pkvm->ept_cap.table_prot =VMX_EPT_RWX_MASK ; +/* * __page_base_offset stores the offset for pkvm * to translate VA to a PA.
@@ -710,7 +727,7 @@ __init int pkvm_init(void)if (ret) goto out; - ret =pkvm_init_mmu (); + ret =pkvm_init_mmu (pkvm);if (ret) goto out;
Signature · 1 line
2.25.1
RFC PATCH 11/22 pkvm: x86: Add early allocator based mm_ops
In reply to Jason Chen CJThe buddy page allocator need setting up based on virtual memory mapping (vmemmap), it is based on MMU page table. Meanwhile MMU page table setup for virtual memory mapping needs page allocator as well.
Before pKVM setup buddy page allocator, uses a early page allocator based mm_ops for MMU page table.
This patch adds such mm_ops based on early page allocator, and it will be used later for MMU page table early setup.
--- arch/x86/kvm/vmx/pkvm/hyp/early_alloc.c | 26 +++++++++++++++++++++++++ arch/x86/kvm/vmx/pkvm/hyp/early_alloc.h | 3 +++ 2 files changed, 29 insertions(+)
arch/x86/kvm/vmx/pkvm/hyp/early_alloc.c +26 -0
@@ -7,12 +7,21 @@#include <asm/pkvm_spinlock.h> #include <pkvm.h> +#include "pgtable.h" +static unsigned long base;static unsigned long end;static unsigned long cur;static pkvm_spinlock_t early_lock = __PKVM_SPINLOCK_UNLOCKED; +struct pkvm_mm_ops pkvm_early_alloc_mm_ops; + +unsigned long pkvm_early_alloc_nr_used_pages (void ) +{ +return (cur - base) >>PAGE_SHIFT ; +} +void *pkvm_early_alloc_contig (unsigned int nr_pages) {unsigned long size = (nr_pages <<PAGE_SHIFT );
@@ -40,8 +49,25 @@ void *pkvm_early_alloc_page(void)return pkvm_early_alloc_contig (1 ); } +static void pkvm_early_alloc_get_page (void *addr) { } +static void pkvm_early_alloc_put_page (void *addr) { } +static void pkvm_early_flush_tlb (void ) { } + +static int pkvm_early_page_count (void *vaddr) +{ +return 512 ; +} +void pkvm_early_alloc_init (void *virt,unsigned long size) { base = cur = (unsigned long )virt; end = base + size; + + pkvm_early_alloc_mm_ops.zalloc_page = pkvm_early_alloc_page; + pkvm_early_alloc_mm_ops.get_page = pkvm_early_alloc_get_page; + pkvm_early_alloc_mm_ops.put_page = pkvm_early_alloc_put_page; + pkvm_early_alloc_mm_ops.phys_to_virt = pkvm_phys_to_virt; + pkvm_early_alloc_mm_ops.virt_to_phys = pkvm_virt_to_phys; + pkvm_early_alloc_mm_ops.page_count = pkvm_early_page_count; + pkvm_early_alloc_mm_ops.flush_tlb = pkvm_early_flush_tlb; }
arch/x86/kvm/vmx/pkvm/hyp/early_alloc.h +3 -0
@@ -5,8 +5,11 @@#ifndef __PKVM_EARLY_ALLOC_H#define __PKVM_EARLY_ALLOC_H +unsigned long pkvm_early_alloc_nr_used_pages (void );void *pkvm_early_alloc_contig (unsigned int nr_pages);void *pkvm_early_alloc_page (void );void pkvm_early_alloc_init (void *virt,unsigned long size); +extern struct pkvm_mm_ops pkvm_early_alloc_mm_ops; +#endif
Signature · 1 line
2.25.1
RFC PATCH 12/22 pkvm: x86: Define linker script alias for kernel-proper symbol
In reply to Jason Chen CJDefine a linker script alias of a kernel-proper symbol referenced by pKVM code.
pKVM can directly use the alias defined symbol when sharing some Linux kernel header files, for example, the global physical_mask used by __PHYSICAL_MASK if CONFIG_DYNAMIC_PHYSICAL_MASK=true.
TODO: security analysis if a kernel symbol can be directly used by pKVM.
--- arch/x86/include/asm/pkvm_image.h | 6 ++++++ 1 file changed, 6 insertions(+)
arch/x86/include/asm/pkvm_image.h +6 -0
@@ -37,6 +37,12 @@ *(NAME NAME##.*) \END_PKVM_SECTION +/* +* Defines a linker script alias of a kernel-proper symbol referenced by +* PKVM code. +*/ +#define PKVM_ALIAS(sym) pkvm_sym(sym) = sym; +#endif /* LINKER_SCRIPT */ #endif /* __X86_INTEL_PKVM_IMAGE_H */
Signature · 1 line
2.25.1
RFC PATCH 13/22 pkvm: x86: Introduce MMU pgtable support
In reply to Jason Chen CJFollow the native Linux setting, pKVM work under paging mode to access its memory, MMU page table need setting up.
Add mmu.c for pKVM, which provide APIs to enable MMU pgtable mapping.
As buddy page allocator is based on MMU pgtable mapping, MMU page table initialization is divided into two stages before and after buddy page allocator setup:
- before buddy page allocator setup, MMU page table mm_ops is based on early page allocator, the initialization is done by pkvm_early_mmu_init(). After this initialization, MMU page table can be created, but its allocated pages usage is not tracked.
- after buddy page allocator setup, the vmemmap MMU mapping is done through above early MMU, MMU page table mm_ops then can be switched to use buddy page allocator, and meantime the early allocated pages' refcount need fixing to ensure the track from buddy allocator.
This patch only implements above first stage.
Some alias of kernel-proper symbols also defined for pKVM usage, like physical_mask for page level mask, sme_me_mask & __default_kernel_pte_mask for page prot setting.
--- arch/x86/include/asm/pkvm_image_vars.h | 18 ++++ arch/x86/kernel/vmlinux.lds.S | 4 + arch/x86/kvm/vmx/pkvm/hyp/Makefile | 2 +- arch/x86/kvm/vmx/pkvm/hyp/mmu.c | 122 +++++++++++++++++++++++++ arch/x86/kvm/vmx/pkvm/hyp/mmu.h | 17 ++++ 5 files changed, 162 insertions(+), 1 deletion(-)
arch/x86/include/asm/pkvm_image_vars.h new file +18 -0
@@ -0,0 +1,18 @@ +// SPDX-License-Identifier: GPL-2.0 +/* +* Copyright (C) 2022 Intel Corporation +*/ +#ifndef __ASM_x86_PKVM_IMAGE_VARS_H +#define __ASM_x86_PKVM_IMAGE_VARS_H + +#ifdef CONFIG_DYNAMIC_PHYSICAL_MASK +PKVM_ALIAS (physical_mask); +#endif + +#ifdef CONFIG_AMD_MEM_ENCRYPT +PKVM_ALIAS (sme_me_mask); +#endif + +PKVM_ALIAS (__default_kernel_pte_mask); + +#endif
arch/x86/kernel/vmlinux.lds.S +4 -0
@@ -541,4 +541,8 @@ INIT_PER_CPU(irq_stack_backing_store); "fixed_percpu_data is not at start of per-cpu area"); #endif +#ifdef CONFIG_PKVM_INTEL +#include <asm/pkvm_image_vars.h> +#endif + #endif /* CONFIG_X86_64 */
arch/x86/kvm/vmx/pkvm/hyp/Makefile +1 -1
@@ -12,7 +12,7 @@ ccflags-y += -D__PKVM_HYP__ lib-dir := lib virt-dir := ../../../../../../$(KVM_PKVM) -pkvm-hyp-y := vmx_asm.o vmexit.o memory.o early_alloc.o pgtable.o +pkvm-hyp-y := vmx_asm.o vmexit.o memory.o early_alloc.o pgtable.o mmu.o pkvm-hyp-y += $(lib-dir)/memset_64.o pkvm-hyp-$(CONFIG_RETPOLINE) += $(lib-dir)/retpoline.o
arch/x86/kvm/vmx/pkvm/hyp/mmu.c new file +122 -0
122 lines not on this page · raw message
arch/x86/kvm/vmx/pkvm/hyp/mmu.h new file +17 -0
@@ -0,0 +1,17 @@ +// SPDX-License-Identifier: GPL-2.0 +/* +* Copyright (C) 2022 Intel Corporation +*/ +#ifndef _PKVM_MMU_H_ +#define _PKVM_MMU_H_ + +int pkvm_mmu_map (unsigned long vaddr_start,unsigned long phys_start, +unsigned long size,int pgsz_mask,u64 prot); + +int pkvm_mmu_unmap (unsigned long vaddr_start,unsigned long phys_start, +unsigned long size); + +int pkvm_early_mmu_init (struct pkvm_pgtable_cap *cap, +void *mmu_pool_base,unsigned long mmu_pool_pages); + +#endif
Signature · 1 line
2.25.1
RFC PATCH 14/22 pkvm: x86: Add global pkvm_hyp pointer
In reply to Jason Chen CJThe unique global data structure pkvm_hyp is used to describe pKVM hypervisor, includes the whole info about its host vm & pcpu settings. It's useful for pKVM runtime to fetch required information, add a global pointer pkvm_hyp for pKVM, set it in pkvm_init when pKVM got successfully initialized.
--- arch/x86/kvm/vmx/pkvm/hyp/Makefile | 2 +- arch/x86/kvm/vmx/pkvm/hyp/pkvm.c | 8 ++++++++ arch/x86/kvm/vmx/pkvm/hyp/pkvm_hyp.h | 10 ++++++++++ arch/x86/kvm/vmx/pkvm/include/pkvm.h | 1 + arch/x86/kvm/vmx/pkvm/pkvm_host.c | 9 ++++++++- 5 files changed, 28 insertions(+), 2 deletions(-)
arch/x86/kvm/vmx/pkvm/hyp/Makefile +1 -1
@@ -12,7 +12,7 @@ ccflags-y += -D__PKVM_HYP__ lib-dir := lib virt-dir := ../../../../../../$(KVM_PKVM) -pkvm-hyp-y := vmx_asm.o vmexit.o memory.o early_alloc.o pgtable.o mmu.o +pkvm-hyp-y := vmx_asm.o vmexit.o memory.o early_alloc.o pgtable.o mmu.o pkvm.o pkvm-hyp-y += $(lib-dir)/memset_64.o pkvm-hyp-$(CONFIG_RETPOLINE) += $(lib-dir)/retpoline.o
arch/x86/kvm/vmx/pkvm/hyp/pkvm.c new file +8 -0
@@ -0,0 +1,8 @@ +// SPDX-License-Identifier: GPL-2.0 +/* +* Copyright (C) 2022 Intel Corporation +*/ + +#include <pkvm.h> + +struct pkvm_hyp *pkvm_hyp;
arch/x86/kvm/vmx/pkvm/hyp/pkvm_hyp.h new file +10 -0
@@ -0,0 +1,10 @@ +// SPDX-License-Identifier: GPL-2.0 +/* +* Copyright (C) 2022 Intel Corporation +*/ +#ifndef __PKVM_HYP_H +#define __PKVM_HYP_H + +extern struct pkvm_hyp *pkvm_hyp; + +#endif
arch/x86/kvm/vmx/pkvm/include/pkvm.h +1 -0
@@ -63,6 +63,7 @@ extern char __pkvm_text_start[], __pkvm_text_end[];extern unsigned long pkvm_sym (__page_base_offset );extern unsigned long pkvm_sym (__symbol_base_offset ); +extern struct pkvm_hyp *pkvm_sym (pkvm_hyp );PKVM_DECLARE (void,__pkvm_vmx_vmexit (void)); PKVM_DECLARE(int, pkvm_main(struct kvm_vcpu *vcpu));
arch/x86/kvm/vmx/pkvm/pkvm_host.c +8 -1
@@ -708,6 +708,11 @@ __init int pkvm_init(void) {int ret =0 , cpu; +if (pkvm_sym (pkvm_hyp)) { +pr_err ("pkvm hypervisor is running!" ); +return -EBUSY ; + } +if (!hyp_mem_base) {pr_err ("pkvm required memory not get reserved!" ); ret = -ENOMEM ;
@@ -717,7 +722,8 @@ __init int pkvm_init(void)pkvm_data_struct_pages (PKVM_PAGES ,PKVM_PERCPU_PAGES , num_possible_cpus()) << PAGE_SHIFT); - pkvm =pkvm_sym (pkvm_early_alloc_contig)(PKVM_PAGES ); +/* pkvm hypervisor keeps same VA mapping as deprivileged host */ + pkvm =pkvm_sym (pkvm_hyp) =pkvm_sym (pkvm_early_alloc_contig)(PKVM_PAGES );if (!pkvm) { ret = -ENOMEM ; goto out;
@@ -749,5 +755,6 @@ __init int pkvm_init(void)return 0 ; out: +pkvm_sym (pkvm_hyp) =NULL ;return ret; }
Signature · 1 line
2.25.1
RFC PATCH 15/22 pkvm: x86: Add init-finalise hypercall
In reply to Jason Chen CJCurrently, pKVM use host cr3 and disabling EPT to do the deprivilege. After deprivilege, the pKVM runtime shall create its own MMU/EPT pgtable to ensure the isolation. This shall be done in VMX root mode, through a hypercall from the host vcpu.
The MMU/EPT pgtable creation is done by the first vcpu triggered such hypercall, the pKVM memory section information is passed as the hypercall parameter to help the pgtable creation. Other vcpus only need to switch their CR3 and set their EPTP to the one first vcpu created through the same hypercall.
Due to the memory section parameter from host hypercall may locate in VMAP_STACK, which could be invalid after pKVM switch to its own CR3 (as no such virtual memory mapping), uses a tmp section array in the pKVM stack to save such parameter.
This patch introduces the init-finalise hypercall, the following patches will create MMU/EPT pgtable within this hypercall's handler.
--- arch/x86/include/asm/kvm_pkvm.h | 3 + arch/x86/kvm/vmx/pkvm/hyp/Makefile | 3 +- arch/x86/kvm/vmx/pkvm/hyp/init_finalise.c | 53 +++++++++++ arch/x86/kvm/vmx/pkvm/hyp/vmexit.c | 31 +++++++ arch/x86/kvm/vmx/pkvm/include/pkvm.h | 13 +++ arch/x86/kvm/vmx/pkvm/pkvm_host.c | 102 ++++++++++++++++++++++ 6 files changed, 204 insertions(+), 1 deletion(-)
arch/x86/include/asm/kvm_pkvm.h +3 -0
@@ -14,6 +14,9 @@#define HYP_MEMBLOCK_REGIONS 128#define PKVM_PGTABLE_MAX_LEVELS 5U +/* PKVM Hypercalls */ +#define PKVM_HC_INIT_FINALISE 1 +extern struct memblock_region pkvm_sym (hyp_memory )[];extern unsigned int pkvm_sym (hyp_memblock_nr );
arch/x86/kvm/vmx/pkvm/hyp/Makefile +2 -1
@@ -12,7 +12,8 @@ ccflags-y += -D__PKVM_HYP__ lib-dir := lib virt-dir := ../../../../../../$(KVM_PKVM) -pkvm-hyp-y := vmx_asm.o vmexit.o memory.o early_alloc.o pgtable.o mmu.o pkvm.o +pkvm-hyp-y := vmx_asm.o vmexit.o memory.o early_alloc.o pgtable.o mmu.o pkvm.o \ + init_finalise.o pkvm-hyp-y += $(lib-dir)/memset_64.o pkvm-hyp-$(CONFIG_RETPOLINE) += $(lib-dir)/retpoline.o
arch/x86/kvm/vmx/pkvm/hyp/init_finalise.c new file +53 -0
53 lines not on this page · raw message
arch/x86/kvm/vmx/pkvm/hyp/vmexit.c +31 -0
@@ -3,6 +3,8 @@* Copyright (C) 2022 Intel Corporation */ +#include <linux/memblock.h> +#include <asm/kvm_pkvm.h> #include <pkvm.h> #include "vmexit.h" #include "debug.h"
@@ -11,6 +13,9 @@#define MOV_TO_CR 0 +extern int __pkvm_init_finalise (struct kvm_vcpu *vcpu, +phys_addr_t phys,unsigned long size); +static void skip_emulated_instruction (void ) {unsigned long rip;
@@ -67,6 +72,28 @@ static void handle_cr(struct kvm_vcpu *vcpu) } } +static unsigned long handle_vmcall (struct kvm_vcpu *vcpu) +{ +u64 nr, a0, a1, a2, a3; +unsigned long ret =0 ; + + nr = vcpu->arch.regs[VCPU_REGS_RAX ]; + a0 = vcpu->arch.regs[VCPU_REGS_RBX ]; + a1 = vcpu->arch.regs[VCPU_REGS_RCX ]; + a2 = vcpu->arch.regs[VCPU_REGS_RDX ]; + a3 = vcpu->arch.regs[VCPU_REGS_RSI ]; + +switch (nr) { +case PKVM_HC_INIT_FINALISE : +__pkvm_init_finalise (vcpu, a0, a1); +break ; +default : + ret = -EINVAL ; + } + +return ret; +} +static void handle_read_msr (struct kvm_vcpu *vcpu) {/* simply return 0 for non-supported MSRs */
@@ -135,6 +162,10 @@ int pkvm_main(struct kvm_vcpu *vcpu)handle_xsetbv (vcpu); skip_instruction = true;break ; +case EXIT_REASON_VMCALL : + vcpu->arch.regs[VCPU_REGS_RAX ] =handle_vmcall (vcpu); + skip_instruction = true; +break ;default :pkvm_dbg ("CPU%d: Unsupported vmexit reason 0x%x.\n" , vcpu->cpu, vmx->exit_reason.full); skip_instruction = true;
arch/x86/kvm/vmx/pkvm/include/pkvm.h +13 -0
@@ -53,6 +53,16 @@ struct pkvm_hyp {struct pkvm_host_vm host_vm; }; +struct pkvm_section { +unsigned long type; +#define PKVM_RESERVED_MEMORY 0UL +#define PKVM_CODE_DATA_SECTIONS 1UL +#define KERNEL_DATA_SECTIONS 2UL +unsigned long addr; +unsigned long size; +u64 prot; +}; +#define PKVM_PAGES (ALIGN(sizeof(struct pkvm_hyp), PAGE_SIZE) >> PAGE_SHIFT)#define PKVM_PCPU_PAGES (ALIGN(sizeof(struct pkvm_pcpu), PAGE_SIZE) >> PAGE_SHIFT)#define PKVM_HOST_VCPU_PAGES (ALIGN(sizeof(struct pkvm_host_vcpu), PAGE_SIZE) >> PAGE_SHIFT)
@@ -60,6 +70,9 @@ struct pkvm_hyp {#define PKVM_PERCPU_PAGES (PKVM_PCPU_PAGES + PKVM_HOST_VCPU_PAGES + PKVM_VMCS_PAGES)extern char __pkvm_text_start[], __pkvm_text_end[]; +extern char __pkvm_rodata_start[], __pkvm_rodata_end[]; +extern char __pkvm_data_start[], __pkvm_data_end[]; +extern char __pkvm_bss_start[], __pkvm_bss_end[];extern unsigned long pkvm_sym (__page_base_offset );extern unsigned long pkvm_sym (__symbol_base_offset );
arch/x86/kvm/vmx/pkvm/pkvm_host.c +102 -0
108 lines not on this page · raw message
Signature · 1 line
2.25.1
RFC PATCH 16/22 pkvm: x86: Create MMU pgtable in init-finalise hypercall
In reply to Jason Chen CJCreate MMU pgtable for pKVM, this is the first stage to initialize MMU pgtable, which is based on early page allocator.
Such early page allocator's memory pool is allocated from pKVM reserved memory, after removed data struct size which already be allocated during pkvm_init.
Take use of pkvm_early_mmu_init() to initialize hyp_mmu pgtable in mmu.c, then do MMU page mapping based on it.
The MMU page mapping is setup for each system memory block in hyp_memory[] array and each memory section from init-finalise hypercall parameter; the address translation is according to previous commit "Define hypervisor runtime VA/PA APIs". In summary, pKVM keep same direct & symbol mapping as Linux kernel.
--- arch/x86/kvm/vmx/pkvm/hyp/init_finalise.c | 95 ++++++++++++++++++++++- arch/x86/kvm/vmx/pkvm/hyp/mmu.c | 2 + arch/x86/kvm/vmx/pkvm/include/pkvm.h | 14 ++++ 3 files changed, 109 insertions(+), 2 deletions(-)
arch/x86/kvm/vmx/pkvm/hyp/init_finalise.c +93 -2
@@ -3,15 +3,93 @@* Copyright (C) 2022 Intel Corporation */ +#include <linux/memblock.h> +#include <mmu.h> +#include <mmu/spte.h> +#include <asm/kvm_pkvm.h> +#include <pkvm.h> +#include "pkvm_hyp.h" +#include "early_alloc.h" +#include "memory.h" +#include "pgtable.h" +#include "mmu.h" #include "debug.h" +void *pkvm_mmu_pgt_base; + +static int divide_memory_pool (phys_addr_t phys,unsigned long size) +{ +int data_struct_size =pkvm_data_struct_pages (PKVM_PAGES , +PKVM_PCPU_PAGES +PKVM_HOST_VCPU_PAGES + +PKVM_VMCS_PAGES , pkvm_hyp->num_cpus) <<PAGE_SHIFT ; +void *virt =__pkvm_va (phys + data_struct_size); +unsigned long nr_pages; + +pkvm_early_alloc_init (virt, size - data_struct_size); + + nr_pages =pkvm_mmu_pgtable_pages (); + pkvm_mmu_pgt_base =pkvm_early_alloc_contig (nr_pages); +if (!pkvm_mmu_pgt_base) +return -ENOMEM ; + +return 0 ; +} + +static int create_mmu_mapping (const struct pkvm_section sections[], +int section_sz) +{ +unsigned long nr_pages =pkvm_mmu_pgtable_pages (); +struct memblock_region *reg; +int ret, i; + + ret =pkvm_early_mmu_init (&pkvm_hyp->mmu_cap, + pkvm_mmu_pgt_base, nr_pages); +if (ret) +return ret; + +/* +* Create mapping for the memory in memblocks. +* This will include all the memory host kernel can see, as well +* as the memory pkvm allocated during init. +* +* The virtual address for this mapping is the same with the kernel +* direct mapping. +*/ +for (i =0 ; i < hyp_memblock_nr; i++) { + reg = &hyp_memory[i]; + ret =pkvm_mmu_map ((unsigned long )__pkvm_va (reg->base), + reg->base, reg->size, +0 , (u64 )pgprot_val (PAGE_KERNEL )); +if (ret) +return ret; + } + +for (i =0 ; i < section_sz; i++) { +if (sections[i].type !=PKVM_RESERVED_MEMORY ) { + ret =pkvm_mmu_map (sections[i].addr, +__pkvm_pa_symbol (sections[i].addr), + sections[i].size, +0 , sections[i].prot); + } +if (ret) +return ret; + } + +/* Switch the mmu pgtable to enable pkvm_vmemmap */ +native_write_cr3 (pkvm_hyp->mmu->root_pa); + +return 0 ; +} +#define TMP_SECTION_SZ 16ULint __pkvm_init_finalise (struct kvm_vcpu *vcpu,struct pkvm_section sections[],int section_sz) {int i, ret =0 ;static bool pkvm_init; +struct pkvm_host_vcpu *pkvm_host_vcpu =to_pkvm_hvcpu (vcpu); +struct pkvm_pcpu *pcpu = pkvm_host_vcpu->pcpu;struct pkvm_section tmp_sections[TMP_SECTION_SZ ];phys_addr_t hyp_mem_base;unsigned long hyp_mem_size =0 ;
@@ -39,14 +117,27 @@ int __pkvm_init_finalise(struct kvm_vcpu *vcpu, struct pkvm_section sections[],if (hyp_mem_size ==0 ) {pkvm_err ("pkvm: no pkvm reserve memory!" ); ret = -ENOTSUPP ; + goto out; + } + + ret =divide_memory_pool (hyp_mem_base, hyp_mem_size); +if (ret) { +pkvm_err ("pkvm: not reserve enough memory!" ); + goto out; } -/* TODO: setup MMU & host EPT page tables */ + ret =create_mmu_mapping (tmp_sections, section_sz); +if (ret) + goto out; + +/* TODO: setup host EPT page table */ pkvm_init = true; switch_pgt: -/* TODO: switch MMU & EPT */ +/* switch mmu, TODO: switch EPT */ +vmcs_writel (HOST_CR3 , pkvm_hyp->mmu->root_pa); + pcpu->cr3 = pkvm_hyp->mmu->root_pa; out:return ret;
arch/x86/kvm/vmx/pkvm/hyp/mmu.c +2 -0
@@ -10,6 +10,7 @@#include <mmu/spte.h> #include <pkvm.h> +#include "pkvm_hyp.h" #include "early_alloc.h" #include "pgtable.h" #include "mmu.h"
@@ -117,6 +118,7 @@ int pkvm_early_mmu_init(struct pkvm_pgtable_cap *cap, void *mmu_pool_base, unsigned long mmu_pool_pages) { pkvm_early_alloc_init(mmu_pool_base, mmu_pool_pages << PAGE_SHIFT); + pkvm_hyp->mmu = &hyp_mmu;return pkvm_pgtable_init (&hyp_mmu, &pkvm_early_alloc_mm_ops, &mmu_ops, cap, true); }
arch/x86/kvm/vmx/pkvm/include/pkvm.h +14 -0
@@ -48,11 +48,25 @@ struct pkvm_hyp {struct pkvm_pgtable_cap mmu_cap;struct pkvm_pgtable_cap ept_cap; +struct pkvm_pgtable *mmu; +struct pkvm_pcpu *pcpus[CONFIG_NR_CPUS ];struct pkvm_host_vm host_vm; }; +static inline struct pkvm_host_vcpu *vmx_to_pkvm_hvcpu (struct vcpu_vmx *vmx) +{ + return container_of(vmx, struct pkvm_host_vcpu, vmx); +} + +static inline struct pkvm_host_vcpu *to_pkvm_hvcpu (struct kvm_vcpu *vcpu) +{ +struct vcpu_vmx *vmx =to_vmx (vcpu); + +return vmx_to_pkvm_hvcpu (vmx); +} +struct pkvm_section {unsigned long type;#define PKVM_RESERVED_MEMORY 0UL
Signature · 1 line
2.25.1
RFC PATCH 17/22 pkvm: x86: Add vmemmap and switch to buddy page allocator
In reply to Jason Chen CJpKVM's buddy page allocator manages all system memory pages through vmemmap region, which contains hyp_page array equal to the page number of system memory. The vmemmap region's memory need allocating from pKVM's reserved memory, and its virtual address is start from 0, such mapping need setting up through MMU page table.
After buddy page allocator got setup, the pKVM's MMU page table can switch to use it, through pkvm_later_mmu_init(). As early allocated pages' usage is not tracked by buddy page allocator, fix the page refcount for all present page in current MMU page table.
--- arch/x86/kvm/vmx/pkvm/hyp/init_finalise.c | 58 +++++++++++++ arch/x86/kvm/vmx/pkvm/hyp/mmu.c | 100 ++++++++++++++++++++++ arch/x86/kvm/vmx/pkvm/hyp/mmu.h | 2 + 3 files changed, 160 insertions(+)
arch/x86/kvm/vmx/pkvm/hyp/init_finalise.c +58 -0
@@ -9,6 +9,7 @@#include <asm/kvm_pkvm.h> #include <pkvm.h> +#include <gfp.h> #include "pkvm_hyp.h" #include "early_alloc.h" #include "memory.h"
@@ -17,6 +18,7 @@#include "debug.h" void *pkvm_mmu_pgt_base; +void *pkvm_vmemmap_base;static int divide_memory_pool (phys_addr_t phys,unsigned long size) {
@@ -28,6 +30,11 @@ static int divide_memory_pool(phys_addr_t phys, unsigned long size)pkvm_early_alloc_init (virt, size - data_struct_size); + nr_pages =pkvm_vmemmap_pages (sizeof (struct hyp_page )); + pkvm_vmemmap_base =pkvm_early_alloc_contig (nr_pages); +if (!pkvm_vmemmap_base) +return -ENOMEM ; + nr_pages =pkvm_mmu_pgtable_pages (); pkvm_mmu_pgt_base =pkvm_early_alloc_contig (nr_pages);if (!pkvm_mmu_pgt_base)
@@ -36,6 +43,51 @@ static int divide_memory_pool(phys_addr_t phys, unsigned long size)return 0 ; } +static int pkvm_back_vmemmap (phys_addr_t back_pa) +{ +unsigned long i, start, start_va, size, end, end_va =0 ; +struct memblock_region *reg; +int ret; + +/* vmemmap region map to virtual address 0 */ + __hyp_vmemmap =0 ; + +for (i =0 ; i < hyp_memblock_nr; i++) { + reg = &hyp_memory[i]; + start = reg->base; +/* Translate a range of memory to vmemmap range */ + start_va =ALIGN_DOWN ((unsigned long )hyp_phys_to_page (start), +PAGE_SIZE ); +/* +* The beginning of the pkvm_vmemmap region for the current +* memblock may already be backed by the page backing the end +* of the previous region, so avoid mapping it twice. +*/ + start_va =max (start_va, end_va); + + end = reg->base + reg->size; + end_va =ALIGN ((unsigned long )hyp_phys_to_page (end),PAGE_SIZE ); +if (start_va >= end_va) +continue ; + + size = end_va - start_va; +/* +* Create mapping for vmemmap virtual address +* [start, start+size) to physical address +* [back, back+size). +*/ + ret =pkvm_mmu_map (start_va, back_pa, size,0 , + (u64 )pgprot_val (PAGE_KERNEL )); +if (ret) +return ret; + +memset (__pkvm_va (back_pa),0 , size); + back_pa += size; + } + +return 0 ; +} +static int create_mmu_mapping (const struct pkvm_section sections[],int section_sz) {
@@ -76,9 +128,15 @@ static int create_mmu_mapping(const struct pkvm_section sections[],return ret; } + ret =pkvm_back_vmemmap (__pkvm_pa (pkvm_vmemmap_base)); +if (ret) +return ret; +/* Switch the mmu pgtable to enable pkvm_vmemmap */ native_write_cr3 (pkvm_hyp->mmu->root_pa); +pkvm_later_mmu_init (pkvm_mmu_pgt_base, nr_pages); +return 0 ; }
arch/x86/kvm/vmx/pkvm/hyp/mmu.c +100 -0
123 lines not on this page · raw message
arch/x86/kvm/vmx/pkvm/hyp/mmu.h +2 -0
@@ -14,4 +14,6 @@ int pkvm_mmu_unmap(unsigned long vaddr_start, unsigned long phys_start,int pkvm_early_mmu_init (struct pkvm_pgtable_cap *cap,void *mmu_pool_base,unsigned long mmu_pool_pages); +int pkvm_later_mmu_init (void *mmu_pool_base,unsigned long mmu_pool_pages); +#endif
Signature · 1 line
2.25.1
RFC PATCH 18/22 pkvm: x86: Introduce host EPT pgtable support
In reply to Jason Chen CJpKVM need to setup host EPT page table to isolate its memory from host VM.
Add ept.c for pKVM, which provide APIs to enable host VM EPT pgtable mapping. As buddy page allocator is ready, use it for this host EPT pgtable.
The tlb flush ops is still a TODO for host EPT, which need to shootdown all CPUs to make INVEPT.
--- arch/x86/kvm/vmx/pkvm/hyp/Makefile | 2 +- arch/x86/kvm/vmx/pkvm/hyp/ept.c | 151 +++++++++++++++++++++++++++++ arch/x86/kvm/vmx/pkvm/hyp/ept.h | 15 +++ 3 files changed, 167 insertions(+), 1 deletion(-)
arch/x86/kvm/vmx/pkvm/hyp/Makefile +1 -1
@@ -13,7 +13,7 @@ lib-dir := lib virt-dir := ../../../../../../$(KVM_PKVM) pkvm-hyp-y := vmx_asm.o vmexit.o memory.o early_alloc.o pgtable.o mmu.o pkvm.o \ - init_finalise.o + init_finalise.o ept.o pkvm-hyp-y += $(lib-dir)/memset_64.o pkvm-hyp-$(CONFIG_RETPOLINE) += $(lib-dir)/retpoline.o
arch/x86/kvm/vmx/pkvm/hyp/ept.c new file +151 -0
151 lines not on this page · raw message
arch/x86/kvm/vmx/pkvm/hyp/ept.h new file +15 -0
@@ -0,0 +1,15 @@ +// SPDX-License-Identifier: GPL-2.0 +/* +* Copyright (C) 2022 Intel Corporation +*/ +#ifndef __PKVM_EPT_H +#define __PKVM_EPT_H + +int pkvm_host_ept_map (unsigned long vaddr_start,unsigned long phys_start, +unsigned long size,int pgsz_mask,u64 prot); +int pkvm_host_ept_unmap (unsigned long vaddr_start,unsigned long phys_start, +unsigned long size); +int pkvm_host_ept_init (struct pkvm_pgtable_cap *cap,void *ept_pool_base, +unsigned long ept_pool_pages); + +#endif
Signature · 1 line
2.25.1
RFC PATCH 19/22 pkvm: x86: Create host EPT pgtable in init-finalise hypercall
In reply to Jason Chen CJCreate host EPT pgtable for pKVM, which ensure host VM keep accessing all system memory except pKVM ones - the code/data sections and reserved memory of pKVM.
The memory pool of this host EPT pgtable is allocated from pKVM reserved memory. All system memory block in hyp_memory[] array is firstly mapped in host EPT, the holes among these memory blocks are also mapped in case for MMIO accessing. Then the code/data sections and reserved memory block of pKVM is unmapped to remove the host accessing.
After host EPT pgtable got created, set host vcpu's EPT pointer to it.
This patch also makes the init-finalise hypercall take effect at the end of pkvm_init.
--- arch/x86/kvm/vmx/pkvm/hyp/ept.c | 2 + arch/x86/kvm/vmx/pkvm/hyp/ept.h | 5 ++ arch/x86/kvm/vmx/pkvm/hyp/init_finalise.c | 98 ++++++++++++++++++++++- arch/x86/kvm/vmx/pkvm/hyp/vmx.h | 48 +++++++++++ arch/x86/kvm/vmx/pkvm/include/pkvm.h | 1 + arch/x86/kvm/vmx/pkvm/pkvm_host.c | 2 +- 6 files changed, 153 insertions(+), 3 deletions(-)
arch/x86/kvm/vmx/pkvm/hyp/ept.c +2 -0
@@ -12,6 +12,7 @@#include <pkvm.h> #include <gfp.h> +#include "pkvm_hyp.h" #include "early_alloc.h" #include "pgtable.h" #include "ept.h"
@@ -147,5 +148,6 @@ int pkvm_host_ept_init(struct pkvm_pgtable_cap *cap,if (ret)return ret; + pkvm_hyp->host_vm.ept = &host_ept;return pkvm_pgtable_init (&host_ept, &host_ept_mm_ops, &ept_ops, cap, true); }
arch/x86/kvm/vmx/pkvm/hyp/ept.h +5 -0
@@ -5,6 +5,11 @@#ifndef __PKVM_EPT_H#define __PKVM_EPT_H +#define HOST_EPT_DEF_MEM_PROT (VMX_EPT_RWX_MASK | \ + (MTRR_TYPE_WRBACK << VMX_EPT_MT_EPTE_SHIFT)) +#define HOST_EPT_DEF_MMIO_PROT (VMX_EPT_RWX_MASK | \ + (MTRR_TYPE_UNCACHABLE << VMX_EPT_MT_EPTE_SHIFT)) +int pkvm_host_ept_map (unsigned long vaddr_start,unsigned long phys_start,unsigned long size,int pgsz_mask,u64 prot); int pkvm_host_ept_unmap(unsigned long vaddr_start, unsigned long phys_start,
arch/x86/kvm/vmx/pkvm/hyp/init_finalise.c +96 -2
139 lines not on this page · raw message
arch/x86/kvm/vmx/pkvm/hyp/vmx.h new file +48 -0
48 lines not on this page · raw message
arch/x86/kvm/vmx/pkvm/include/pkvm.h +1 -0
@@ -37,6 +37,7 @@ struct pkvm_host_vcpu {struct pkvm_host_vm {struct pkvm_host_vcpu *host_vcpus[CONFIG_NR_CPUS ]; +struct pkvm_pgtable *ept; };struct pkvm_hyp {
arch/x86/kvm/vmx/pkvm/pkvm_host.c +1 -1
@@ -854,7 +854,7 @@ __init int pkvm_init(void) pkvm->num_cpus =num_possible_cpus (); -return 0 ; +return pkvm_init_finalise (); out:pkvm_sym (pkvm_hyp) =NULL ;
Signature · 1 line
2.25.1
RFC PATCH 20/22 pkvm: x86: Add pgtable API pkvm_pgtable_lookup
In reply to Jason Chen CJAdd pkvm_pgtable_lookup API for pgtable. It can be used to walk page table for a translation lookup, such as looking up if a page mapping is present in a specific page table.
--- arch/x86/kvm/vmx/pkvm/hyp/ept.c | 6 +++ arch/x86/kvm/vmx/pkvm/hyp/memory.h | 2 + arch/x86/kvm/vmx/pkvm/hyp/mmu.c | 6 +++ arch/x86/kvm/vmx/pkvm/hyp/pgtable.c | 74 ++++++++++++++++++++++++++++- arch/x86/kvm/vmx/pkvm/hyp/pgtable.h | 5 ++ 5 files changed, 92 insertions(+), 1 deletion(-)
arch/x86/kvm/vmx/pkvm/hyp/ept.c +6 -0
@@ -102,6 +102,11 @@ static int ept_level_to_entries(int level)return SPTE_ENT_PER_PAGE ; } +static u64 ept_level_page_mask (int level) +{ +return (~((1UL <<SPTE_LEVEL_SHIFT (level)) -1 )); +} +static unsigned long ept_level_to_size (int level) {return KVM_HPAGE_SIZE (level);
@@ -119,6 +124,7 @@ struct pkvm_pgtable_ops ept_ops = { .pgt_entry_to_phys = ept_entry_to_phys, .pgt_entry_to_prot = ept_entry_to_prot, .pgt_entry_to_index = ept_entry_to_index, + .pgt_level_page_mask = ept_level_page_mask, .pgt_entry_is_leaf = ept_entry_is_leaf, .pgt_level_entry_size = ept_level_entry_size, .pgt_level_to_entries = ept_level_to_entries,
arch/x86/kvm/vmx/pkvm/hyp/memory.h +2 -0
@@ -7,6 +7,8 @@#include <asm/kvm_pkvm.h> +#define INVALID_ADDR (~0UL) +unsigned long pkvm_virt_to_symbol_phys (void *virt);#define __pkvm_pa_symbol (x) pkvm_virt_to_symbol_phys((void *)x)
arch/x86/kvm/vmx/pkvm/hyp/mmu.c +6 -0
@@ -109,6 +109,11 @@ static void mmu_set_entry(void *ptep, u64 pte)native_set_pte ((pte_t *)ptep,native_make_pte (pte)); } +static u64 mmu_level_page_mask (int level) +{ +return (~((1UL <<SPTE_LEVEL_SHIFT (level)) -1 )); +} +struct pkvm_pgtable_ops mmu_ops = { .pgt_entry_present = mmu_entry_present, .pgt_entry_huge = mmu_entry_huge,
@@ -116,6 +121,7 @@ struct pkvm_pgtable_ops mmu_ops = { .pgt_entry_to_phys = mmu_entry_to_phys, .pgt_entry_to_prot = mmu_entry_to_prot, .pgt_entry_to_index = mmu_entry_to_index, + .pgt_level_page_mask = mmu_level_page_mask, .pgt_entry_is_leaf = mmu_entry_is_leaf, .pgt_level_entry_size = mmu_level_entry_size, .pgt_level_to_entries = mmu_level_to_entries,
arch/x86/kvm/vmx/pkvm/hyp/pgtable.c +73 -1
98 lines not on this page · raw message
arch/x86/kvm/vmx/pkvm/hyp/pgtable.h +5 -0
@@ -25,6 +25,7 @@ struct pkvm_pgtable_ops {unsigned long (*pgt_entry_to_phys)(void *pte);u64 (*pgt_entry_to_prot)(void *pte);int (*pgt_entry_to_index)(unsigned long vaddr,int level); +u64 (*pgt_level_page_mask)(int level);bool (*pgt_entry_is_leaf)(void *ptep,int level);int (*pgt_level_entry_size)(int level);int (*pgt_level_to_entries)(int level);
@@ -51,6 +52,8 @@ typedef int (*pgtable_visit_fn_t)(struct pkvm_pgtable *pgt, unsigned long vaddr, unsigned long flags, struct pgt_flush_data *flush_data, void *const arg); +#define PGTABLE_WALK_DONE 1 +struct pkvm_pgtable_walker {const pgtable_visit_fn_t cb;void *const arg;
@@ -72,5 +75,7 @@ int pkvm_pgtable_map(struct pkvm_pgtable *pgt, unsigned long vaddr_start, int pgsz_mask, u64 entry_prot);int pkvm_pgtable_unmap (struct pkvm_pgtable *pgt,unsigned long vaddr_start,unsigned long phys_start,unsigned long size); +void pkvm_pgtable_lookup (struct pkvm_pgtable *pgt,unsigned long vaddr, +unsigned long *pphys,u64 *pprot,int *plevel);void pkvm_pgtable_destroy (struct pkvm_pgtable *pgt);#endif
Signature · 1 line
2.25.1
RFC PATCH 21/22 pkvm: x86: Introduce find_mem_range API
In reply to Jason Chen CJFrom: Chuanxiao Dong <[email protected]>
Add find_mem_range API support to indicate if a memory address belonging to system memory. This is used to check if an EPT violation caused by memory accessing in the later patch.
--- arch/x86/kvm/vmx/pkvm/hyp/memory.c | 37 ++++++++++++++++++++++++++++++ arch/x86/kvm/vmx/pkvm/hyp/memory.h | 8 +++++++ 2 files changed, 45 insertions(+)
arch/x86/kvm/vmx/pkvm/hyp/memory.c +37 -0
@@ -6,6 +6,8 @@#include <linux/types.h> #include <asm/kvm_pkvm.h> +#include "memory.h" +unsigned long __page_base_offset;unsigned long __symbol_base_offset;
@@ -26,3 +28,38 @@ unsigned long pkvm_virt_to_symbol_phys(void *virt) {return (unsigned long )virt - __symbol_base_offset; } + +bool find_mem_range (unsigned long addr,struct mem_range *range) +{ +int cur, left =0 , right = hyp_memblock_nr; +struct memblock_region *reg; +unsigned long end; + + range->start =0 ; + range->end =ULONG_MAX ; + +/* The list of memblock regions is sorted, binary search it */ +while (left < right) { + cur = (left + right) >>1 ; + reg = &hyp_memory[cur]; + end = reg->base + reg->size; +if (addr < reg->base) { + right = cur; + range->end = reg->base; + }else if (addr >= end) { + left = cur +1 ; + range->start = end; + }else { + range->start = reg->base; + range->end = end; +return true; + } + } + +return false; +} + +bool mem_range_included (struct mem_range *child,struct mem_range *parent) +{ +return parent->start <= child->start && child->end <= parent->end; +}
arch/x86/kvm/vmx/pkvm/hyp/memory.h +8 -0
@@ -12,4 +12,12 @@unsigned long pkvm_virt_to_symbol_phys (void *virt);#define __pkvm_pa_symbol (x) pkvm_virt_to_symbol_phys((void *)x) +struct mem_range { +unsigned long start; +unsigned long end; +}; + +bool find_mem_range (unsigned long addr,struct mem_range *range); +bool mem_range_included (struct mem_range *child,struct mem_range *parent); +#endif
Signature · 1 line
2.25.1
RFC PATCH 22/22 pkvm: x86: Dynamically handle host MMIO EPT violation
In reply to Jason Chen CJFrom: Chuanxiao Dong <[email protected]>
The host EPT is prepopulated for all the memory and MMIO in the low address range, but not for the MMIO in high address range. If host VM is accessing such MMIO, EPT violation happen and pKVM shall do the map for it if the MMIO range belongs to the host VM, which means this MMIO cannot be pKVM owned devices(eg. IOMMU) or any pass-through device to the protected VM (this part will be supported in the future)
--- arch/x86/kvm/vmx/pkvm/hyp/ept.c | 64 ++++++++++++++++++++++++++++++ arch/x86/kvm/vmx/pkvm/hyp/ept.h | 1 + arch/x86/kvm/vmx/pkvm/hyp/vmexit.c | 5 +++ 3 files changed, 70 insertions(+)
arch/x86/kvm/vmx/pkvm/hyp/ept.c +64 -0
@@ -6,6 +6,7 @@#include <linux/types.h> #include <linux/memblock.h> #include <asm/kvm_pkvm.h> +#include <asm/pkvm_spinlock.h> #include <mmu.h> #include <mmu/spte.h>
@@ -16,9 +17,12 @@#include "early_alloc.h" #include "pgtable.h" #include "ept.h" +#include "memory.h" +#include "debug.h" static struct hyp_pool host_ept_pool;static struct pkvm_pgtable host_ept; +static pkvm_spinlock_t host_ept_lock = __PKVM_SPINLOCK_UNLOCKED;static void flush_tlb_noop (void ) { };static void *host_ept_zalloc_page (void )
@@ -157,3 +161,63 @@ int pkvm_host_ept_init(struct pkvm_pgtable_cap *cap, pkvm_hyp->host_vm.ept = &host_ept;return pkvm_pgtable_init (&host_ept, &host_ept_mm_ops, &ept_ops, cap, true); } + +int handle_host_ept_violation (unsigned long gpa) +{ +unsigned long hpa; +struct mem_range range, cur; +bool is_memory =find_mem_range (gpa, &range); +u64 prot =HOST_EPT_DEF_MMIO_PROT ; +int level; +int ret; + +if (is_memory) { +pkvm_err ("%s: not handle for memory address 0x%lx\n" , __func__, gpa); +return -EPERM ; + } + +pkvm_spin_lock (&host_ept_lock); + +pkvm_pgtable_lookup (&host_ept, gpa, &hpa,NULL , &level); +if (hpa !=INVALID_ADDR ) { + ret = -EAGAIN ; + goto out; + } + +do { +unsigned long size =ept_level_to_size (level); + + cur.start =ALIGN_DOWN (gpa, size); + cur.end = cur.start + size -1 ; +/* +* TODO: +* check if this MMIO belongs to pkvm owned devices (e.g. IOMMU) +* check if this MMIO belongs to a secure VM pass-through device. +*/ +if ((1 << level & host_ept.allowed_pgsz) && +mem_range_included (&cur, &range)) +break ; + level--; + }while (level !=PG_LEVEL_NONE ); + +if (level ==PG_LEVEL_NONE ) { +pkvm_err ("pkvm: No valid range: gpa 0x%lx, cur 0x%lx ~ 0x%lx size 0x%lx level %d\n" , + gpa, cur.start, cur.end, cur.end - cur.start +1 , level); + ret = -EPERM ; + goto out; + } + +pkvm_dbg ("pkvm: %s: cur MMIO range 0x%lx ~ 0x%lx size 0x%lx level %d\n" , + __func__, cur.start, cur.end, cur.end - cur.start +1 , level); + + ret =pkvm_host_ept_map (cur.start, cur.start, cur.end - cur.start +1 , +1 << level, prot); +if (ret == -ENOMEM ) { +/* TODO: reclaim MMIO range pages first and try do map again */ +pkvm_dbg ("%s: no memory to set host ept for addr 0x%lx\n" , + __func__, gpa); + } +out: +pkvm_spin_unlock (&host_ept_lock); +return ret; +}
arch/x86/kvm/vmx/pkvm/hyp/ept.h +1 -0
@@ -16,5 +16,6 @@ int pkvm_host_ept_unmap(unsigned long vaddr_start, unsigned long phys_start, unsigned long size);int pkvm_host_ept_init (struct pkvm_pgtable_cap *cap,void *ept_pool_base,unsigned long ept_pool_pages); +int handle_host_ept_violation (unsigned long gpa);#endif
arch/x86/kvm/vmx/pkvm/hyp/vmexit.c +5 -0
@@ -7,6 +7,7 @@#include <asm/kvm_pkvm.h> #include <pkvm.h> #include "vmexit.h" +#include "ept.h" #include "debug.h" #define CR4 4
@@ -166,6 +167,10 @@ int pkvm_main(struct kvm_vcpu *vcpu) vcpu->arch.regs[VCPU_REGS_RAX ] =handle_vmcall (vcpu); skip_instruction = true;break ; +case EXIT_REASON_EPT_VIOLATION : +if (handle_host_ept_violation (vmcs_read64 (GUEST_PHYSICAL_ADDRESS ))) + skip_instruction = true; +break ;default :pkvm_dbg ("CPU%d: Unsupported vmexit reason 0x%x.\n" , vcpu->cpu, vmx->exit_reason.full); skip_instruction = true;
Signature · 1 line
2.25.1