VBS/VSM-on-KVM: QEMU support for the secure VM plane
cover VBS/VSM-on-KVM: QEMU support for the secure VM plane
This RFC adds the QEMU support needed to run a Virtualization-Based Security (VBS), or Hyper-V VSM-style, secure environment using KVM VM Planes. It is the QEMU counterpart to the Linux VBS/VSM-on-KVM series.
The five patches in this mail series are authored by me and add:
- userspace handlers for VM Plane and VBS VTL-call exits from KVM;
- the VBS VTL hypercall definitions from the Linux UAPI;
- loading and running the secure-plane kernel in-kernel; and
- plane configuration access through the QEMU address-space API.
This series depends on the QEMU VM Planes infrastructure from Joerg Roedel and related prerequisite work. Those patches are not duplicated in this mail series. The integrated QEMU branch linked below contains the complete tree, including Joerg's patches and the other prerequisites, used for development and testing.
The complete implementation is available in three repositories under the safe-tee organization:
QEMU support and complete integrated tree:
https://github.com/safe-tee/qemu/tree/qemu-planes-v11.1.0-rc1
Linux and KVM support:
https://github.com/safe-tee/linux/tree/kvm-planes-v7.2-rc5
Build, test, and integration tooling:
https://github.com/safe-tee/lvbs
Current build and run instructions are maintained in the lvbs repository:
https://github.com/safe-tee/lvbs/blob/main/docs/how-to/how-to-build.md
This is prototype code and is not yet intended for production use. Feedback on the QEMU integration, VTL-call handling, and division of responsibility between QEMU and the in-kernel VM Plane implementation would be appreciated.
Acknowledgments
===============
This work stands on top of, and is indebted to, several prior efforts:
- Joerg Roedel, whose QEMU and KVM VM Planes work provides the infrastructure that the VBS/VSM secure plane relies on.
- Paolo Bonzini, whose "[RFC PATCH 00/29] KVM: VM planes" introduced the VM Plane concept to KVM as a common in-kernel model for AMD VMPLs, Intel TDX partitions, Hyper-V VTLs, and Arm CCA planes.
https://lwn.net/Articles/1016113/
- James Bottomley and James Morris, for their ongoing VSM-on-KVM work, which informed the design and direction of this series.
Sriram Nambakam (5):
kvm: add userspace handlers for VM planes and VBS VTL calls
vm_planes: Add VBS VTL call handling and plane memory sealing
linux-headers: sync kvm_para.h VBS VTL hypercalls target/i386/kvm: run the secure plane in-kernel (Option B) target/i386/kvm: read plane config via address_space API
accel/kvm/kvm-all.c | 18 + include/standard-headers/linux/kvm_para.h | 7 +- include/system/kvm_int.h | 18 + linux-headers/linux/kvm.h | 16 + target/i386/kvm/kvm.c | 561 ++++++++++++++++++++++ 5 files changed, 619 insertions(+), 1 deletion(-)
Signature · 1 line
2.55.0
RFC PATCH 1/5 kvm: add userspace handlers for VM planes and VBS VTL calls
(cherry picked from commit 47d0da9ac50640f96564997edc4b79f44e70cffa)
--- accel/kvm/kvm-all.c | 18 + include/standard-headers/linux/kvm_para.h | 3 + include/system/kvm_int.h | 17 + linux-headers/linux/kvm.h | 3 + target/i386/kvm/kvm.c | 776 ++++++++++++++++++++++ 5 files changed, 817 insertions(+)
accel/kvm/kvm-all.c +18 -0
@@ -822,6 +822,24 @@ void kvm_close(void)close (kvm_get_plane_fd (kvm_state, plane_id));kvm_set_plane_fd (kvm_state, plane_id,-1 ); }while (plane_id !=0 ); +if (kvm_state->vm_planes) { +unsigned int i, j; +for (i =1 ; i < kvm_state->vm_plane_count; i++) { +struct kvm_vm_plane_state *ps = &kvm_state->vm_planes[i]; +if (ps->vcpu_fds) { +for (j =0 ; j < ps->vcpu_count; j++) { +if (ps->vcpu_fds[j] >=0 ) { +close (ps->vcpu_fds[j]); + } + } +g_free (ps->vcpu_fds); + ps->vcpu_fds =NULL ; + } + } +g_free (kvm_state->vm_planes); + kvm_state->vm_planes =NULL ; + kvm_state->vm_plane_count =0 ; + }close (kvm_state->fd); kvm_state->fd =-1 ; }
include/standard-headers/linux/kvm_para.h +3 -0
@@ -30,6 +30,9 @@#define KVM_HC_SEND_IPI 10#define KVM_HC_SCHED_YIELD 11#define KVM_HC_MAP_GPA_RANGE 12 +#define KVM_HC_VM_PLANES_CONFIG 13 +#define KVM_HC_VM_PLANES_ACTIVATE 14 +#define KVM_HC_VBS_VTL_CALL 15/* * hypercalls use architecture specific
include/system/kvm_int.h +17 -0
@@ -109,6 +109,19 @@ struct KVMPlane {bool vcpu_dirty; }; +/* Per-plane VM state managed by LVBS hypercall handlers. +* The plane fd itself is owned by the accel layer and accessed via +* kvm_get_plane_fd(s, plane_id); only LVBS-specific state lives here. */ +struct kvm_vm_plane_state { +int *vcpu_fds; +unsigned int vcpu_count; +uint64_t load_offset; +uint64_t memory_size; +uint64_t entry_point; +void *host_addr;/* host pointer to plane RAM (cleared after launch) */ +char cmdline[512 ]; +}; +struct KVMState {AccelState parent_obj;
@@ -176,6 +189,10 @@ struct KVMStateuint16_t xen_evtchn_max_pirq;char *device;OnOffAuto honor_guest_pat; +/* VM planes state (populated by LVBS hypercall handlers) */ +struct kvm_vm_plane_state *vm_planes; +unsigned int vm_plane_count; +unsigned int vm_planes_max; };static inline void kvm_set_plane_fd (KVMState *s,unsigned plane,int fd)
linux-headers/linux/kvm.h +3 -0
@@ -1654,6 +1654,9 @@ struct kvm_memory_attributes {#define KVM_MEMORY_ATTRIBUTE_PRIVATE (1ULL << 3) +#define KVM_MEMORY_ATTRIBUTE_NO_WRITE (1ULL << 4) +#define KVM_MEMORY_ATTRIBUTE_NO_EXEC (1ULL << 5) +#define KVM_CREATE_GUEST_MEMFD _IOWR(KVMIO, 0xd4, struct kvm_create_guest_memfd)#define GUEST_MEMFD_FLAG_MMAP (1ULL << 0)#define GUEST_MEMFD_FLAG_INIT_SHARED (1ULL << 1)
target/i386/kvm/kvm.c +776 -0
798 lines not on this page · raw message
Signature · 1 line
2.55.0
RFC PATCH 2/5 vm_planes: Add VBS VTL call handling and plane memory sealing
In reply to Sriram NambakamWire up the VBS inter-plane communication and memory protection infrastructure on the QEMU side.
Memory sealing (plane activate path):
After plane-1 boots and its vCPU threads are running, seal plane-1's memory from plane-0 access:
1. Issue KVM_SET_PLANE_MEMORY_ATTRIBUTES to set NO_WRITE|NO_EXEC on plane-0's EPT for the GPA range occupied by plane-1.
2. Clear ps->host_addr so QEMU userspace can no longer access the region.
VBS VTL hypercall handler:
Handle KVM_HC_VBS_VTL_CALL (hypercall 15) exits from the guest. The plane-0 VBS subsystem passes the GPA of a shared calling-area (CAA) page; QEMU reads the call_id from the page, logs it, and returns -ENOSYS for now (plane-1 responder not yet implemented). Enable the hypercall in kvm_enable_hypercall() alongside the existing VM planes hypercalls.
Header sync:
- linux-headers/linux/kvm.h: Add KVM_MEMORY_ATTRIBUTE_NO_WRITE, KVM_MEMORY_ATTRIBUTE_NO_EXEC, struct kvm_plane_memory_attributes, and KVM_SET_PLANE_MEMORY_ATTRIBUTES ioctl definition.
- include/standard-headers/linux/kvm_para.h: Add KVM_HC_VBS_VTL_CALL.
(cherry picked from commit 414016c40ec8226b0f5d4cbd5eed0589b7c1689e)
--- linux-headers/linux/kvm.h | 13 +++++++++++++ 1 file changed, 13 insertions(+)
linux-headers/linux/kvm.h +13 -0
@@ -1653,6 +1653,19 @@ struct kvm_memory_attributes { };#define KVM_MEMORY_ATTRIBUTE_PRIVATE (1ULL << 3) +#define KVM_MEMORY_ATTRIBUTE_NO_WRITE (1ULL << 4) +#define KVM_MEMORY_ATTRIBUTE_NO_EXEC (1ULL << 5) + +/* Set memory attributes on a specific plane's address space. */ +struct kvm_plane_memory_attributes { +__u32 plane; +__u32 flags; +__u64 address; +__u64 size; +__u64 attributes; +}; + +#define KVM_SET_PLANE_MEMORY_ATTRIBUTES _IOW(KVMIO, 0xd6, struct kvm_plane_memory_attributes)#define KVM_MEMORY_ATTRIBUTE_NO_WRITE (1ULL << 4)#define KVM_MEMORY_ATTRIBUTE_NO_EXEC (1ULL << 5)
Signature · 1 line
2.55.0
RFC PATCH 3/5 linux-headers: sync kvm_para.h VBS VTL hypercalls
In reply to Sriram NambakamImport the two new VBS secure-plane hypercall numbers from the kernel
UAPI: KVM_HC_VBS_VTL_RETURN (16) and KVM_HC_VBS_SET_MEM_ATTRS (17).
--- include/standard-headers/linux/kvm_para.h | 2 ++ 1 file changed, 2 insertions(+)
include/standard-headers/linux/kvm_para.h +2 -0
@@ -33,6 +33,8 @@#define KVM_HC_VM_PLANES_CONFIG 13#define KVM_HC_VM_PLANES_ACTIVATE 14#define KVM_HC_VBS_VTL_CALL 15 +#define KVM_HC_VBS_VTL_RETURN 16 +#define KVM_HC_VBS_SET_MEM_ATTRS 17/* * hypercalls use architecture specific
Signature · 1 line
2.55.0
RFC PATCH 4/5 target/i386/kvm: run the secure plane in-kernel (Option B)
In reply to Sriram NambakamMove secure-plane execution into the kernel and out of QEMU. Plane vCPUs are now created and initialised on their owning plane-0 CPU thread via run_on_cpu() (plane_vcpu_create_cb / plane_vcpu_init_cb), and the activate path leaves the secure vCPU stopped: it boots lazily inside KVM on the first KVM_HC_VBS_VTL_CALL, which KVM now services itself.
Drop the userspace secure-world emulation that this replaces: the dedicated vm_plane_vcpu_thread (with its 8250 emulation and per-plane serial capture), vbs_apply_protection, vbs_handle_protect_memory, vbs_handle_seal_kernel and the kvm_handle_hc_vbs_vtl_call dispatch. Track each plane vCPU's owning CPU index (vcpu_cpu_index) so the run_on_cpu callbacks target the right thread.
--- include/system/kvm_int.h | 1 + target/i386/kvm/kvm.c | 536 ++++++++++++--------------------------- 2 files changed, 157 insertions(+), 380 deletions(-)
include/system/kvm_int.h +1 -0
@@ -114,6 +114,7 @@ struct KVMPlane {* kvm_get_plane_fd(s, plane_id); only LVBS-specific state lives here. */ struct kvm_vm_plane_state {int *vcpu_fds; +unsigned int *vcpu_cpu_index;/* owning plane-0 CPU index per plane vCPU */ unsigned int vcpu_count;uint64_t load_offset;uint64_t memory_size;
target/i386/kvm/kvm.c +156 -380
649 lines not on this page · raw message
Signature · 1 line
2.55.0
RFC PATCH 5/5 target/i386/kvm: read plane config via address_space API
In reply to Sriram NambakamMatch the vm-planes-merged integration result: read the guest vm_plane_config/calling-area through address_space_read() (with the address-spaces/memory/thread includes) instead of the cpu_physical_memory_read() helpers, and pick up the kvm_para.h whitespace from the header re-sync. This delta originated in the integration branch's merge-commit conflict resolution.
--- include/standard-headers/linux/kvm_para.h | 2 +- target/i386/kvm/kvm.c | 23 ++++++++++++++++------- 2 files changed, 17 insertions(+), 8 deletions(-)
include/standard-headers/linux/kvm_para.h +1 -1
@@ -27,7 +27,7 @@#define KVM_HC_MIPS_EXIT_VM 7#define KVM_HC_MIPS_CONSOLE_OUTPUT 8#define KVM_HC_CLOCK_PAIRING 9 -#define KVM_HC_SEND_IPI 10 +#define KVM_HC_SEND_IPI 10#define KVM_HC_SCHED_YIELD 11#define KVM_HC_MAP_GPA_RANGE 12#define KVM_HC_VM_PLANES_CONFIG 13
target/i386/kvm/kvm.c +16 -7
@@ -21,6 +21,7 @@#include <sys/utsname.h> #include <sys/syscall.h> #include <sys/resource.h> +#include <sys/mman.h> #include <linux/kvm.h> #include <linux/kvm_para.h>
@@ -36,6 +37,8 @@#include "system/kvm_int.h" #include "system/runstate.h" #include "system/ramblock.h" +#include "system/address-spaces.h" +#include "system/memory.h" #include "kvm_i386.h" #include "../confidential-guest.h" #include "sev.h"
@@ -47,6 +50,7 @@#include "gdbstub/enums.h" #include "qemu/host-utils.h" #include "qemu/main-loop.h" +#include "qemu/thread.h" #include "qemu/ratelimit.h" #include "qemu/config-file.h" #include "qemu/error-report.h"
@@ -6665,9 +6669,12 @@ static int kvm_handle_hc_vm_planes_config(X86CPU *cpu, struct kvm_run *run)int plane_fd;unsigned int i; -cpu_physical_memory_read (plane_gpa +0 , &load_offset,8 ); -cpu_physical_memory_read (plane_gpa +8 , &memory_size,8 ); -cpu_physical_memory_read (plane_gpa +16 , &entry_point,8 ); +address_space_read (&address_space_memory, plane_gpa +0 , +MEMTXATTRS_UNSPECIFIED , &load_offset,8 ); +address_space_read (&address_space_memory, plane_gpa +8 , +MEMTXATTRS_UNSPECIFIED , &memory_size,8 ); +address_space_read (&address_space_memory, plane_gpa +16 , +MEMTXATTRS_UNSPECIFIED , &entry_point,8 );/* * joergroedel plane model: a plane has exactly one vCPU per
@@ -6685,8 +6692,9 @@ static int kvm_handle_hc_vm_planes_config(X86CPU *cpu, struct kvm_run *run) }memset (cmdline_buf,0 ,sizeof (cmdline_buf)); -cpu_physical_memory_read (plane_gpa +156 , cmdline_buf, -sizeof (cmdline_buf)); +address_space_read (&address_space_memory, plane_gpa +156 , +MEMTXATTRS_UNSPECIFIED , cmdline_buf, +sizeof (cmdline_buf)); cmdline_buf[sizeof (cmdline_buf) -1 ] ='\0' ;memcpy (ps->cmdline, cmdline_buf,sizeof (ps->cmdline));
@@ -6899,8 +6907,9 @@ static int kvm_handle_hc_vm_planes_activate(X86CPU *cpu, struct kvm_run *run)return 0 ; } -cpu_physical_memory_read (gpa + (plane_id *VM_PLANE_CFG_STRIDE ) +16 , - &entry_point,8 ); +address_space_read (&address_space_memory, + gpa + (plane_id *VM_PLANE_CFG_STRIDE ) +16 , +MEMTXATTRS_UNSPECIFIED , &entry_point,8 );if (!entry_point) {error_report ("vm_planes: plane %" PRIu64" bad entry_point" , plane_id);
Signature · 1 line
2.55.0