Skip to contentSkip to conversation

VBS/VSM-on-KVM: QEMU support for the secure VM plane

RFC PATCH New Sriram Nambakam 1 participant 6 messages Aug 5 KVM

lore
Share & tools
lore
#0 lore raw

cover VBS/VSM-on-KVM: QEMU support for the secure VM plane

This RFC adds the QEMU support needed to run a Virtualization-Based Security (VBS), or Hyper-V VSM-style, secure environment using KVM VM Planes. It is the QEMU counterpart to the Linux VBS/VSM-on-KVM series.

The five patches in this mail series are authored by me and add:

- userspace handlers for VM Plane and VBS VTL-call exits from KVM;
- the VBS VTL hypercall definitions from the Linux UAPI;
- loading and running the secure-plane kernel in-kernel; and
- plane configuration access through the QEMU address-space API.

This series depends on the QEMU VM Planes infrastructure from Joerg Roedel and related prerequisite work. Those patches are not duplicated in this mail series. The integrated QEMU branch linked below contains the complete tree, including Joerg's patches and the other prerequisites, used for development and testing.

The complete implementation is available in three repositories under the safe-tee organization:

QEMU support and complete integrated tree:
https://github.com/safe-tee/qemu/tree/qemu-planes-v11.1.0-rc1

Linux and KVM support:
https://github.com/safe-tee/linux/tree/kvm-planes-v7.2-rc5

Build, test, and integration tooling:
https://github.com/safe-tee/lvbs

Current build and run instructions are maintained in the lvbs repository:

https://github.com/safe-tee/lvbs/blob/main/docs/how-to/how-to-build.md

This is prototype code and is not yet intended for production use. Feedback on the QEMU integration, VTL-call handling, and division of responsibility between QEMU and the in-kernel VM Plane implementation would be appreciated.

Acknowledgments
===============

This work stands on top of, and is indebted to, several prior efforts:

- Joerg Roedel, whose QEMU and KVM VM Planes work provides the infrastructure that the VBS/VSM secure plane relies on.

- Paolo Bonzini, whose "[RFC PATCH 00/29] KVM: VM planes" introduced the VM Plane concept to KVM as a common in-kernel model for AMD VMPLs, Intel TDX partitions, Hyper-V VTLs, and Arm CCA planes.
https://lwn.net/Articles/1016113/

- James Bottomley and James Morris, for their ongoing VSM-on-KVM work, which informed the design and direction of this series.

Sriram Nambakam (5):
kvm: add userspace handlers for VM planes and VBS VTL calls
vm_planes: Add VBS VTL call handling and plane memory sealing
linux-headers: sync kvm_para.h VBS VTL hypercalls target/i386/kvm: run the secure plane in-kernel (Option B) target/i386/kvm: read plane config via address_space API

 accel/kvm/kvm-all.c                       |  18 +
 include/standard-headers/linux/kvm_para.h |   7 +-
 include/system/kvm_int.h                  |  18 +
 linux-headers/linux/kvm.h                 |  16 +
 target/i386/kvm/kvm.c                     | 561 ++++++++++++++++++++++
 5 files changed, 619 insertions(+), 1 deletion(-)
Signature · 1 line
2.55.0
#1 lore raw

RFC PATCH 1/5 kvm: add userspace handlers for VM planes and VBS VTL calls

(cherry picked from commit 47d0da9ac50640f96564997edc4b79f44e70cffa)

---
 accel/kvm/kvm-all.c                       |  18 +
 include/standard-headers/linux/kvm_para.h |   3 +
 include/system/kvm_int.h                  |  17 +
 linux-headers/linux/kvm.h                 |   3 +
 target/i386/kvm/kvm.c                     | 776 ++++++++++++++++++++++
 5 files changed, 817 insertions(+)
accel/kvm/kvm-all.c +18 -0
index 46a14ac0f4..80a9791078 100644
@@ -822,6 +822,24 @@ void kvm_close(void)
             close(kvm_get_plane_fd(kvm_state, plane_id));
             kvm_set_plane_fd(kvm_state, plane_id, -1);
         } while (plane_id != 0);
+        if (kvm_state->vm_planes) {
+            unsigned int i, j;
+            for (i = 1; i < kvm_state->vm_plane_count; i++) {
+                struct kvm_vm_plane_state *ps = &kvm_state->vm_planes[i];
+                if (ps->vcpu_fds) {
+                    for (j = 0; j < ps->vcpu_count; j++) {
+                        if (ps->vcpu_fds[j] >= 0) {
+                            close(ps->vcpu_fds[j]);
+                        }
+                    }
+                    g_free(ps->vcpu_fds);
+                    ps->vcpu_fds = NULL;
+                }
+            }
+            g_free(kvm_state->vm_planes);
+            kvm_state->vm_planes = NULL;
+            kvm_state->vm_plane_count = 0;
+        }
         close(kvm_state->fd);
         kvm_state->fd = -1;
     }
include/standard-headers/linux/kvm_para.h +3 -0
index 015c166302..ad19aac9d0 100644
@@ -30,6 +30,9 @@
 #define KVM_HC_SEND_IPI		10
 #define KVM_HC_SCHED_YIELD		11
 #define KVM_HC_MAP_GPA_RANGE		12
+#define KVM_HC_VM_PLANES_CONFIG		13
+#define KVM_HC_VM_PLANES_ACTIVATE	14
+#define KVM_HC_VBS_VTL_CALL		15
 
 /*
  * hypercalls use architecture specific
include/system/kvm_int.h +17 -0
index 70b381f1ba..e7c9dc95a2 100644
@@ -109,6 +109,19 @@ struct KVMPlane {
     bool vcpu_dirty;
 };
 
+/* Per-plane VM state managed by LVBS hypercall handlers.
+ * The plane fd itself is owned by the accel layer and accessed via
+ * kvm_get_plane_fd(s, plane_id); only LVBS-specific state lives here. */
+struct kvm_vm_plane_state {
+    int *vcpu_fds;
+    unsigned int vcpu_count;
+    uint64_t load_offset;
+    uint64_t memory_size;
+    uint64_t entry_point;
+    void *host_addr;        /* host pointer to plane RAM (cleared after launch) */
+    char cmdline[512];
+};
+
 struct KVMState
 {
     AccelState parent_obj;
@@ -176,6 +189,10 @@ struct KVMState
     uint16_t xen_evtchn_max_pirq;
     char *device;
     OnOffAuto honor_guest_pat;
+    /* VM planes state (populated by LVBS hypercall handlers) */
+    struct kvm_vm_plane_state *vm_planes;
+    unsigned int vm_plane_count;
+    unsigned int vm_planes_max;
 };
 
 static inline void kvm_set_plane_fd(KVMState *s, unsigned plane, int fd)
linux-headers/linux/kvm.h +3 -0
index 8caa3eccce..d66ce6d272 100644
@@ -1654,6 +1654,9 @@ struct kvm_memory_attributes {
 
 #define KVM_MEMORY_ATTRIBUTE_PRIVATE           (1ULL << 3)
 
+#define KVM_MEMORY_ATTRIBUTE_NO_WRITE          (1ULL << 4)
+#define KVM_MEMORY_ATTRIBUTE_NO_EXEC           (1ULL << 5)
+
 #define KVM_CREATE_GUEST_MEMFD	_IOWR(KVMIO,  0xd4, struct kvm_create_guest_memfd)
 #define GUEST_MEMFD_FLAG_MMAP		(1ULL << 0)
 #define GUEST_MEMFD_FLAG_INIT_SHARED	(1ULL << 1)
target/i386/kvm/kvm.c +776 -0

798 lines not on this page · raw message

Signature · 1 line
2.55.0
#2 lore raw

RFC PATCH 2/5 vm_planes: Add VBS VTL call handling and plane memory sealing

In reply to Sriram Nambakam

Wire up the VBS inter-plane communication and memory protection infrastructure on the QEMU side.

Memory sealing (plane activate path):
After plane-1 boots and its vCPU threads are running, seal plane-1's memory from plane-0 access:
1. Issue KVM_SET_PLANE_MEMORY_ATTRIBUTES to set NO_WRITE|NO_EXEC on plane-0's EPT for the GPA range occupied by plane-1.
2. Clear ps->host_addr so QEMU userspace can no longer access the region.

VBS VTL hypercall handler:
Handle KVM_HC_VBS_VTL_CALL (hypercall 15) exits from the guest. The plane-0 VBS subsystem passes the GPA of a shared calling-area (CAA) page; QEMU reads the call_id from the page, logs it, and returns -ENOSYS for now (plane-1 responder not yet implemented). Enable the hypercall in kvm_enable_hypercall() alongside the existing VM planes hypercalls.

Header sync:
- linux-headers/linux/kvm.h: Add KVM_MEMORY_ATTRIBUTE_NO_WRITE, KVM_MEMORY_ATTRIBUTE_NO_EXEC, struct kvm_plane_memory_attributes, and KVM_SET_PLANE_MEMORY_ATTRIBUTES ioctl definition.
- include/standard-headers/linux/kvm_para.h: Add KVM_HC_VBS_VTL_CALL.

(cherry picked from commit 414016c40ec8226b0f5d4cbd5eed0589b7c1689e)

---
 linux-headers/linux/kvm.h | 13 +++++++++++++
 1 file changed, 13 insertions(+)
linux-headers/linux/kvm.h +13 -0
index d66ce6d272..9dbd3824e9 100644
@@ -1653,6 +1653,19 @@ struct kvm_memory_attributes {
 };
 
 #define KVM_MEMORY_ATTRIBUTE_PRIVATE           (1ULL << 3)
+#define KVM_MEMORY_ATTRIBUTE_NO_WRITE          (1ULL << 4)
+#define KVM_MEMORY_ATTRIBUTE_NO_EXEC           (1ULL << 5)
+
+/* Set memory attributes on a specific plane's address space. */
+struct kvm_plane_memory_attributes {
+	__u32 plane;
+	__u32 flags;
+	__u64 address;
+	__u64 size;
+	__u64 attributes;
+};
+
+#define KVM_SET_PLANE_MEMORY_ATTRIBUTES	_IOW(KVMIO, 0xd6, struct kvm_plane_memory_attributes)
 
 #define KVM_MEMORY_ATTRIBUTE_NO_WRITE          (1ULL << 4)
 #define KVM_MEMORY_ATTRIBUTE_NO_EXEC           (1ULL << 5)
Signature · 1 line
2.55.0
#3 lore raw

RFC PATCH 3/5 linux-headers: sync kvm_para.h VBS VTL hypercalls

In reply to Sriram Nambakam

Import the two new VBS secure-plane hypercall numbers from the kernel
UAPI: KVM_HC_VBS_VTL_RETURN (16) and KVM_HC_VBS_SET_MEM_ATTRS (17).

(cherry picked from commit 92f71c68c2eb42588694ac23fb4a6a8ef8dfa12b)
---
 include/standard-headers/linux/kvm_para.h | 2 ++
 1 file changed, 2 insertions(+)
include/standard-headers/linux/kvm_para.h +2 -0
index ad19aac9d0..def451c43c 100644
@@ -33,6 +33,8 @@
 #define KVM_HC_VM_PLANES_CONFIG		13
 #define KVM_HC_VM_PLANES_ACTIVATE	14
 #define KVM_HC_VBS_VTL_CALL		15
+#define KVM_HC_VBS_VTL_RETURN		16
+#define KVM_HC_VBS_SET_MEM_ATTRS	17
 
 /*
  * hypercalls use architecture specific
Signature · 1 line
2.55.0
#4 lore raw

RFC PATCH 4/5 target/i386/kvm: run the secure plane in-kernel (Option B)

In reply to Sriram Nambakam

Move secure-plane execution into the kernel and out of QEMU. Plane vCPUs are now created and initialised on their owning plane-0 CPU thread via run_on_cpu() (plane_vcpu_create_cb / plane_vcpu_init_cb), and the activate path leaves the secure vCPU stopped: it boots lazily inside KVM on the first KVM_HC_VBS_VTL_CALL, which KVM now services itself.

Drop the userspace secure-world emulation that this replaces: the dedicated vm_plane_vcpu_thread (with its 8250 emulation and per-plane serial capture), vbs_apply_protection, vbs_handle_protect_memory, vbs_handle_seal_kernel and the kvm_handle_hc_vbs_vtl_call dispatch. Track each plane vCPU's owning CPU index (vcpu_cpu_index) so the run_on_cpu callbacks target the right thread.

(cherry picked from commit cbecc9eda1514d200fefe2bb89f330be4f1ab1be)
---
 include/system/kvm_int.h |   1 +
 target/i386/kvm/kvm.c    | 536 ++++++++++++---------------------------
 2 files changed, 157 insertions(+), 380 deletions(-)
include/system/kvm_int.h +1 -0
index e7c9dc95a2..cde0a148e1 100644
@@ -114,6 +114,7 @@ struct KVMPlane {
  * kvm_get_plane_fd(s, plane_id); only LVBS-specific state lives here. */
 struct kvm_vm_plane_state {
     int *vcpu_fds;
+    unsigned int *vcpu_cpu_index;  /* owning plane-0 CPU index per plane vCPU */
     unsigned int vcpu_count;
     uint64_t load_offset;
     uint64_t memory_size;
target/i386/kvm/kvm.c +156 -380

649 lines not on this page · raw message

Signature · 1 line
2.55.0
#5 lore raw

RFC PATCH 5/5 target/i386/kvm: read plane config via address_space API

In reply to Sriram Nambakam

Match the vm-planes-merged integration result: read the guest vm_plane_config/calling-area through address_space_read() (with the address-spaces/memory/thread includes) instead of the cpu_physical_memory_read() helpers, and pick up the kvm_para.h whitespace from the header re-sync. This delta originated in the integration branch's merge-commit conflict resolution.

---
 include/standard-headers/linux/kvm_para.h |  2 +-
 target/i386/kvm/kvm.c                     | 23 ++++++++++++++++-------
 2 files changed, 17 insertions(+), 8 deletions(-)
include/standard-headers/linux/kvm_para.h +1 -1
index def451c43c..3094d70c8d 100644
@@ -27,7 +27,7 @@
 #define KVM_HC_MIPS_EXIT_VM		7
 #define KVM_HC_MIPS_CONSOLE_OUTPUT	8
 #define KVM_HC_CLOCK_PAIRING		9
-#define KVM_HC_SEND_IPI		10
+#define KVM_HC_SEND_IPI			10
 #define KVM_HC_SCHED_YIELD		11
 #define KVM_HC_MAP_GPA_RANGE		12
 #define KVM_HC_VM_PLANES_CONFIG		13
target/i386/kvm/kvm.c +16 -7
index 5ac7962e61..b8a68a1c56 100644
@@ -21,6 +21,7 @@
 #include <sys/utsname.h>
 #include <sys/syscall.h>
 #include <sys/resource.h>
+#include <sys/mman.h>
 
 #include <linux/kvm.h>
 #include <linux/kvm_para.h>
@@ -36,6 +37,8 @@
 #include "system/kvm_int.h"
 #include "system/runstate.h"
 #include "system/ramblock.h"
+#include "system/address-spaces.h"
+#include "system/memory.h"
 #include "kvm_i386.h"
 #include "../confidential-guest.h"
 #include "sev.h"
@@ -47,6 +50,7 @@
 #include "gdbstub/enums.h"
 #include "qemu/host-utils.h"
 #include "qemu/main-loop.h"
+#include "qemu/thread.h"
 #include "qemu/ratelimit.h"
 #include "qemu/config-file.h"
 #include "qemu/error-report.h"
@@ -6665,9 +6669,12 @@ static int kvm_handle_hc_vm_planes_config(X86CPU *cpu, struct kvm_run *run)
         int plane_fd;
         unsigned int i;
 
-        cpu_physical_memory_read(plane_gpa + 0,  &load_offset, 8);
-        cpu_physical_memory_read(plane_gpa + 8,  &memory_size, 8);
-        cpu_physical_memory_read(plane_gpa + 16, &entry_point, 8);
+        address_space_read(&address_space_memory, plane_gpa + 0,
+                           MEMTXATTRS_UNSPECIFIED, &load_offset, 8);
+        address_space_read(&address_space_memory, plane_gpa + 8,
+                           MEMTXATTRS_UNSPECIFIED, &memory_size, 8);
+        address_space_read(&address_space_memory, plane_gpa + 16,
+                           MEMTXATTRS_UNSPECIFIED, &entry_point, 8);
 
         /*
          * joergroedel plane model: a plane has exactly one vCPU per
@@ -6685,8 +6692,9 @@ static int kvm_handle_hc_vm_planes_config(X86CPU *cpu, struct kvm_run *run)
         }
 
         memset(cmdline_buf, 0, sizeof(cmdline_buf));
-        cpu_physical_memory_read(plane_gpa + 156, cmdline_buf,
-                                 sizeof(cmdline_buf));
+        address_space_read(&address_space_memory, plane_gpa + 156,
+                           MEMTXATTRS_UNSPECIFIED, cmdline_buf,
+                           sizeof(cmdline_buf));
         cmdline_buf[sizeof(cmdline_buf) - 1] = '\0';
         memcpy(ps->cmdline, cmdline_buf, sizeof(ps->cmdline));
 
@@ -6899,8 +6907,9 @@ static int kvm_handle_hc_vm_planes_activate(X86CPU *cpu, struct kvm_run *run)
             return 0;
         }
 
-        cpu_physical_memory_read(gpa + (plane_id * VM_PLANE_CFG_STRIDE) + 16,
-                                 &entry_point, 8);
+        address_space_read(&address_space_memory,
+                           gpa + (plane_id * VM_PLANE_CFG_STRIDE) + 16,
+                           MEMTXATTRS_UNSPECIFIED, &entry_point, 8);
         if (!entry_point) {
             error_report("vm_planes: plane %" PRIu64 " bad entry_point",
                          plane_id);
Signature · 1 line
2.55.0