Qemu SEV-ES guest support

From: Tom Lendacky <[email protected]>

Provide initial support for SEV-ES. This includes creating a function to indicate the guest is an SEV-ES guest (which will return false until all support is in place), performing the proper SEV initialization and ensuring that the guest CPU state is measured as part of the launch.

Co-developed-by Jiri Slaby <[email protected]>

Signed-off-by Jiri Slaby <[email protected]>

Signed-off-by Tom Lendacky <[email protected]>

---
 target/i386/cpu.c      |  1 +
 target/i386/sev-stub.c |  5 +++++
 target/i386/sev.c      | 46 ++++++++++++++++++++++++++++++++++++++++--
 target/i386/sev_i386.h |  1 +
 4 files changed, 51 insertions(+), 2 deletions(-)

target/i386/cpu.c +0 -0

index 588f32e136..bbbe581d35 100644

target/i386/cpu.c +0 -0

         *ebx = sev_get_cbit_position();
         *ebx |= sev_get_reduced_phys_bits() << 6;
         *ecx = 0;

target/i386/sev-stub.c +5 -0

index 88e3f39a1e..040ac90563 100644
@@ -49,3 +49,8 @@ SevCapability *sev_get_capabilities(Error **errp)
     error_setg(errp, "SEV is not available in this QEMU");
     return NULL;
 }
+
+bool sev_es_enabled(void)
+{
+    return false;
+}

target/i386/sev.c +44 -2

index c3ecf86704..6c9cd0854b 100644
@@ -359,6 +359,12 @@ sev_enabled(void)
     return !!sev_guest;
 }
 
+bool
+sev_es_enabled(void)
+{
+    return false;
+}
+
 uint64_t
 sev_get_me_mask(void)
 {

@@ -578,6 +584,22 @@ sev_launch_update_data(SevGuestState *sev, uint8_t *addr, uint64_t len)
     return ret;
 }
 
+static int
+sev_launch_update_vmsa(SevGuestState *sev)
+{
+    int ret, fw_error;
+
+    ret = sev_ioctl(sev->sev_fd, KVM_SEV_LAUNCH_UPDATE_VMSA, NULL, &fw_error);
+    if (ret) {
+        error_report("%s: LAUNCH_UPDATE_VMSA ret=%d fw_error=%d '%s'",
+                __func__, ret, fw_error, fw_error_to_str(fw_error));
+        goto err;
+    }
+
+err:
+    return ret;
+}
+
 static void
 sev_launch_get_measure(Notifier *notifier, void *unused)
 {

@@ -590,6 +612,14 @@ sev_launch_get_measure(Notifier *notifier, void *unused)
         return;
     }
 
+    if (sev_es_enabled()) {
+        /* measure all the VM save areas before getting launch_measure */
+        ret = sev_launch_update_vmsa(sev);
+        if (ret) {
+            exit(1);
+        }
+    }
+
     measurement = g_new0(struct kvm_sev_launch_measure, 1);
 
     /* query the measurement blob length */

@@ -684,7 +714,7 @@ sev_guest_init(const char *id)
 {
     SevGuestState *sev;
     char *devname;
-    int ret, fw_error;
+    int ret, fw_error, cmd;
     uint32_t ebx;
     uint32_t host_cbitpos;
     struct sev_user_data_status status = {};

@@ -745,8 +775,20 @@ sev_guest_init(const char *id)
     sev->api_major = status.api_major;
     sev->api_minor = status.api_minor;
 
+    if (sev_es_enabled()) {
+        if (!(status.flags & SEV_STATUS_FLAGS_CONFIG_ES)) {
+            error_report("%s: guest policy requires SEV-ES, but "
+                         "host SEV-ES support unavailable",
+                         __func__);
+            goto err;
+        }
+        cmd = KVM_SEV_ES_INIT;
+    } else {
+        cmd = KVM_SEV_INIT;
+    }
+
     trace_kvm_sev_init();
-    ret = sev_ioctl(sev->sev_fd, KVM_SEV_INIT, NULL, &fw_error);
+    ret = sev_ioctl(sev->sev_fd, cmd, NULL, &fw_error);
     if (ret) {
         error_report("%s: failed to initialize ret=%d fw_error=%d '%s'",
                      __func__, ret, fw_error, fw_error_to_str(fw_error));

target/i386/sev_i386.h +1 -0

index 4db6960f60..4f9a5e9b21 100644
@@ -29,6 +29,7 @@
 #define SEV_POLICY_SEV          0x20
 
 extern bool sev_enabled(void);
+extern bool sev_es_enabled(void);
 extern uint64_t sev_get_me_mask(void);
 extern SevInfo *sev_get_info(void);
 extern uint32_t sev_get_cbit_position(void);

Signature · 1 line

2.28.0

From: Tom Lendacky <[email protected]>

Provide initial support for SEV-ES. This includes creating a function to indicate the guest is an SEV-ES guest (which will return false until all support is in place), performing the proper SEV initialization and ensuring that the guest CPU state is measured as part of the launch.

Co-developed-by Jiri Slaby <[email protected]>

Signed-off-by Jiri Slaby <[email protected]>

Signed-off-by Tom Lendacky <[email protected]>

---
  target/i386/cpu.c      |  1 +
  target/i386/sev-stub.c |  5 +++++
  target/i386/sev.c      | 46 ++++++++++++++++++++++++++++++++++++++++--
  target/i386/sev_i386.h |  1 +
  4 files changed, 51 insertions(+), 2 deletions(-)

target/i386/cpu.c +1 -0

index 588f32e136..bbbe581d35 100644
@@ -5969,6 +5969,7 @@ void cpu_x86_cpuid(CPUX86State *env, uint32_t index, uint32_t count,
          break;
      case 0x8000001F:
          *eax = sev_enabled() ? 0x2 : 0;
+        *eax |= sev_es_enabled() ? 0x8 : 0;
          *ebx = sev_get_cbit_position();
          *ebx |= sev_get_reduced_phys_bits() << 6;
          *ecx = 0;

target/i386/sev-stub.c +5 -0

index 88e3f39a1e..040ac90563 100644
@@ -49,3 +49,8 @@ SevCapability *sev_get_capabilities(Error **errp)
      error_setg(errp, "SEV is not available in this QEMU");
      return NULL;
  }
+
+bool sev_es_enabled(void)
+{
+    return false;
+}

target/i386/sev.c +6 -0

index c3ecf86704..6c9cd0854b 100644
@@ -359,6 +359,12 @@ sev_enabled(void)
      return !!sev_guest;
  }
 
+bool
+sev_es_enabled(void)
+{
+    return false;
+}
+
  uint64_t
  sev_get_me_mask(void)
  {

@@ -578,6 +584,22 @@ sev_launch_update_data(SevGuestState *sev, uint8_t *addr, uint64_t len)
      return ret;
  }
 

target/i386/sev.c +6 -0

+    }
+
+err:
+    return ret;
+}
+
  static void
  sev_launch_get_measure(Notifier *notifier, void *unused)
  {

target/i386/sev.c +17 -2

+        }
+    }
+
      measurement = g_new0(struct kvm_sev_launch_measure, 1);
 
      /* query the measurement blob length */

@@ -684,7 +714,7 @@ sev_guest_init(const char *id)
  {
      SevGuestState *sev;
      char *devname;
-    int ret, fw_error;
+    int ret, fw_error, cmd;
      uint32_t ebx;
      uint32_t host_cbitpos;
      struct sev_user_data_status status = {};

@@ -745,8 +775,20 @@ sev_guest_init(const char *id)
      sev->api_major = status.api_major;
      sev->api_minor = status.api_minor;
 
+    if (sev_es_enabled()) {
+        if (!(status.flags & SEV_STATUS_FLAGS_CONFIG_ES)) {
+            error_report("%s: guest policy requires SEV-ES, but "
+                         "host SEV-ES support unavailable",
+                         __func__);
+            goto err;
+        }
+        cmd = KVM_SEV_ES_INIT;
+    } else {
+        cmd = KVM_SEV_INIT;
+    }
+
      trace_kvm_sev_init();
-    ret = sev_ioctl(sev->sev_fd, KVM_SEV_INIT, NULL, &fw_error);
+    ret = sev_ioctl(sev->sev_fd, cmd, NULL, &fw_error);
      if (ret) {
          error_report("%s: failed to initialize ret=%d fw_error=%d '%s'",
                       __func__, ret, fw_error, fw_error_to_str(fw_error));

target/i386/sev_i386.h +1 -0

index 4db6960f60..4f9a5e9b21 100644
@@ -29,6 +29,7 @@
  #define SEV_POLICY_SEV          0x20
 
  extern bool sev_enabled(void);
+extern bool sev_es_enabled(void);
  extern uint64_t sev_get_me_mask(void);
  extern SevInfo *sev_get_info(void);
  extern uint32_t sev_get_cbit_position(void);

Quoted 65 lines · Tom Lendacky

From: Tom Lendacky <[email protected]>

Provide initial support for SEV-ES. This includes creating a function to indicate the guest is an SEV-ES guest (which will return false until all support is in place), performing the proper SEV initialization and ensuring that the guest CPU state is measured as part of the launch.

Co-developed-by Jiri Slaby <[email protected]>

Signed-off-by Jiri Slaby <[email protected]>

Signed-off-by Tom Lendacky <[email protected]>

---
  target/i386/cpu.c      |  1 +
  target/i386/sev-stub.c |  5 +++++
  target/i386/sev.c      | 46 ++++++++++++++++++++++++++++++++++++++++-- target/i386/sev_i386.h |  1 +

  4 files changed, 51 insertions(+), 2 deletions(-)

target/i386/cpu.c +0 -0

index 588f32e136..bbbe581d35 100644
@@ -5969,6 +5969,7 @@ void cpu_x86_cpuid(CPUX86State *env, uint32_t
 index, uint32_t count,
           break;

      case 0x8000001F:
          *eax = sev_enabled() ? 0x2 : 0;
+        *eax |= sev_es_enabled() ? 0x8 : 0; *ebx = sev_get_cbit_position(); *ebx |= sev_get_reduced_phys_bits() << 6; *ecx = 0;

target/i386/sev-stub.c +5 -0

index 88e3f39a1e..040ac90563 100644
@@ -49,3 +49,8 @@ SevCapability *sev_get_capabilities(Error **errp)
       error_setg(errp, "SEV is not available in this QEMU");
       return NULL;
   }
+
+bool sev_es_enabled(void)
+{
+    return false;
+}

target/i386/sev.c +6 -0

index c3ecf86704..6c9cd0854b 100644
@@ -359,6 +359,12 @@ sev_enabled(void)
       return !!sev_guest;
   }
 
+bool
+sev_es_enabled(void)
+{
+    return false;
+}
+

  uint64_t
  sev_get_me_mask(void)
  {

target/i386/sev.c +0 -0

@@ -578,6 +584,22 @@ sev_launch_update_data(SevGuestState *sev, uint8_t
 *addr, uint64_t len)
       return ret;
   }
 

target/i386/sev.c sev_launch_update_data(SevGuestState *sev, uint8_t *addr, uint64_t len) · line 596

Quoted 11 lines

target/i386/sev.c +10 -0

+static int
+sev_launch_update_vmsa(SevGuestState *sev)
+{
+    int ret, fw_error;
+
+    ret = sev_ioctl(sev->sev_fd, KVM_SEV_LAUNCH_UPDATE_VMSA, NULL,
 &fw_error);
+    if (ret) {
+        error_report("%s: LAUNCH_UPDATE_VMSA ret=%d fw_error=%d '%s'",
+                __func__, ret, fw_error, fw_error_to_str(fw_error));
+        goto err;

goto (and the err: label) is not needed.

Quoted 15 lines

target/i386/sev.c +6 -0

+    }
+
+err:
+    return ret;
+}
+

  static void
  sev_launch_get_measure(Notifier *notifier, void *unused) {

target/i386/sev.c +0 -0

@@ -590,6 +612,14 @@ sev_launch_get_measure(Notifier *notifier, void
 *unused)
           return;
       }
 

target/i386/sev.c sev_launch_get_measure(Notifier *notifier, void *unused) · line 619

Quoted 5 lines

target/i386/sev.c +5 -0

+    if (sev_es_enabled()) {
+        /* measure all the VM save areas before getting launch_measure */
+        ret = sev_launch_update_vmsa(sev);
+        if (ret) {
+            exit(1);

Other error cases in this function just return on error. Why quit QEMU here?

Quoted 51 lines

target/i386/sev.c +3 -0

+        }
+    }
+
       measurement = g_new0(struct kvm_sev_launch_measure, 1);
 

      /* query the measurement blob length */

target/i386/sev.c +14 -2

@@ -684,7 +714,7 @@ sev_guest_init(const char *id)
   {
       SevGuestState *sev;
       char *devname;
-    int ret, fw_error;
+    int ret, fw_error, cmd;
       uint32_t ebx;
       uint32_t host_cbitpos;
       struct sev_user_data_status status = {};

@@ -745,8 +775,20 @@ sev_guest_init(const char *id)
       sev->api_major = status.api_major;
       sev->api_minor = status.api_minor;
 
+    if (sev_es_enabled()) {
+        if (!(status.flags & SEV_STATUS_FLAGS_CONFIG_ES)) {
+            error_report("%s: guest policy requires SEV-ES, but "
+                         "host SEV-ES support unavailable",
+                         __func__);
+            goto err;
+        }
+        cmd = KVM_SEV_ES_INIT;
+    } else {
+        cmd = KVM_SEV_INIT;
+    }
+
       trace_kvm_sev_init();
-    ret = sev_ioctl(sev->sev_fd, KVM_SEV_INIT, NULL, &fw_error);
+    ret = sev_ioctl(sev->sev_fd, cmd, NULL, &fw_error);
       if (ret) {
           error_report("%s: failed to initialize ret=%d fw_error=%d '%s'",
                        __func__, ret, fw_error, fw_error_to_str(fw_error));

target/i386/sev_i386.h +1 -0

index 4db6960f60..4f9a5e9b21 100644
@@ -29,6 +29,7 @@
   #define SEV_POLICY_SEV          0x20
 
   extern bool sev_enabled(void);
+extern bool sev_es_enabled(void);
   extern uint64_t sev_get_me_mask(void);
   extern SevInfo *sev_get_info(void);
   extern uint32_t sev_get_cbit_position(void);

From: Tom Lendacky <[email protected]>

When SEV-ES is enabled, it is not possible modify the guests register state after it has been initially created, encrypted and measured.

Normally, an INIT-SIPI-SIPI request is used to boot the AP. However, the hypervisor cannot emulate this because it cannot update the AP register state. For the very first boot by an AP, the reset vector CS segment value and the EIP value must be programmed before the register has been encrypted and measured.

Signed-off-by Tom Lendacky <[email protected]>

---
 accel/kvm/kvm-all.c    | 64 ++++++++++++++++++++++++++++++++++++++++++
 accel/stubs/kvm-stub.c |  5 ++++
 hw/i386/pc_sysfw.c     | 10 ++++++-
 include/sysemu/kvm.h   | 16 +++++++++++
 include/sysemu/sev.h   |  3 ++
 target/i386/kvm.c      |  2 ++
 target/i386/sev.c      | 51 +++++++++++++++++++++++++++++++++
 7 files changed, 150 insertions(+), 1 deletion(-)

accel/kvm/kvm-all.c +64 -0

index 63ef6af9a1..20725b0368 100644
@@ -39,6 +39,7 @@
 #include "qemu/main-loop.h"
 #include "trace.h"
 #include "hw/irq.h"
+#include "sysemu/kvm.h"
 #include "sysemu/sev.h"
 #include "qapi/visitor.h"
 #include "qapi/qapi-types-common.h"

@@ -120,6 +121,12 @@ struct KVMState
     /* memory encryption */
     void *memcrypt_handle;
     int (*memcrypt_encrypt_data)(void *handle, uint8_t *ptr, uint64_t len);
+    int (*memcrypt_save_reset_vector)(void *handle, void *flash_ptr,
+                                      uint64_t flash_size, uint32_t *addr);
+
+    unsigned int reset_cs;
+    unsigned int reset_ip;
+    bool reset_valid;
 
     /* For "info mtree -f" to tell if an MR is registered in KVM */
     int nr_as;

@@ -239,6 +246,62 @@ int kvm_memcrypt_encrypt_data(uint8_t *ptr, uint64_t len)
     return 1;
 }
 
+void kvm_memcrypt_set_reset_vector(CPUState *cpu)
+{
+    X86CPU *x86;
+    CPUX86State *env;
+
+    /* Only update if we have valid reset information */
+    if (!kvm_state->reset_valid) {
+        return;
+    }
+
+    /* Do not update the BSP reset state */
+    if (cpu->cpu_index == 0) {
+        return;
+    }
+
+    x86 = X86_CPU(cpu);
+    env = &x86->env;
+
+    cpu_x86_load_seg_cache(env, R_CS, 0xf000, kvm_state->reset_cs, 0xffff,
+                           DESC_P_MASK | DESC_S_MASK | DESC_CS_MASK |
+                           DESC_R_MASK | DESC_A_MASK);
+
+    env->eip = kvm_state->reset_ip;
+}
+
+int kvm_memcrypt_save_reset_vector(void *flash_ptr, uint64_t flash_size)
+{
+    CPUState *cpu;
+    uint32_t addr;
+    int ret;
+
+    if (kvm_memcrypt_enabled() &&
+        kvm_state->memcrypt_save_reset_vector) {
+
+        addr = 0;
+        ret = kvm_state->memcrypt_save_reset_vector(kvm_state->memcrypt_handle,
+                                                    flash_ptr, flash_size,
+                                                    &addr);
+        if (ret) {
+            return ret;
+        }
+
+        if (addr) {
+            kvm_state->reset_cs = addr & 0xffff0000;
+            kvm_state->reset_ip = addr & 0x0000ffff;
+            kvm_state->reset_valid = true;
+
+            CPU_FOREACH(cpu) {
+                kvm_memcrypt_set_reset_vector(cpu);
+            }
+        }
+    }
+
+    return 0;
+}
+
 /* Called with KVMMemoryListener.slots_lock held */
 static KVMSlot *kvm_get_free_slot(KVMMemoryListener *kml)
 {

@@ -2193,6 +2256,7 @@ static int kvm_init(MachineState *ms)
         }
 
         kvm_state->memcrypt_encrypt_data = sev_encrypt_data;
+        kvm_state->memcrypt_save_reset_vector = sev_es_save_reset_vector;
     }
 
     ret = kvm_arch_init(ms, s);

accel/stubs/kvm-stub.c +5 -0

index 82f118d2df..3aece9b513 100644
@@ -114,6 +114,11 @@ int kvm_memcrypt_encrypt_data(uint8_t *ptr, uint64_t len)
   return 1;
 }
 
+int kvm_memcrypt_save_reset_vector(void *flash_ptr, uint64_t flash_size)
+{
+    return -ENOSYS;
+}
+
 #ifndef CONFIG_USER_ONLY
 int kvm_irqchip_add_msi_route(KVMState *s, int vector, PCIDevice *dev)
 {

hw/i386/pc_sysfw.c +9 -1

index b6c0822fe3..321ff94261 100644
@@ -156,7 +156,8 @@ static void pc_system_flash_map(PCMachineState *pcms,
     PFlashCFI01 *system_flash;
     MemoryRegion *flash_mem;
     void *flash_ptr;
-    int ret, flash_size;
+    uint64_t flash_size;
+    int ret;
 
     assert(PC_MACHINE_GET_CLASS(pcms)->pci_enabled);
 

@@ -204,6 +205,13 @@ static void pc_system_flash_map(PCMachineState *pcms,
             if (kvm_memcrypt_enabled()) {
                 flash_ptr = memory_region_get_ram_ptr(flash_mem);
                 flash_size = memory_region_size(flash_mem);
+
+                ret = kvm_memcrypt_save_reset_vector(flash_ptr, flash_size);
+                if (ret) {
+                    error_report("failed to locate and/or save reset vector");
+                    exit(1);
+                }
+
                 ret = kvm_memcrypt_encrypt_data(flash_ptr, flash_size);
                 if (ret) {
                     error_report("failed to encrypt pflash rom");

include/sysemu/kvm.h +16 -0

index b4174d941c..f74cfa85ab 100644
@@ -247,6 +247,22 @@ bool kvm_memcrypt_enabled(void);
  */
 int kvm_memcrypt_encrypt_data(uint8_t *ptr, uint64_t len);
 
+/**
+ * kvm_memcrypt_set_reset_vector - sets the CS/IP value for the AP if SEV-ES
+ *                                 is active.
+ */
+void kvm_memcrypt_set_reset_vector(CPUState *cpu);
+
+/**
+ * kvm_memcrypt_save_reset_vector - locates and saves the reset vector to be
+ *                                  used as the initial CS/IP value for APs
+ *                                  if SEV-ES is active.
+ *
+ * Return: 1 SEV-ES is active and failed to locate a valid reset vector
+ *         0 SEV-ES is not active or successfully located and saved the
+ *           reset vector address
+ */
+int kvm_memcrypt_save_reset_vector(void *flash_prt, uint64_t flash_size);
 
 #ifdef NEED_CPU_H
 #include "cpu.h"

include/sysemu/sev.h +3 -0

index 98c1ec8d38..5198e5a621 100644
@@ -18,4 +18,7 @@
 
 void *sev_guest_init(const char *id);
 int sev_encrypt_data(void *handle, uint8_t *ptr, uint64_t len);
+int sev_es_save_reset_vector(void *handle, void *flash_ptr,
+                             uint64_t flash_size, uint32_t *addr);
+
 #endif

target/i386/kvm.c +2 -0

index 6f18d940a5..10eaba8943 100644
@@ -1912,6 +1912,8 @@ void kvm_arch_reset_vcpu(X86CPU *cpu)
     }
     /* enabled by default */
     env->poll_control_msr = 1;
+
+    kvm_memcrypt_set_reset_vector(CPU(cpu));
 }
 
 void kvm_arch_do_init_vcpu(X86CPU *cpu)

target/i386/sev.c +6 -0

index 5055b1fe00..6ddefc65fa 100644
@@ -70,6 +70,19 @@ struct SevGuestState {
 #define DEFAULT_GUEST_POLICY    0x1 /* disable debug */
 #define DEFAULT_SEV_DEVICE      "/dev/sev"
 
+/* SEV Information Block GUID = 00f771de-1a7e-4fcb-890e-68c77e2fb44e */
+#define SEV_INFO_BLOCK_GUID \
+    "\xde\x71\xf7\x00\x7e\x1a\xcb\x4f\x89\x0e\x68\xc7\x7e\x2f\xb4\x4e"
+
+typedef struct __attribute__((__packed__)) SevInfoBlock {
+    /* SEV-ES Reset Vector Address */

target/i386/sev.c +38 -0

@@ -833,6 +846,44 @@ sev_encrypt_data(void *handle, uint8_t *ptr, uint64_t len)
     return 0;
 }
 
+int
+sev_es_save_reset_vector(void *handle, void *flash_ptr, uint64_t flash_size,
+                         uint32_t *addr)
+{
+    SevInfoBlock *info;
+
+    assert(handle);
+
+    /*
+     * Initialize the address to zero. An address of zero with a successful
+     * return code indicates that SEV-ES is not active.
+     */
+    *addr = 0;
+    if (!sev_es_enabled()) {
+        return 0;
+    }
+
+    /*
+     * Extract the AP reset vector for SEV-ES guests by locating the SEV GUID.
+     * The SEV GUID is located 32 bytes from the end of the flash. Use this
+     * address to base the SEV information block.
+     */
+    info = flash_ptr + flash_size - 0x20 - sizeof(*info);
+    if (memcmp(info->guid, SEV_INFO_BLOCK_GUID, 16)) {
+        error_report("SEV information block not found in pflash rom");
+        return 1;
+    }
+
+    if (!info->reset_addr) {
+        error_report("SEV-ES reset address is zero");
+        return 1;
+    }
+
+    *addr = info->reset_addr;
+
+    return 0;
+}
+
 static void
 sev_register_types(void)
 {

Hi Tom,

sorry for the random feedback -- I haven't followed (and don't really intend to follow) the QEMU side of the feature. Just one style idea:

Quoted 228 lines · Tom Lendacky

From: Tom Lendacky <[email protected]>

When SEV-ES is enabled, it is not possible modify the guests register state after it has been initially created, encrypted and measured.

Normally, an INIT-SIPI-SIPI request is used to boot the AP. However, the hypervisor cannot emulate this because it cannot update the AP register state. For the very first boot by an AP, the reset vector CS segment value and the EIP value must be programmed before the register has been encrypted and measured.

Signed-off-by Tom Lendacky <[email protected]>

---
 accel/kvm/kvm-all.c    | 64 ++++++++++++++++++++++++++++++++++++++++++
 accel/stubs/kvm-stub.c |  5 ++++
 hw/i386/pc_sysfw.c     | 10 ++++++-
 include/sysemu/kvm.h   | 16 +++++++++++
 include/sysemu/sev.h   |  3 ++
 target/i386/kvm.c      |  2 ++
 target/i386/sev.c      | 51 +++++++++++++++++++++++++++++++++
 7 files changed, 150 insertions(+), 1 deletion(-)

accel/kvm/kvm-all.c +64 -0

index 63ef6af9a1..20725b0368 100644
@@ -39,6 +39,7 @@
 #include "qemu/main-loop.h"
 #include "trace.h"
 #include "hw/irq.h"
+#include "sysemu/kvm.h"
 #include "sysemu/sev.h"
 #include "qapi/visitor.h"
 #include "qapi/qapi-types-common.h"

@@ -120,6 +121,12 @@ struct KVMState
     /* memory encryption */
     void *memcrypt_handle;
     int (*memcrypt_encrypt_data)(void *handle, uint8_t *ptr, uint64_t len);
+    int (*memcrypt_save_reset_vector)(void *handle, void *flash_ptr,
+                                      uint64_t flash_size, uint32_t *addr);
+
+    unsigned int reset_cs;
+    unsigned int reset_ip;
+    bool reset_valid;
 
     /* For "info mtree -f" to tell if an MR is registered in KVM */
     int nr_as;

@@ -239,6 +246,62 @@ int kvm_memcrypt_encrypt_data(uint8_t *ptr, uint64_t len)
     return 1;
 }
 
+void kvm_memcrypt_set_reset_vector(CPUState *cpu)
+{
+    X86CPU *x86;
+    CPUX86State *env;
+
+    /* Only update if we have valid reset information */
+    if (!kvm_state->reset_valid) {
+        return;
+    }
+
+    /* Do not update the BSP reset state */
+    if (cpu->cpu_index == 0) {
+        return;
+    }
+
+    x86 = X86_CPU(cpu);
+    env = &x86->env;
+
+    cpu_x86_load_seg_cache(env, R_CS, 0xf000, kvm_state->reset_cs, 0xffff,
+                           DESC_P_MASK | DESC_S_MASK | DESC_CS_MASK |
+                           DESC_R_MASK | DESC_A_MASK);
+
+    env->eip = kvm_state->reset_ip;
+}
+
+int kvm_memcrypt_save_reset_vector(void *flash_ptr, uint64_t flash_size)
+{
+    CPUState *cpu;
+    uint32_t addr;
+    int ret;
+
+    if (kvm_memcrypt_enabled() &&
+        kvm_state->memcrypt_save_reset_vector) {
+
+        addr = 0;
+        ret = kvm_state->memcrypt_save_reset_vector(kvm_state->memcrypt_handle,
+                                                    flash_ptr, flash_size,
+                                                    &addr);
+        if (ret) {
+            return ret;
+        }
+
+        if (addr) {
+            kvm_state->reset_cs = addr & 0xffff0000;
+            kvm_state->reset_ip = addr & 0x0000ffff;
+            kvm_state->reset_valid = true;
+
+            CPU_FOREACH(cpu) {
+                kvm_memcrypt_set_reset_vector(cpu);
+            }
+        }
+    }
+
+    return 0;
+}
+
 /* Called with KVMMemoryListener.slots_lock held */
 static KVMSlot *kvm_get_free_slot(KVMMemoryListener *kml)
 {

@@ -2193,6 +2256,7 @@ static int kvm_init(MachineState *ms)
         }
 
         kvm_state->memcrypt_encrypt_data = sev_encrypt_data;
+        kvm_state->memcrypt_save_reset_vector = sev_es_save_reset_vector;
     }
 
     ret = kvm_arch_init(ms, s);

accel/stubs/kvm-stub.c +5 -0

index 82f118d2df..3aece9b513 100644
@@ -114,6 +114,11 @@ int kvm_memcrypt_encrypt_data(uint8_t *ptr, uint64_t len)
   return 1;
 }
 
+int kvm_memcrypt_save_reset_vector(void *flash_ptr, uint64_t flash_size)
+{
+    return -ENOSYS;
+}
+
 #ifndef CONFIG_USER_ONLY
 int kvm_irqchip_add_msi_route(KVMState *s, int vector, PCIDevice *dev)
 {

hw/i386/pc_sysfw.c +9 -1

index b6c0822fe3..321ff94261 100644
@@ -156,7 +156,8 @@ static void pc_system_flash_map(PCMachineState *pcms,
     PFlashCFI01 *system_flash;
     MemoryRegion *flash_mem;
     void *flash_ptr;
-    int ret, flash_size;
+    uint64_t flash_size;
+    int ret;
 
     assert(PC_MACHINE_GET_CLASS(pcms)->pci_enabled);
 

@@ -204,6 +205,13 @@ static void pc_system_flash_map(PCMachineState *pcms,
             if (kvm_memcrypt_enabled()) {
                 flash_ptr = memory_region_get_ram_ptr(flash_mem);
                 flash_size = memory_region_size(flash_mem);
+
+                ret = kvm_memcrypt_save_reset_vector(flash_ptr, flash_size);
+                if (ret) {
+                    error_report("failed to locate and/or save reset vector");
+                    exit(1);
+                }
+
                 ret = kvm_memcrypt_encrypt_data(flash_ptr, flash_size);
                 if (ret) {
                     error_report("failed to encrypt pflash rom");

include/sysemu/kvm.h +16 -0

index b4174d941c..f74cfa85ab 100644
@@ -247,6 +247,22 @@ bool kvm_memcrypt_enabled(void);
  */
 int kvm_memcrypt_encrypt_data(uint8_t *ptr, uint64_t len);
 
+/**
+ * kvm_memcrypt_set_reset_vector - sets the CS/IP value for the AP if SEV-ES
+ *                                 is active.
+ */
+void kvm_memcrypt_set_reset_vector(CPUState *cpu);
+
+/**
+ * kvm_memcrypt_save_reset_vector - locates and saves the reset vector to be
+ *                                  used as the initial CS/IP value for APs
+ *                                  if SEV-ES is active.
+ *
+ * Return: 1 SEV-ES is active and failed to locate a valid reset vector
+ *         0 SEV-ES is not active or successfully located and saved the
+ *           reset vector address
+ */
+int kvm_memcrypt_save_reset_vector(void *flash_prt, uint64_t flash_size);
 
 #ifdef NEED_CPU_H
 #include "cpu.h"

include/sysemu/sev.h +3 -0

index 98c1ec8d38..5198e5a621 100644
@@ -18,4 +18,7 @@
 
 void *sev_guest_init(const char *id);
 int sev_encrypt_data(void *handle, uint8_t *ptr, uint64_t len);
+int sev_es_save_reset_vector(void *handle, void *flash_ptr,
+                             uint64_t flash_size, uint32_t *addr);
+
 #endif

target/i386/kvm.c +2 -0

index 6f18d940a5..10eaba8943 100644
@@ -1912,6 +1912,8 @@ void kvm_arch_reset_vcpu(X86CPU *cpu)
     }
     /* enabled by default */
     env->poll_control_msr = 1;
+
+    kvm_memcrypt_set_reset_vector(CPU(cpu));
 }
 
 void kvm_arch_do_init_vcpu(X86CPU *cpu)

target/i386/sev.c +6 -0

index 5055b1fe00..6ddefc65fa 100644
@@ -70,6 +70,19 @@ struct SevGuestState {
 #define DEFAULT_GUEST_POLICY    0x1 /* disable debug */
 #define DEFAULT_SEV_DEVICE      "/dev/sev"
 
+/* SEV Information Block GUID = 00f771de-1a7e-4fcb-890e-68c77e2fb44e */
+#define SEV_INFO_BLOCK_GUID \
+    "\xde\x71\xf7\x00\x7e\x1a\xcb\x4f\x89\x0e\x68\xc7\x7e\x2f\xb4\x4e"
+
+typedef struct __attribute__((__packed__)) SevInfoBlock {
+    /* SEV-ES Reset Vector Address */

target/i386/sev.c struct SevGuestState { · line 88

Quoted 10 lines

target/i386/sev.c +7 -0

+    uint32_t reset_addr;
+
+    /* SEV Information Block size and GUID */
+    uint16_t size;
+    char guid[16];
+} SevInfoBlock;
+
 static SevGuestState *sev_guest;
 static Error *sev_mig_blocker;
 

Can we replace "char guid[16]" with "QemuUUID guid"?

Also, can we replace the SEV_INFO_BLOCK_GUID macro with a static const structure, initialized with UUID_LE()?

Something like

  static const QemuUUID sev_info_block_guid_le = {
      .data = UUID_LE(...)
  };

Thanks
Laszlo

Quoted 46 lines

target/i386/sev.c +38 -0

@@ -833,6 +846,44 @@ sev_encrypt_data(void *handle, uint8_t *ptr, uint64_t len)
     return 0;
 }
 
+int
+sev_es_save_reset_vector(void *handle, void *flash_ptr, uint64_t flash_size,
+                         uint32_t *addr)
+{
+    SevInfoBlock *info;
+
+    assert(handle);
+
+    /*
+     * Initialize the address to zero. An address of zero with a successful
+     * return code indicates that SEV-ES is not active.
+     */
+    *addr = 0;
+    if (!sev_es_enabled()) {
+        return 0;
+    }
+
+    /*
+     * Extract the AP reset vector for SEV-ES guests by locating the SEV GUID.
+     * The SEV GUID is located 32 bytes from the end of the flash. Use this
+     * address to base the SEV information block.
+     */
+    info = flash_ptr + flash_size - 0x20 - sizeof(*info);
+    if (memcmp(info->guid, SEV_INFO_BLOCK_GUID, 16)) {
+        error_report("SEV information block not found in pflash rom");
+        return 1;
+    }
+
+    if (!info->reset_addr) {
+        error_report("SEV-ES reset address is zero");
+        return 1;
+    }
+
+    *addr = info->reset_addr;
+
+    return 0;
+}
+
 static void
 sev_register_types(void)
 {

From: Tom Lendacky <[email protected]>

When SEV-ES is enabled, it is not possible modify the guests register state after it has been initially created, encrypted and measured.

Normally, an INIT-SIPI-SIPI request is used to boot the AP. However, the hypervisor cannot emulate this because it cannot update the AP register state. For the very first boot by an AP, the reset vector CS segment value and the EIP value must be programmed before the register has been encrypted and measured.

Signed-off-by Tom Lendacky <[email protected]>

---
 accel/kvm/kvm-all.c    | 64 ++++++++++++++++++++++++++++++++++++++++++
 accel/stubs/kvm-stub.c |  5 ++++
 hw/i386/pc_sysfw.c     | 10 ++++++-
 include/sysemu/kvm.h   | 16 +++++++++++
 include/sysemu/sev.h   |  3 ++
 target/i386/kvm.c      |  2 ++
 target/i386/sev.c      | 51 +++++++++++++++++++++++++++++++++
 7 files changed, 150 insertions(+), 1 deletion(-)

accel/kvm/kvm-all.c +1 -0

index 63ef6af9a1..20725b0368 100644
@@ -39,6 +39,7 @@
 #include "qemu/main-loop.h"
 #include "trace.h"
 #include "hw/irq.h"
+#include "sysemu/kvm.h"
 #include "sysemu/sev.h"
 #include "qapi/visitor.h"
 #include "qapi/qapi-types-common.h"

accel/kvm/kvm-all.c +58 -0

+    bool reset_valid;
 
     /* For "info mtree -f" to tell if an MR is registered in KVM */
     int nr_as;

@@ -239,6 +246,62 @@ int kvm_memcrypt_encrypt_data(uint8_t *ptr, uint64_t len)
     return 1;
 }
 
+void kvm_memcrypt_set_reset_vector(CPUState *cpu)
+{
+    X86CPU *x86;
+    CPUX86State *env;
+
+    /* Only update if we have valid reset information */
+    if (!kvm_state->reset_valid) {
+        return;
+    }
+
+    /* Do not update the BSP reset state */
+    if (cpu->cpu_index == 0) {
+        return;
+    }
+
+    x86 = X86_CPU(cpu);
+    env = &x86->env;
+
+    cpu_x86_load_seg_cache(env, R_CS, 0xf000, kvm_state->reset_cs, 0xffff,
+                           DESC_P_MASK | DESC_S_MASK | DESC_CS_MASK |
+                           DESC_R_MASK | DESC_A_MASK);
+
+    env->eip = kvm_state->reset_ip;
+}
+
+int kvm_memcrypt_save_reset_vector(void *flash_ptr, uint64_t flash_size)
+{
+    CPUState *cpu;
+    uint32_t addr;
+    int ret;
+
+    if (kvm_memcrypt_enabled() &&
+        kvm_state->memcrypt_save_reset_vector) {
+
+        addr = 0;
+        ret = kvm_state->memcrypt_save_reset_vector(kvm_state->memcrypt_handle,
+                                                    flash_ptr, flash_size,
+                                                    &addr);
+        if (ret) {
+            return ret;
+        }
+
+        if (addr) {
+            kvm_state->reset_cs = addr & 0xffff0000;
+            kvm_state->reset_ip = addr & 0x0000ffff;
+            kvm_state->reset_valid = true;
+
+            CPU_FOREACH(cpu) {
+                kvm_memcrypt_set_reset_vector(cpu);
+            }
+        }
+    }
+
+    return 0;
+}
+
 /* Called with KVMMemoryListener.slots_lock held */
 static KVMSlot *kvm_get_free_slot(KVMMemoryListener *kml)
 {

@@ -2193,6 +2256,7 @@ static int kvm_init(MachineState *ms)
         }
 
         kvm_state->memcrypt_encrypt_data = sev_encrypt_data;
+        kvm_state->memcrypt_save_reset_vector = sev_es_save_reset_vector;
     }
 
     ret = kvm_arch_init(ms, s);

accel/stubs/kvm-stub.c +5 -0

index 82f118d2df..3aece9b513 100644
@@ -114,6 +114,11 @@ int kvm_memcrypt_encrypt_data(uint8_t *ptr, uint64_t len)
   return 1;
 }
 
+int kvm_memcrypt_save_reset_vector(void *flash_ptr, uint64_t flash_size)
+{
+    return -ENOSYS;
+}
+
 #ifndef CONFIG_USER_ONLY
 int kvm_irqchip_add_msi_route(KVMState *s, int vector, PCIDevice *dev)
 {

hw/i386/pc_sysfw.c +9 -1

index b6c0822fe3..321ff94261 100644
@@ -156,7 +156,8 @@ static void pc_system_flash_map(PCMachineState *pcms,
     PFlashCFI01 *system_flash;
     MemoryRegion *flash_mem;
     void *flash_ptr;
-    int ret, flash_size;
+    uint64_t flash_size;
+    int ret;
 
     assert(PC_MACHINE_GET_CLASS(pcms)->pci_enabled);
 

@@ -204,6 +205,13 @@ static void pc_system_flash_map(PCMachineState *pcms,
             if (kvm_memcrypt_enabled()) {
                 flash_ptr = memory_region_get_ram_ptr(flash_mem);
                 flash_size = memory_region_size(flash_mem);
+
+                ret = kvm_memcrypt_save_reset_vector(flash_ptr, flash_size);
+                if (ret) {
+                    error_report("failed to locate and/or save reset vector");
+                    exit(1);
+                }
+
                 ret = kvm_memcrypt_encrypt_data(flash_ptr, flash_size);
                 if (ret) {
                     error_report("failed to encrypt pflash rom");

include/sysemu/kvm.h +16 -0

index b4174d941c..f74cfa85ab 100644
@@ -247,6 +247,22 @@ bool kvm_memcrypt_enabled(void);
  */
 int kvm_memcrypt_encrypt_data(uint8_t *ptr, uint64_t len);
 
+/**
+ * kvm_memcrypt_set_reset_vector - sets the CS/IP value for the AP if SEV-ES
+ *                                 is active.
+ */
+void kvm_memcrypt_set_reset_vector(CPUState *cpu);
+
+/**
+ * kvm_memcrypt_save_reset_vector - locates and saves the reset vector to be
+ *                                  used as the initial CS/IP value for APs
+ *                                  if SEV-ES is active.
+ *
+ * Return: 1 SEV-ES is active and failed to locate a valid reset vector
+ *         0 SEV-ES is not active or successfully located and saved the
+ *           reset vector address
+ */
+int kvm_memcrypt_save_reset_vector(void *flash_prt, uint64_t flash_size);
 
 #ifdef NEED_CPU_H
 #include "cpu.h"

include/sysemu/sev.h +3 -0

index 98c1ec8d38..5198e5a621 100644
@@ -18,4 +18,7 @@
 
 void *sev_guest_init(const char *id);
 int sev_encrypt_data(void *handle, uint8_t *ptr, uint64_t len);
+int sev_es_save_reset_vector(void *handle, void *flash_ptr,
+                             uint64_t flash_size, uint32_t *addr);
+
 #endif

target/i386/kvm.c +2 -0

index 6f18d940a5..10eaba8943 100644
@@ -1912,6 +1912,8 @@ void kvm_arch_reset_vcpu(X86CPU *cpu)
     }
     /* enabled by default */
     env->poll_control_msr = 1;
+
+    kvm_memcrypt_set_reset_vector(CPU(cpu));
 }
 
 void kvm_arch_do_init_vcpu(X86CPU *cpu)

target/i386/sev.c +3 -0

index 5055b1fe00..6ddefc65fa 100644
@@ -70,6 +70,19 @@ struct SevGuestState {
 #define DEFAULT_GUEST_POLICY    0x1 /* disable debug */
 #define DEFAULT_SEV_DEVICE      "/dev/sev"
 
+/* SEV Information Block GUID = 00f771de-1a7e-4fcb-890e-68c77e2fb44e */
+#define SEV_INFO_BLOCK_GUID \
+    "\xde\x71\xf7\x00\x7e\x1a\xcb\x4f\x89\x0e\x68\xc7\x7e\x2f\xb4\x4e"

target/i386/sev.c +11 -0

 static SevGuestState *sev_guest;
 static Error *sev_mig_blocker;
 

@@ -833,6 +846,44 @@ sev_encrypt_data(void *handle, uint8_t *ptr, uint64_t len)
     return 0;
 }
 
+int
+sev_es_save_reset_vector(void *handle, void *flash_ptr, uint64_t flash_size,
+                         uint32_t *addr)
+{
+    SevInfoBlock *info;
+
+    assert(handle);
+
+    /*
+     * Initialize the address to zero. An address of zero with a successful
+     * return code indicates that SEV-ES is not active.

target/i386/sev.c +17 -0

+     */
+    info = flash_ptr + flash_size - 0x20 - sizeof(*info);
+    if (memcmp(info->guid, SEV_INFO_BLOCK_GUID, 16)) {
+        error_report("SEV information block not found in pflash rom");
+        return 1;
+    }
+
+    if (!info->reset_addr) {
+        error_report("SEV-ES reset address is zero");
+        return 1;
+    }
+
+    *addr = info->reset_addr;
+
+    return 0;
+}
+
 static void
 sev_register_types(void)
 {

Signature · 1 line

2.28.0

Quoted 37 lines · * Tom Lendacky ([email protected]) wrote:

From: Tom Lendacky <[email protected]>

When SEV-ES is enabled, it is not possible modify the guests register state after it has been initially created, encrypted and measured.

Normally, an INIT-SIPI-SIPI request is used to boot the AP. However, the hypervisor cannot emulate this because it cannot update the AP register state. For the very first boot by an AP, the reset vector CS segment value and the EIP value must be programmed before the register has been encrypted and measured.

Signed-off-by Tom Lendacky <[email protected]>

---
  accel/kvm/kvm-all.c    | 64 ++++++++++++++++++++++++++++++++++++++++++
  accel/stubs/kvm-stub.c |  5 ++++
  hw/i386/pc_sysfw.c     | 10 ++++++-
  include/sysemu/kvm.h   | 16 +++++++++++
  include/sysemu/sev.h   |  3 ++
  target/i386/kvm.c      |  2 ++
  target/i386/sev.c      | 51 +++++++++++++++++++++++++++++++++
  7 files changed, 150 insertions(+), 1 deletion(-)

accel/kvm/kvm-all.c +1 -0

index 63ef6af9a1..20725b0368 100644
@@ -39,6 +39,7 @@
  #include "qemu/main-loop.h"
  #include "trace.h"
  #include "hw/irq.h"
+#include "sysemu/kvm.h"
  #include "sysemu/sev.h"
  #include "qapi/visitor.h"
  #include "qapi/qapi-types-common.h"

@@ -120,6 +121,12 @@ struct KVMState
      /* memory encryption */
      void *memcrypt_handle;

accel/kvm/kvm-all.c struct KVMState · line 128

Quoted 6 lines

accel/kvm/kvm-all.c +5 -0

      int (*memcrypt_encrypt_data)(void *handle, uint8_t *ptr, uint64_t len);
+    int (*memcrypt_save_reset_vector)(void *handle, void *flash_ptr,
+                                      uint64_t flash_size, uint32_t *addr);
+
+    unsigned int reset_cs;
+    unsigned int reset_ip;

uint32_t's ?

Quoted 180 lines

accel/kvm/kvm-all.c +58 -0

+    bool reset_valid;
 
      /* For "info mtree -f" to tell if an MR is registered in KVM */
      int nr_as;

@@ -239,6 +246,62 @@ int kvm_memcrypt_encrypt_data(uint8_t *ptr, uint64_t len)
      return 1;
  }
 
+void kvm_memcrypt_set_reset_vector(CPUState *cpu)
+{
+    X86CPU *x86;
+    CPUX86State *env;
+
+    /* Only update if we have valid reset information */
+    if (!kvm_state->reset_valid) {
+        return;
+    }
+
+    /* Do not update the BSP reset state */
+    if (cpu->cpu_index == 0) {
+        return;
+    }
+
+    x86 = X86_CPU(cpu);
+    env = &x86->env;
+
+    cpu_x86_load_seg_cache(env, R_CS, 0xf000, kvm_state->reset_cs, 0xffff,
+                           DESC_P_MASK | DESC_S_MASK | DESC_CS_MASK |
+                           DESC_R_MASK | DESC_A_MASK);
+
+    env->eip = kvm_state->reset_ip;
+}
+
+int kvm_memcrypt_save_reset_vector(void *flash_ptr, uint64_t flash_size)
+{
+    CPUState *cpu;
+    uint32_t addr;
+    int ret;
+
+    if (kvm_memcrypt_enabled() &&
+        kvm_state->memcrypt_save_reset_vector) {
+
+        addr = 0;
+        ret = kvm_state->memcrypt_save_reset_vector(kvm_state->memcrypt_handle,
+                                                    flash_ptr, flash_size,
+                                                    &addr);
+        if (ret) {
+            return ret;
+        }
+
+        if (addr) {
+            kvm_state->reset_cs = addr & 0xffff0000;
+            kvm_state->reset_ip = addr & 0x0000ffff;
+            kvm_state->reset_valid = true;
+
+            CPU_FOREACH(cpu) {
+                kvm_memcrypt_set_reset_vector(cpu);
+            }
+        }
+    }
+
+    return 0;
+}
+
  /* Called with KVMMemoryListener.slots_lock held */
  static KVMSlot *kvm_get_free_slot(KVMMemoryListener *kml)
  {

@@ -2193,6 +2256,7 @@ static int kvm_init(MachineState *ms)
          }
 
          kvm_state->memcrypt_encrypt_data = sev_encrypt_data;
+        kvm_state->memcrypt_save_reset_vector = sev_es_save_reset_vector;
      }
 
      ret = kvm_arch_init(ms, s);

accel/stubs/kvm-stub.c +5 -0

index 82f118d2df..3aece9b513 100644
@@ -114,6 +114,11 @@ int kvm_memcrypt_encrypt_data(uint8_t *ptr, uint64_t len)
    return 1;
  }
 
+int kvm_memcrypt_save_reset_vector(void *flash_ptr, uint64_t flash_size)
+{
+    return -ENOSYS;
+}
+
  #ifndef CONFIG_USER_ONLY
  int kvm_irqchip_add_msi_route(KVMState *s, int vector, PCIDevice *dev)
  {

hw/i386/pc_sysfw.c +9 -1

index b6c0822fe3..321ff94261 100644
@@ -156,7 +156,8 @@ static void pc_system_flash_map(PCMachineState *pcms,
      PFlashCFI01 *system_flash;
      MemoryRegion *flash_mem;
      void *flash_ptr;
-    int ret, flash_size;
+    uint64_t flash_size;
+    int ret;
 
      assert(PC_MACHINE_GET_CLASS(pcms)->pci_enabled);
 

@@ -204,6 +205,13 @@ static void pc_system_flash_map(PCMachineState *pcms,
              if (kvm_memcrypt_enabled()) {
                  flash_ptr = memory_region_get_ram_ptr(flash_mem);
                  flash_size = memory_region_size(flash_mem);
+
+                ret = kvm_memcrypt_save_reset_vector(flash_ptr, flash_size);
+                if (ret) {
+                    error_report("failed to locate and/or save reset vector");
+                    exit(1);
+                }
+
                  ret = kvm_memcrypt_encrypt_data(flash_ptr, flash_size);
                  if (ret) {
                      error_report("failed to encrypt pflash rom");

include/sysemu/kvm.h +16 -0

index b4174d941c..f74cfa85ab 100644
@@ -247,6 +247,22 @@ bool kvm_memcrypt_enabled(void);
   */
  int kvm_memcrypt_encrypt_data(uint8_t *ptr, uint64_t len);
 
+/**
+ * kvm_memcrypt_set_reset_vector - sets the CS/IP value for the AP if SEV-ES
+ *                                 is active.
+ */
+void kvm_memcrypt_set_reset_vector(CPUState *cpu);
+
+/**
+ * kvm_memcrypt_save_reset_vector - locates and saves the reset vector to be
+ *                                  used as the initial CS/IP value for APs
+ *                                  if SEV-ES is active.
+ *
+ * Return: 1 SEV-ES is active and failed to locate a valid reset vector
+ *         0 SEV-ES is not active or successfully located and saved the
+ *           reset vector address
+ */
+int kvm_memcrypt_save_reset_vector(void *flash_prt, uint64_t flash_size);
 
  #ifdef NEED_CPU_H
  #include "cpu.h"

include/sysemu/sev.h +3 -0

index 98c1ec8d38..5198e5a621 100644
@@ -18,4 +18,7 @@
 
  void *sev_guest_init(const char *id);
  int sev_encrypt_data(void *handle, uint8_t *ptr, uint64_t len);
+int sev_es_save_reset_vector(void *handle, void *flash_ptr,
+                             uint64_t flash_size, uint32_t *addr);
+
  #endif

target/i386/kvm.c +2 -0

index 6f18d940a5..10eaba8943 100644
@@ -1912,6 +1912,8 @@ void kvm_arch_reset_vcpu(X86CPU *cpu)
      }
      /* enabled by default */
      env->poll_control_msr = 1;
+
+    kvm_memcrypt_set_reset_vector(CPU(cpu));
  }
 
  void kvm_arch_do_init_vcpu(X86CPU *cpu)

target/i386/sev.c +0 -0

index 5055b1fe00..6ddefc65fa 100644
@@ -70,6 +70,19 @@ struct SevGuestState {
  #define DEFAULT_GUEST_POLICY    0x1 /* disable debug */
  #define DEFAULT_SEV_DEVICE      "/dev/sev"
 

target/i386/sev.c struct SevGuestState { · line 85

Quoted 13 lines

target/i386/sev.c +13 -0

+/* SEV Information Block GUID = 00f771de-1a7e-4fcb-890e-68c77e2fb44e */
+#define SEV_INFO_BLOCK_GUID \
+    "\xde\x71\xf7\x00\x7e\x1a\xcb\x4f\x89\x0e\x68\xc7\x7e\x2f\xb4\x4e"
+
+typedef struct __attribute__((__packed__)) SevInfoBlock {
+    /* SEV-ES Reset Vector Address */
+    uint32_t reset_addr;
+
+    /* SEV Information Block size and GUID */
+    uint16_t size;
+    char guid[16];
+} SevInfoBlock;
+

Is that all signed off and happy from the OVMF guys?

Quoted 23 lines

target/i386/sev.c +15 -0

  static SevGuestState *sev_guest;
  static Error *sev_mig_blocker;
 

@@ -833,6 +846,44 @@ sev_encrypt_data(void *handle, uint8_t *ptr, uint64_t len)
      return 0;
  }
 
+int
+sev_es_save_reset_vector(void *handle, void *flash_ptr, uint64_t flash_size,
+                         uint32_t *addr)
+{
+    SevInfoBlock *info;
+
+    assert(handle);
+
+    /*
+     * Initialize the address to zero. An address of zero with a successful
+     * return code indicates that SEV-ES is not active.
+     */
+    *addr = 0;
+    if (!sev_es_enabled()) {
+        return 0;

target/i386/sev.c sev_encrypt_data(void *handle, uint8_t *ptr, uint64_t len) · line 869

Quoted 6 lines

target/i386/sev.c +6 -0

+    }
+
+    /*
+     * Extract the AP reset vector for SEV-ES guests by locating the SEV GUID.
+     * The SEV GUID is located 32 bytes from the end of the flash. Use this
+     * address to base the SEV information block.

It surprises me a bit it's at a fixed offset.

Dave

Quoted 23 lines

target/i386/sev.c +17 -0

+     */
+    info = flash_ptr + flash_size - 0x20 - sizeof(*info);
+    if (memcmp(info->guid, SEV_INFO_BLOCK_GUID, 16)) {
+        error_report("SEV information block not found in pflash rom");
+        return 1;
+    }
+
+    if (!info->reset_addr) {
+        error_report("SEV-ES reset address is zero");
+        return 1;
+    }
+
+    *addr = info->reset_addr;
+
+    return 0;
+}
+
  static void
  sev_register_types(void)
  {

Signature · 1 line

2.28.0

From: Tom Lendacky <[email protected]>

An SEV-ES guest does not allow register state to be altered once it has been measured. When a SEV-ES guest issues a reboot command, Qemu will reset the vCPU state and resume the guest. This will cause failures under SEV-ES, so prevent that from occurring.

Signed-off-by Tom Lendacky <[email protected]>

---
 accel/kvm/kvm-all.c       | 9 +++++++++
 include/sysemu/cpus.h     | 2 ++
 include/sysemu/hw_accel.h | 5 +++++
 include/sysemu/kvm.h      | 2 ++
 softmmu/cpus.c            | 5 +++++
 softmmu/vl.c              | 5 ++++-
 6 files changed, 27 insertions(+), 1 deletion(-)

accel/kvm/kvm-all.c +0 -0

index 20725b0368..63153b6e53 100644

accel/kvm/kvm-all.c +2 -0

+}
+
 static void do_kvm_cpu_synchronize_state(CPUState *cpu, run_on_cpu_data arg)
 {
     if (!cpu->vcpu_dirty) {

include/sysemu/cpus.h +2 -0

index 3c1da6a018..6d688c757f 100644
@@ -24,6 +24,8 @@ void dump_drift_info(void);
 void qemu_cpu_kick_self(void);
 void qemu_timer_notify_cb(void *opaque, QEMUClockType type);
 
+bool cpu_is_resettable(void);
+
 void cpu_synchronize_all_states(void);
 void cpu_synchronize_all_post_reset(void);
 void cpu_synchronize_all_post_init(void);

include/sysemu/hw_accel.h +5 -0

index e128f8b06b..8b4536e7ae 100644
@@ -17,6 +17,11 @@
 #include "sysemu/hvf.h"
 #include "sysemu/whpx.h"
 
+static inline bool cpu_check_resettable(void)
+{
+    return kvm_enabled() ? kvm_cpu_check_resettable() : true;
+}
+
 static inline void cpu_synchronize_state(CPUState *cpu)
 {
     if (kvm_enabled()) {

include/sysemu/kvm.h +2 -0

index f74cfa85ab..eb94bbbff9 100644
@@ -494,6 +494,8 @@ int kvm_physical_memory_addr_from_host(KVMState *s, void *ram_addr,
 
 #endif /* NEED_CPU_H */
 
+bool kvm_cpu_check_resettable(void);
+
 void kvm_cpu_synchronize_state(CPUState *cpu);
 void kvm_cpu_synchronize_post_reset(CPUState *cpu);
 void kvm_cpu_synchronize_post_init(CPUState *cpu);

softmmu/cpus.c +5 -0

index a802e899ab..32f286643f 100644
@@ -927,6 +927,11 @@ void hw_error(const char *fmt, ...)
     abort();
 }
 
+bool cpu_is_resettable(void)
+{
+    return cpu_check_resettable();
+}
+
 void cpu_synchronize_all_states(void)
 {
     CPUState *cpu;

softmmu/vl.c +4 -1

index 4eb9d1f7fd..422fbb1650 100644
@@ -1475,7 +1475,10 @@ void qemu_system_guest_crashloaded(GuestPanicInformation *info)
 
 void qemu_system_reset_request(ShutdownCause reason)
 {
-    if (no_reboot && reason != SHUTDOWN_CAUSE_SUBSYSTEM_RESET) {
+    if (!cpu_is_resettable()) {
+        error_report("cpus are not resettable, terminating");
+        shutdown_requested = reason;
+    } else if (no_reboot && reason != SHUTDOWN_CAUSE_SUBSYSTEM_RESET) {
         shutdown_requested = reason;
     } else {
         reset_requested = reason;

Signature · 1 line

2.28.0

Quoted 25 lines · * Tom Lendacky ([email protected]) wrote:

From: Tom Lendacky <[email protected]>

An SEV-ES guest does not allow register state to be altered once it has been measured. When a SEV-ES guest issues a reboot command, Qemu will reset the vCPU state and resume the guest. This will cause failures under SEV-ES, so prevent that from occurring.

Signed-off-by Tom Lendacky <[email protected]>

---
  accel/kvm/kvm-all.c       | 9 +++++++++
  include/sysemu/cpus.h     | 2 ++
  include/sysemu/hw_accel.h | 5 +++++
  include/sysemu/kvm.h      | 2 ++
  softmmu/cpus.c            | 5 +++++
  softmmu/vl.c              | 5 ++++-
  6 files changed, 27 insertions(+), 1 deletion(-)

accel/kvm/kvm-all.c +0 -0

index 20725b0368..63153b6e53 100644
@@ -2388,6 +2388,15 @@ void kvm_flush_coalesced_mmio_buffer(void)
      s->coalesced_flush_in_progress = false;
  }
 

accel/kvm/kvm-all.c void kvm_flush_coalesced_mmio_buffer(void) · line 2397

Quoted 7 lines

accel/kvm/kvm-all.c +7 -0

+bool kvm_cpu_check_resettable(void)
+{
+    /*
+     * If we have a valid reset vector override, then SEV-ES is active
+     * and the CPU can't be reset.
+     */
+    return !kvm_state->reset_valid;

This seems a bit weird since it's in generic rather than x86 specific code.

Dave

Quoted 82 lines

accel/kvm/kvm-all.c +2 -0

+}
+
  static void do_kvm_cpu_synchronize_state(CPUState *cpu, run_on_cpu_data arg)
  {
      if (!cpu->vcpu_dirty) {

include/sysemu/cpus.h +2 -0

index 3c1da6a018..6d688c757f 100644
@@ -24,6 +24,8 @@ void dump_drift_info(void);
  void qemu_cpu_kick_self(void);
  void qemu_timer_notify_cb(void *opaque, QEMUClockType type);
 
+bool cpu_is_resettable(void);
+
  void cpu_synchronize_all_states(void);
  void cpu_synchronize_all_post_reset(void);
  void cpu_synchronize_all_post_init(void);

include/sysemu/hw_accel.h +5 -0

index e128f8b06b..8b4536e7ae 100644
@@ -17,6 +17,11 @@
  #include "sysemu/hvf.h"
  #include "sysemu/whpx.h"
 
+static inline bool cpu_check_resettable(void)
+{
+    return kvm_enabled() ? kvm_cpu_check_resettable() : true;
+}
+
  static inline void cpu_synchronize_state(CPUState *cpu)
  {
      if (kvm_enabled()) {

include/sysemu/kvm.h +2 -0

index f74cfa85ab..eb94bbbff9 100644
@@ -494,6 +494,8 @@ int kvm_physical_memory_addr_from_host(KVMState *s, void *ram_addr,
 
  #endif /* NEED_CPU_H */
 
+bool kvm_cpu_check_resettable(void);
+
  void kvm_cpu_synchronize_state(CPUState *cpu);
  void kvm_cpu_synchronize_post_reset(CPUState *cpu);
  void kvm_cpu_synchronize_post_init(CPUState *cpu);

softmmu/cpus.c +5 -0

index a802e899ab..32f286643f 100644
@@ -927,6 +927,11 @@ void hw_error(const char *fmt, ...)
      abort();
  }
 
+bool cpu_is_resettable(void)
+{
+    return cpu_check_resettable();
+}
+
  void cpu_synchronize_all_states(void)
  {
      CPUState *cpu;

softmmu/vl.c +4 -1

index 4eb9d1f7fd..422fbb1650 100644
@@ -1475,7 +1475,10 @@ void qemu_system_guest_crashloaded(GuestPanicInformation *info)
 
  void qemu_system_reset_request(ShutdownCause reason)
  {
-    if (no_reboot && reason != SHUTDOWN_CAUSE_SUBSYSTEM_RESET) {
+    if (!cpu_is_resettable()) {
+        error_report("cpus are not resettable, terminating");
+        shutdown_requested = reason;
+    } else if (no_reboot && reason != SHUTDOWN_CAUSE_SUBSYSTEM_RESET) {
          shutdown_requested = reason;
      } else {
          reset_requested = reason;

Signature · 1 line

2.28.0

Quoted 27 lines · Dr. David Alan Gilbert

Quoted 26 lines · * Tom Lendacky ([email protected]) wrote:

From: Tom Lendacky <[email protected]>

An SEV-ES guest does not allow register state to be altered once it has been measured. When a SEV-ES guest issues a reboot command, Qemu will reset the vCPU state and resume the guest. This will cause failures under SEV-ES, so prevent that from occurring.

Signed-off-by Tom Lendacky <[email protected]>

---
  accel/kvm/kvm-all.c       | 9 +++++++++
  include/sysemu/cpus.h     | 2 ++
  include/sysemu/hw_accel.h | 5 +++++
  include/sysemu/kvm.h      | 2 ++
  softmmu/cpus.c            | 5 +++++
  softmmu/vl.c              | 5 ++++-
  6 files changed, 27 insertions(+), 1 deletion(-)

accel/kvm/kvm-all.c +2 -0

index 20725b0368..63153b6e53 100644
@@ -2388,6 +2388,15 @@ void kvm_flush_coalesced_mmio_buffer(void)
      s->coalesced_flush_in_progress = false;
  }
+bool kvm_cpu_check_resettable(void)
+{

Quoted 8 lines

accel/kvm/kvm-all.c void kvm_flush_coalesced_mmio_buffer(void) · line 2397

Quoted 5 lines

accel/kvm/kvm-all.c +5 -0

+    /*
+     * If we have a valid reset vector override, then SEV-ES is active
+     * and the CPU can't be reset.
+     */
+    return !kvm_state->reset_valid;

This seems a bit weird since it's in generic rather than x86 specific code.

I could push it down to arch specific code.

Is there a way to do that
without defining the function for all the other arches?

Thanks,
Tom

Quoted 79 lines

Dave

Quoted 76 lines

accel/kvm/kvm-all.c +2 -0

+}
+
  static void do_kvm_cpu_synchronize_state(CPUState *cpu, run_on_cpu_data arg)
  {
      if (!cpu->vcpu_dirty) {

include/sysemu/cpus.h +2 -0

index 3c1da6a018..6d688c757f 100644
@@ -24,6 +24,8 @@ void dump_drift_info(void);
  void qemu_cpu_kick_self(void);
  void qemu_timer_notify_cb(void *opaque, QEMUClockType type);
+bool cpu_is_resettable(void);
+
  void cpu_synchronize_all_states(void);
  void cpu_synchronize_all_post_reset(void);
  void cpu_synchronize_all_post_init(void);

include/sysemu/hw_accel.h +5 -0

index e128f8b06b..8b4536e7ae 100644
@@ -17,6 +17,11 @@
  #include "sysemu/hvf.h"
  #include "sysemu/whpx.h"
+static inline bool cpu_check_resettable(void)
+{
+    return kvm_enabled() ? kvm_cpu_check_resettable() : true;
+}
+
  static inline void cpu_synchronize_state(CPUState *cpu)
  {
      if (kvm_enabled()) {

include/sysemu/kvm.h +2 -0

index f74cfa85ab..eb94bbbff9 100644
@@ -494,6 +494,8 @@ int kvm_physical_memory_addr_from_host(KVMState *s, void *ram_addr,
  #endif /* NEED_CPU_H */
+bool kvm_cpu_check_resettable(void);
+
  void kvm_cpu_synchronize_state(CPUState *cpu);
  void kvm_cpu_synchronize_post_reset(CPUState *cpu);
  void kvm_cpu_synchronize_post_init(CPUState *cpu);

softmmu/cpus.c +5 -0

index a802e899ab..32f286643f 100644
@@ -927,6 +927,11 @@ void hw_error(const char *fmt, ...)
      abort();
  }
+bool cpu_is_resettable(void)
+{
+    return cpu_check_resettable();
+}
+
  void cpu_synchronize_all_states(void)
  {
      CPUState *cpu;

softmmu/vl.c +4 -1

index 4eb9d1f7fd..422fbb1650 100644
@@ -1475,7 +1475,10 @@ void qemu_system_guest_crashloaded(GuestPanicInformation *info)
  void qemu_system_reset_request(ShutdownCause reason)
  {
-    if (no_reboot && reason != SHUTDOWN_CAUSE_SUBSYSTEM_RESET) {
+    if (!cpu_is_resettable()) {
+        error_report("cpus are not resettable, terminating");
+        shutdown_requested = reason;
+    } else if (no_reboot && reason != SHUTDOWN_CAUSE_SUBSYSTEM_RESET) {
          shutdown_requested = reason;
      } else {
          reset_requested = reason;

Signature · 1 line

2.28.0

From: Tom Lendacky <[email protected]>

Update the sev_es_enabled() function return value to be based on the SEV policy that has been specified. SEV-ES is enabled if SEV is enabled and the SEV-ES policy bit is set in the policy object.

Signed-off-by Tom Lendacky <[email protected]>

---
 target/i386/sev.c | 4 +++-
 1 file changed, 3 insertions(+), 1 deletion(-)

target/i386/sev.c +0 -0

index 6ddefc65fa..bcaadaa2f9 100644

target/i386/sev.c +1 -1

 /* SEV Information Block GUID = 00f771de-1a7e-4fcb-890e-68c77e2fb44e */
 #define SEV_INFO_BLOCK_GUID \
     "\xde\x71\xf7\x00\x7e\x1a\xcb\x4f\x89\x0e\x68\xc7\x7e\x2f\xb4\x4e"

@@ -375,7 +377,7 @@ sev_enabled(void)
 bool
 sev_es_enabled(void)
 {
-    return false;
+    return sev_enabled() && (sev_guest->policy & GUEST_POLICY_SEV_ES_BIT);
 }
 
 uint64_t

Signature · 1 line

2.28.0

Quoted 19 lines · * Tom Lendacky ([email protected]) wrote:

From: Tom Lendacky <[email protected]>

Update the sev_es_enabled() function return value to be based on the SEV policy that has been specified. SEV-ES is enabled if SEV is enabled and the SEV-ES policy bit is set in the policy object.

Signed-off-by Tom Lendacky <[email protected]>

---
  target/i386/sev.c | 4 +++-
  1 file changed, 3 insertions(+), 1 deletion(-)

target/i386/sev.c +0 -0

index 6ddefc65fa..bcaadaa2f9 100644
@@ -70,6 +70,8 @@ struct SevGuestState {
  #define DEFAULT_GUEST_POLICY    0x1 /* disable debug */
  #define DEFAULT_SEV_DEVICE      "/dev/sev"
 

target/i386/sev.c struct SevGuestState { · line 74

Quoted 2 lines

target/i386/sev.c +2 -0

+#define GUEST_POLICY_SEV_ES_BIT (1 << 2)
+

I'm surprised that all the policy bits aren't defined in a header somewhere.

But other than that,

Reviewed-by Dr. David Alan Gilbert <[email protected]>

Quoted 15 lines

target/i386/sev.c +1 -1

  /* SEV Information Block GUID = 00f771de-1a7e-4fcb-890e-68c77e2fb44e */
  #define SEV_INFO_BLOCK_GUID \
      "\xde\x71\xf7\x00\x7e\x1a\xcb\x4f\x89\x0e\x68\xc7\x7e\x2f\xb4\x4e"

@@ -375,7 +377,7 @@ sev_enabled(void)
  bool
  sev_es_enabled(void)
  {
-    return false;
+    return sev_enabled() && (sev_guest->policy & GUEST_POLICY_SEV_ES_BIT);
  }
 
  uint64_t

Signature · 1 line

2.28.0

Quoted 12 lines · Dr. David Alan Gilbert

Quoted 11 lines · * Tom Lendacky ([email protected]) wrote:

From: Tom Lendacky <[email protected]>

Update the sev_es_enabled() function return value to be based on the SEV policy that has been specified. SEV-ES is enabled if SEV is enabled and the SEV-ES policy bit is set in the policy object.

Signed-off-by Tom Lendacky <[email protected]>

---
  target/i386/sev.c | 4 +++-

  1 file changed, 3 insertions(+), 1 deletion(-)

Quoted 11 lines

target/i386/sev.c struct SevGuestState { · line 73

Quoted 9 lines

target/i386/sev.c +2 -0

index 6ddefc65fa..bcaadaa2f9 100644
@@ -70,6 +70,8 @@ struct SevGuestState {
   #define DEFAULT_GUEST_POLICY    0x1 /* disable debug */
   #define DEFAULT_SEV_DEVICE      "/dev/sev"
+#define GUEST_POLICY_SEV_ES_BIT (1 << 2)
+

I'm surprised that all the policy bits aren't defined in a header somewhere.

I have another version to be issued with changes to use QemuUUID, so I can look at moving the bits to a header.

Thanks,
Tom

Quoted 21 lines

But other than that,

Reviewed-by Dr. David Alan Gilbert <[email protected]>

Quoted 15 lines

target/i386/sev.c +0 -0

   /* SEV Information Block GUID = 00f771de-1a7e-4fcb-890e-68c77e2fb44e */
   #define SEV_INFO_BLOCK_GUID \

      "\xde\x71\xf7\x00\x7e\x1a\xcb\x4f\x89\x0e\x68\xc7\x7e\x2f\xb4\x4e"

target/i386/sev.c +0 -0

@@ -375,7 +377,7 @@ sev_enabled(void)

  bool
  sev_es_enabled(void)
  {
-    return false;
+    return sev_enabled() && (sev_guest->policy &
GUEST_POLICY_SEV_ES_BIT);
  }
  uint64_t

Signature · 1 line

2.28.0

From: Tom Lendacky <[email protected]>

This patch series provides support for launching an SEV-ES guest.

Secure Encrypted Virtualization - Encrypted State (SEV-ES) expands on the SEV support to protect the guest register state from the hypervisor. See "AMD64 Architecture Programmer's Manual Volume 2: System Programming", section "15.35 Encrypted State (SEV-ES)" [1].

In order to allow a hypervisor to perform functions on behalf of a guest, there is architectural support for notifying a guest's operating system when certain types of VMEXITs are about to occur. This allows the guest to selectively share information with the hypervisor to satisfy the requested function. The notification is performed using a new exception, the VMM Communication exception (#VC). The information is shared through the Guest-Hypervisor Communication Block (GHCB) using the VMGEXIT instruction. The GHCB format and the protocol for using it is documented in "SEV-ES Guest-Hypervisor Communication Block Standardization" [2].

The main areas of the Qemu code that are updated to support SEV-ES are around the SEV guest launch process and AP booting in order to support booting multiple vCPUs.

There are no new command line switches required. Instead, the desire for SEV-ES is presented using the SEV policy object. Bit 2 of the SEV policy object indicates that SEV-ES is required.

The SEV launch process is updated in two ways. The first is that a the KVM_SEV_ES_INIT ioctl is used to initialize the guest instead of the standard KVM_SEV_INIT ioctl. The second is that before the SEV launch measurement is calculated, the LAUNCH_UPDATE_VMSA SEV API is invoked for each vCPU that Qemu has created. Once the LAUNCH_UPDATE_VMSA API has been invoked, no direct changes to the guest register state can be made.

AP booting poses some interesting challenges. The INIT-SIPI-SIPI sequence is typically used to boot the APs. However, the hypervisor is not allowed to update the guest registers. For the APs, the reset vector must be known in advance. An OVMF method to provide a known reset vector address exists by providing an SEV information block, identified by UUID, near the end of the firmware [3]. OVMF will program the jump to the actual reset vector in this area of memory. Since the memory location is known in advance, an AP can be created with the known reset vector address as its starting CS:IP. The GHCB document [2] talks about how SMP booting under SEV-ES is performed. SEV-ES also requires the use of the in-kernel irqchip support in order to minimize the changes required to Qemu to support AP booting.

[1] https://www.amd.com/system/files/TechDocs/24593.pdf
[2] https://developer.amd.com/wp-content/resources/56421.pdf
[3] 30937f2f98c4 ("OvmfPkg: Use the SEV-ES work area for the SEV-ES AP reset vector")

    https://github.com/tianocore/edk2/commit/30937f2f98c42496f2f143fe8374ae7f7e684847

---

These patches are based on commit:
d0ed6a69d3 ("Update version for v5.1.0 release")

(I tried basing on the latest Qemu commit, but I was having build issues that level)

A version of the tree can be found at:
https://github.com/AMDESE/qemu/tree/sev-es-v11

Changes since v2:
- Add in-kernel irqchip requirement for SEV-ES guests

Changes since v1:
- Fixed checkpatch.pl errors/warnings

Tom Lendacky (5):
sev/i386: Add initial support for SEV-ES
sev/i386: Require in-kernel irqchip support for SEV-ES guests sev/i386: Allow AP booting under SEV-ES
sev/i386: Don't allow a system reset under an SEV-ES guest sev/i386: Enable an SEV-ES guest based on SEV policy

 accel/kvm/kvm-all.c       |  73 ++++++++++++++++++++++++++
 accel/stubs/kvm-stub.c    |   5 ++
 hw/i386/pc_sysfw.c        |  10 +++-
 include/sysemu/cpus.h     |   2 +
 include/sysemu/hw_accel.h |   5 ++
 include/sysemu/kvm.h      |  18 +++++++
 include/sysemu/sev.h      |   3 ++
 softmmu/cpus.c            |   5 ++
 softmmu/vl.c              |   5 +-
 target/i386/cpu.c         |   1 +
 target/i386/kvm.c         |   2 +
 target/i386/sev-stub.c    |   5 ++
 target/i386/sev.c         | 105 +++++++++++++++++++++++++++++++++++++-
 target/i386/sev_i386.h    |   1 +
 14 files changed, 236 insertions(+), 4 deletions(-)

Signature · 1 line

2.28.0

Quoted 34 lines · * Tom Lendacky ([email protected]) wrote:

From: Tom Lendacky <[email protected]>

This patch series provides support for launching an SEV-ES guest.

Secure Encrypted Virtualization - Encrypted State (SEV-ES) expands on the SEV support to protect the guest register state from the hypervisor. See "AMD64 Architecture Programmer's Manual Volume 2: System Programming", section "15.35 Encrypted State (SEV-ES)" [1].

In order to allow a hypervisor to perform functions on behalf of a guest, there is architectural support for notifying a guest's operating system when certain types of VMEXITs are about to occur. This allows the guest to selectively share information with the hypervisor to satisfy the requested function. The notification is performed using a new exception, the VMM Communication exception (#VC). The information is shared through the Guest-Hypervisor Communication Block (GHCB) using the VMGEXIT instruction. The GHCB format and the protocol for using it is documented in "SEV-ES Guest-Hypervisor Communication Block Standardization" [2].

The main areas of the Qemu code that are updated to support SEV-ES are around the SEV guest launch process and AP booting in order to support booting multiple vCPUs.

There are no new command line switches required. Instead, the desire for SEV-ES is presented using the SEV policy object. Bit 2 of the SEV policy object indicates that SEV-ES is required.

The SEV launch process is updated in two ways. The first is that a the KVM_SEV_ES_INIT ioctl is used to initialize the guest instead of the standard KVM_SEV_INIT ioctl. The second is that before the SEV launch measurement is calculated, the LAUNCH_UPDATE_VMSA SEV API is invoked for each vCPU that Qemu has created. Once the LAUNCH_UPDATE_VMSA API has been invoked, no direct changes to the guest register state can be made.

Quoted 11 lines

AP booting poses some interesting challenges. The INIT-SIPI-SIPI sequence is typically used to boot the APs. However, the hypervisor is not allowed to update the guest registers. For the APs, the reset vector must be known in advance. An OVMF method to provide a known reset vector address exists by providing an SEV information block, identified by UUID, near the end of the firmware [3]. OVMF will program the jump to the actual reset vector in this area of memory. Since the memory location is known in advance, an AP can be created with the known reset vector address as its starting CS:IP. The GHCB document [2] talks about how SMP booting under SEV-ES is performed. SEV-ES also requires the use of the in-kernel irqchip support in order to minimize the changes required to Qemu to support AP booting.

Some random thoughts:
a) Is there something that explicitly disallows SMM?

b) I think all the interfaces you're using are already defined in
Linux header files - even if the code to implement them isn't actually upstream in the kernel yet (the launch_update in particular) - we normally wait for the kernel interface to be accepted before taking the QEMU patches, but if the constants are in the headers already I'm not sure what the rule is.

c) What happens if QEMU reads the register values from the state if
the guest is paused - does it just see junk? I'm just wondering if you need to add checks in places it might try to.

Dave

Quoted 48 lines

[1] https://nam11.safelinks.protection.outlook.com/?url=https%3A%2F%2Fwww.amd.com%2Fsystem%2Ffiles%2FTechDocs%2F24593.pdf&amp;data=02%7C01%7Cthomas.lendacky%40amd.com%7Cb07b788e09054a91143308d85b2f1a89%7C3dd8961fe4884e608e11a82d994e183d%7C0%7C0%7C637359606292398926&amp;sdata=B2naGIEXuhD7a%2Fi4NDsRzeHwvDvNJ%2FP7nf5HmAzk9CU%3D&amp;reserved=0
[2] https://nam11.safelinks.protection.outlook.com/?url=https%3A%2F%2Fdeveloper.amd.com%2Fwp-content%2Fresources%2F56421.pdf&amp;data=02%7C01%7Cthomas.lendacky%40amd.com%7Cb07b788e09054a91143308d85b2f1a89%7C3dd8961fe4884e608e11a82d994e183d%7C0%7C0%7C637359606292398926&amp;sdata=0HrHZxdTEK%2FWM1KxxasMAghpzTNGvuKKSlg6nBgPjJY%3D&amp;reserved=0
[3] 30937f2f98c4 ("OvmfPkg: Use the SEV-ES work area for the SEV-ES AP reset vector")

     https://nam11.safelinks.protection.outlook.com/?url=https%3A%2F%2Fgithub.com%2Ftianocore%2Fedk2%2Fcommit%2F30937f2f98c42496f2f143fe8374ae7f7e684847&data=02%7C01%7Cthomas.lendacky%40amd.com%7Cb07b788e09054a91143308d85b2f1a89%7C3dd8961fe4884e608e11a82d994e183d%7C0%7C0%7C637359606292408916&sdata=ISAjIahZH4izDHnXgdWDX0GK61kwgtTw%2BEE%2BS8FBls0%3D&reserved=0

---

These patches are based on commit:
d0ed6a69d3 ("Update version for v5.1.0 release")

(I tried basing on the latest Qemu commit, but I was having build issues that level)

A version of the tree can be found at:
https://nam11.safelinks.protection.outlook.com/?url=https%3A%2F%2Fgithub.com%2FAMDESE%2Fqemu%2Ftree%2Fsev-es-v11&amp;data=02%7C01%7Cthomas.lendacky%40amd.com%7Cb07b788e09054a91143308d85b2f1a89%7C3dd8961fe4884e608e11a82d994e183d%7C0%7C0%7C637359606292408916&amp;sdata=pWd8HAZkAILIMRb1i5TNz9XoHyrhCgRu%2Bq%2BXN2NJ4ag%3D&amp;reserved=0

Changes since v2:
- Add in-kernel irqchip requirement for SEV-ES guests

Changes since v1:
- Fixed checkpatch.pl errors/warnings

Tom Lendacky (5):
sev/i386: Add initial support for SEV-ES
sev/i386: Require in-kernel irqchip support for SEV-ES guests sev/i386: Allow AP booting under SEV-ES
sev/i386: Don't allow a system reset under an SEV-ES guest sev/i386: Enable an SEV-ES guest based on SEV policy

  accel/kvm/kvm-all.c       |  73 ++++++++++++++++++++++++++
  accel/stubs/kvm-stub.c    |   5 ++
  hw/i386/pc_sysfw.c        |  10 +++-
  include/sysemu/cpus.h     |   2 +
  include/sysemu/hw_accel.h |   5 ++
  include/sysemu/kvm.h      |  18 +++++++
  include/sysemu/sev.h      |   3 ++
  softmmu/cpus.c            |   5 ++
  softmmu/vl.c              |   5 +-
  target/i386/cpu.c         |   1 +
  target/i386/kvm.c         |   2 +
  target/i386/sev-stub.c    |   5 ++
  target/i386/sev.c         | 105 +++++++++++++++++++++++++++++++++++++-
  target/i386/sev_i386.h    |   1 +
  14 files changed, 236 insertions(+), 4 deletions(-)

Signature · 1 line

2.28.0

Quoted 47 lines · Dr. David Alan Gilbert

Quoted 45 lines · * Tom Lendacky ([email protected]) wrote:

From: Tom Lendacky <[email protected]>

This patch series provides support for launching an SEV-ES guest.

Secure Encrypted Virtualization - Encrypted State (SEV-ES) expands on the SEV support to protect the guest register state from the hypervisor. See "AMD64 Architecture Programmer's Manual Volume 2: System Programming", section "15.35 Encrypted State (SEV-ES)" [1].

In order to allow a hypervisor to perform functions on behalf of a guest, there is architectural support for notifying a guest's operating system when certain types of VMEXITs are about to occur. This allows the guest to selectively share information with the hypervisor to satisfy the requested function. The notification is performed using a new exception, the VMM Communication exception (#VC). The information is shared through the Guest-Hypervisor Communication Block (GHCB) using the VMGEXIT instruction. The GHCB format and the protocol for using it is documented in "SEV-ES Guest-Hypervisor Communication Block Standardization" [2].

The main areas of the Qemu code that are updated to support SEV-ES are around the SEV guest launch process and AP booting in order to support booting multiple vCPUs.

There are no new command line switches required. Instead, the desire for SEV-ES is presented using the SEV policy object. Bit 2 of the SEV policy object indicates that SEV-ES is required.

The SEV launch process is updated in two ways. The first is that a the KVM_SEV_ES_INIT ioctl is used to initialize the guest instead of the standard KVM_SEV_INIT ioctl. The second is that before the SEV launch measurement is calculated, the LAUNCH_UPDATE_VMSA SEV API is invoked for each vCPU that Qemu has created. Once the LAUNCH_UPDATE_VMSA API has been invoked, no direct changes to the guest register state can be made.

AP booting poses some interesting challenges. The INIT-SIPI-SIPI sequence is typically used to boot the APs. However, the hypervisor is not allowed to update the guest registers. For the APs, the reset vector must be known in advance. An OVMF method to provide a known reset vector address exists by providing an SEV information block, identified by UUID, near the end of the firmware [3]. OVMF will program the jump to the actual reset vector in this area of memory. Since the memory location is known in advance, an AP can be created with the known reset vector address as its starting CS:IP. The GHCB document [2] talks about how SMP booting under SEV-ES is performed. SEV-ES also requires the use of the in-kernel irqchip support in order to minimize the changes required to Qemu to support AP booting.

Quoted 2 lines

Some random thoughts:
a) Is there something that explicitly disallows SMM?

There isn't currently. Is there a way to know early on that SMM is enabled? Could I just call x86_machine_is_smm_enabled() to check that?

Quoted 6 lines

b) I think all the interfaces you're using are already defined in
Linux header files - even if the code to implement them isn't actually upstream in the kernel yet (the launch_update in particular) - we normally wait for the kernel interface to be accepted before taking the QEMU patches, but if the constants are in the headers already I'm not sure what the rule is.

Correct, everything was already present from a Linux header perspective.

Quoted 3 lines

c) What happens if QEMU reads the register values from the state if
the guest is paused - does it just see junk? I'm just wondering if you need to add checks in places it might try to.

I thought about what to do about calls to read the registers once the guest state has become encrypted. I think it would take a lot of changes to make Qemu "protected state aware" for what I see as little gain. Qemu is likely to see a lot of zeroes or actual register values from the GHCB protocol for previous VMGEXITs that took place.

Quoted 48 lines · Tom Lendacky

Quoted 47 lines · Dr. David Alan Gilbert

Quoted 45 lines · * Tom Lendacky ([email protected]) wrote:

From: Tom Lendacky <[email protected]>

This patch series provides support for launching an SEV-ES guest.

Secure Encrypted Virtualization - Encrypted State (SEV-ES) expands on the SEV support to protect the guest register state from the hypervisor. See "AMD64 Architecture Programmer's Manual Volume 2: System Programming", section "15.35 Encrypted State (SEV-ES)" [1].

In order to allow a hypervisor to perform functions on behalf of a guest, there is architectural support for notifying a guest's operating system when certain types of VMEXITs are about to occur. This allows the guest to selectively share information with the hypervisor to satisfy the requested function. The notification is performed using a new exception, the VMM Communication exception (#VC). The information is shared through the Guest-Hypervisor Communication Block (GHCB) using the VMGEXIT instruction. The GHCB format and the protocol for using it is documented in "SEV-ES Guest-Hypervisor Communication Block Standardization" [2].

The main areas of the Qemu code that are updated to support SEV-ES are around the SEV guest launch process and AP booting in order to support booting multiple vCPUs.

There are no new command line switches required. Instead, the desire for SEV-ES is presented using the SEV policy object. Bit 2 of the SEV policy object indicates that SEV-ES is required.

The SEV launch process is updated in two ways. The first is that a the KVM_SEV_ES_INIT ioctl is used to initialize the guest instead of the standard KVM_SEV_INIT ioctl. The second is that before the SEV launch measurement is calculated, the LAUNCH_UPDATE_VMSA SEV API is invoked for each vCPU that Qemu has created. Once the LAUNCH_UPDATE_VMSA API has been invoked, no direct changes to the guest register state can be made.

AP booting poses some interesting challenges. The INIT-SIPI-SIPI sequence is typically used to boot the APs. However, the hypervisor is not allowed to update the guest registers. For the APs, the reset vector must be known in advance. An OVMF method to provide a known reset vector address exists by providing an SEV information block, identified by UUID, near the end of the firmware [3]. OVMF will program the jump to the actual reset vector in this area of memory. Since the memory location is known in advance, an AP can be created with the known reset vector address as its starting CS:IP. The GHCB document [2] talks about how SMP booting under SEV-ES is performed. SEV-ES also requires the use of the in-kernel irqchip support in order to minimize the changes required to Qemu to support AP booting.

Quoted 5 lines

Quoted 2 lines

Some random thoughts:
a) Is there something that explicitly disallows SMM?

There isn't currently. Is there a way to know early on that SMM is enabled? Could I just call x86_machine_is_smm_enabled() to check that?

KVM_CAP_X86_SMM is currently checked as a KVM-wide capability. One option is to change that to use a per-VM ioctl() and then have KVM return 0 for SEV-ES VMs.

target/i386/kvm.c +0 -0

index 416c82048a..4d7f84ed1b 100644

target/i386/kvm.c +0 -0

  bool kvm_has_adjust_clock_stable(void)
 

Quoted 13 lines

Quoted 6 lines

b) I think all the interfaces you're using are already defined in
Linux header files - even if the code to implement them isn't actually upstream in the kernel yet (the launch_update in particular) - we normally wait for the kernel interface to be accepted before taking the QEMU patches, but if the constants are in the headers already I'm not sure what the rule is.

Correct, everything was already present from a Linux header perspective.

Quoted 3 lines

c) What happens if QEMU reads the register values from the state if
the guest is paused - does it just see junk? I'm just wondering if you need to add checks in places it might try to.

Quoted 5 lines

I thought about what to do about calls to read the registers once the guest state has become encrypted. I think it would take a lot of changes to make Qemu "protected state aware" for what I see as little gain. Qemu is likely to see a lot of zeroes or actual register values from the GHCB protocol for previous VMGEXITs that took place.

Yeah, we more or less came to the same conclusion for TDX. It's easy enough to throw an error if QEMU attempts to read protected state, but without other invasive changes, it's too easy to unintentionally kill the VM. MSRs are a bit of a special case, but for REGS, SREGS, and whatever other state is read out, simply letting KVM return zeros/garbage seems like the lesser of all evils.

Quoted 47 lines · Dr. David Alan Gilbert

Quoted 34 lines · * Tom Lendacky ([email protected]) wrote:

From: Tom Lendacky <[email protected]>

This patch series provides support for launching an SEV-ES guest.

Secure Encrypted Virtualization - Encrypted State (SEV-ES) expands on the SEV support to protect the guest register state from the hypervisor. See "AMD64 Architecture Programmer's Manual Volume 2: System Programming", section "15.35 Encrypted State (SEV-ES)" [1].

In order to allow a hypervisor to perform functions on behalf of a guest, there is architectural support for notifying a guest's operating system when certain types of VMEXITs are about to occur. This allows the guest to selectively share information with the hypervisor to satisfy the requested function. The notification is performed using a new exception, the VMM Communication exception (#VC). The information is shared through the Guest-Hypervisor Communication Block (GHCB) using the VMGEXIT instruction. The GHCB format and the protocol for using it is documented in "SEV-ES Guest-Hypervisor Communication Block Standardization" [2].

The main areas of the Qemu code that are updated to support SEV-ES are around the SEV guest launch process and AP booting in order to support booting multiple vCPUs.

There are no new command line switches required. Instead, the desire for SEV-ES is presented using the SEV policy object. Bit 2 of the SEV policy object indicates that SEV-ES is required.

The SEV launch process is updated in two ways. The first is that a the KVM_SEV_ES_INIT ioctl is used to initialize the guest instead of the standard KVM_SEV_INIT ioctl. The second is that before the SEV launch measurement is calculated, the LAUNCH_UPDATE_VMSA SEV API is invoked for each vCPU that Qemu has created. Once the LAUNCH_UPDATE_VMSA API has been invoked, no direct changes to the guest register state can be made.

Quoted 11 lines

AP booting poses some interesting challenges. The INIT-SIPI-SIPI sequence is typically used to boot the APs. However, the hypervisor is not allowed to update the guest registers. For the APs, the reset vector must be known in advance. An OVMF method to provide a known reset vector address exists by providing an SEV information block, identified by UUID, near the end of the firmware [3]. OVMF will program the jump to the actual reset vector in this area of memory. Since the memory location is known in advance, an AP can be created with the known reset vector address as its starting CS:IP. The GHCB document [2] talks about how SMP booting under SEV-ES is performed. SEV-ES also requires the use of the in-kernel irqchip support in order to minimize the changes required to Qemu to support AP booting.

Quoted 2 lines

Some random thoughts:
a) Is there something that explicitly disallows SMM?

There isn't currently. Is there a way to know early on that SMM is enabled? Could I just call x86_machine_is_smm_enabled() to check that?

Quoted 6 lines

b) I think all the interfaces you're using are already defined in
Linux header files - even if the code to implement them isn't actually upstream in the kernel yet (the launch_update in particular) - we normally wait for the kernel interface to be accepted before taking the QEMU patches, but if the constants are in the headers already I'm not sure what the rule is.

Correct, everything was already present from a Linux header perspective.

Quoted 3 lines

c) What happens if QEMU reads the register values from the state if
the guest is paused - does it just see junk? I'm just wondering if you need to add checks in places it might try to.

I thought about what to do about calls to read the registers once the guest state has become encrypted. I think it would take a lot of changes to make Qemu "protected state aware" for what I see as little gain. Qemu is likely to see a lot of zeroes or actual register values from the GHCB protocol for previous VMGEXITs that took place.

Thanks,
Tom

Quoted 51 lines

Dave

Quoted 48 lines

[1] https://nam11.safelinks.protection.outlook.com/?url=https%3A%2F%2Fwww.amd.com%2Fsystem%2Ffiles%2FTechDocs%2F24593.pdf&amp;data=02%7C01%7Cthomas.lendacky%40amd.com%7Cb07b788e09054a91143308d85b2f1a89%7C3dd8961fe4884e608e11a82d994e183d%7C0%7C0%7C637359606292398926&amp;sdata=B2naGIEXuhD7a%2Fi4NDsRzeHwvDvNJ%2FP7nf5HmAzk9CU%3D&amp;reserved=0
[2] https://nam11.safelinks.protection.outlook.com/?url=https%3A%2F%2Fdeveloper.amd.com%2Fwp-content%2Fresources%2F56421.pdf&amp;data=02%7C01%7Cthomas.lendacky%40amd.com%7Cb07b788e09054a91143308d85b2f1a89%7C3dd8961fe4884e608e11a82d994e183d%7C0%7C0%7C637359606292398926&amp;sdata=0HrHZxdTEK%2FWM1KxxasMAghpzTNGvuKKSlg6nBgPjJY%3D&amp;reserved=0
[3] 30937f2f98c4 ("OvmfPkg: Use the SEV-ES work area for the SEV-ES AP reset vector")

     https://nam11.safelinks.protection.outlook.com/?url=https%3A%2F%2Fgithub.com%2Ftianocore%2Fedk2%2Fcommit%2F30937f2f98c42496f2f143fe8374ae7f7e684847&data=02%7C01%7Cthomas.lendacky%40amd.com%7Cb07b788e09054a91143308d85b2f1a89%7C3dd8961fe4884e608e11a82d994e183d%7C0%7C0%7C637359606292408916&sdata=ISAjIahZH4izDHnXgdWDX0GK61kwgtTw%2BEE%2BS8FBls0%3D&reserved=0

---

These patches are based on commit:
d0ed6a69d3 ("Update version for v5.1.0 release")

(I tried basing on the latest Qemu commit, but I was having build issues that level)

A version of the tree can be found at:
https://nam11.safelinks.protection.outlook.com/?url=https%3A%2F%2Fgithub.com%2FAMDESE%2Fqemu%2Ftree%2Fsev-es-v11&amp;data=02%7C01%7Cthomas.lendacky%40amd.com%7Cb07b788e09054a91143308d85b2f1a89%7C3dd8961fe4884e608e11a82d994e183d%7C0%7C0%7C637359606292408916&amp;sdata=pWd8HAZkAILIMRb1i5TNz9XoHyrhCgRu%2Bq%2BXN2NJ4ag%3D&amp;reserved=0

Changes since v2:
- Add in-kernel irqchip requirement for SEV-ES guests

Changes since v1:
- Fixed checkpatch.pl errors/warnings

Tom Lendacky (5):
sev/i386: Add initial support for SEV-ES
sev/i386: Require in-kernel irqchip support for SEV-ES guests sev/i386: Allow AP booting under SEV-ES
sev/i386: Don't allow a system reset under an SEV-ES guest sev/i386: Enable an SEV-ES guest based on SEV policy

  accel/kvm/kvm-all.c       |  73 ++++++++++++++++++++++++++
  accel/stubs/kvm-stub.c    |   5 ++
  hw/i386/pc_sysfw.c        |  10 +++-
  include/sysemu/cpus.h     |   2 +
  include/sysemu/hw_accel.h |   5 ++
  include/sysemu/kvm.h      |  18 +++++++
  include/sysemu/sev.h      |   3 ++
  softmmu/cpus.c            |   5 ++
  softmmu/vl.c              |   5 +-
  target/i386/cpu.c         |   1 +
  target/i386/kvm.c         |   2 +
  target/i386/sev-stub.c    |   5 ++
  target/i386/sev.c         | 105 +++++++++++++++++++++++++++++++++++++-
  target/i386/sev_i386.h    |   1 +
  14 files changed, 236 insertions(+), 4 deletions(-)

Signature · 1 line

2.28.0

Quoted 67 lines · * Tom Lendacky ([email protected]) wrote:

Quoted 47 lines · Dr. David Alan Gilbert

Quoted 34 lines · * Tom Lendacky ([email protected]) wrote:

From: Tom Lendacky <[email protected]>

This patch series provides support for launching an SEV-ES guest.

Secure Encrypted Virtualization - Encrypted State (SEV-ES) expands on the SEV support to protect the guest register state from the hypervisor. See "AMD64 Architecture Programmer's Manual Volume 2: System Programming", section "15.35 Encrypted State (SEV-ES)" [1].

In order to allow a hypervisor to perform functions on behalf of a guest, there is architectural support for notifying a guest's operating system when certain types of VMEXITs are about to occur. This allows the guest to selectively share information with the hypervisor to satisfy the requested function. The notification is performed using a new exception, the VMM Communication exception (#VC). The information is shared through the Guest-Hypervisor Communication Block (GHCB) using the VMGEXIT instruction. The GHCB format and the protocol for using it is documented in "SEV-ES Guest-Hypervisor Communication Block Standardization" [2].

The main areas of the Qemu code that are updated to support SEV-ES are around the SEV guest launch process and AP booting in order to support booting multiple vCPUs.

There are no new command line switches required. Instead, the desire for SEV-ES is presented using the SEV policy object. Bit 2 of the SEV policy object indicates that SEV-ES is required.

The SEV launch process is updated in two ways. The first is that a the KVM_SEV_ES_INIT ioctl is used to initialize the guest instead of the standard KVM_SEV_INIT ioctl. The second is that before the SEV launch measurement is calculated, the LAUNCH_UPDATE_VMSA SEV API is invoked for each vCPU that Qemu has created. Once the LAUNCH_UPDATE_VMSA API has been invoked, no direct changes to the guest register state can be made.

Quoted 11 lines

AP booting poses some interesting challenges. The INIT-SIPI-SIPI sequence is typically used to boot the APs. However, the hypervisor is not allowed to update the guest registers. For the APs, the reset vector must be known in advance. An OVMF method to provide a known reset vector address exists by providing an SEV information block, identified by UUID, near the end of the firmware [3]. OVMF will program the jump to the actual reset vector in this area of memory. Since the memory location is known in advance, an AP can be created with the known reset vector address as its starting CS:IP. The GHCB document [2] talks about how SMP booting under SEV-ES is performed. SEV-ES also requires the use of the in-kernel irqchip support in order to minimize the changes required to Qemu to support AP booting.

Quoted 2 lines

Some random thoughts:
a) Is there something that explicitly disallows SMM?

There isn't currently. Is there a way to know early on that SMM is enabled? Could I just call x86_machine_is_smm_enabled() to check that?

Quoted 6 lines

b) I think all the interfaces you're using are already defined in
Linux header files - even if the code to implement them isn't actually upstream in the kernel yet (the launch_update in particular) - we normally wait for the kernel interface to be accepted before taking the QEMU patches, but if the constants are in the headers already I'm not sure what the rule is.

Correct, everything was already present from a Linux header perspective.

Quoted 3 lines

c) What happens if QEMU reads the register values from the state if
the guest is paused - does it just see junk? I'm just wondering if you need to add checks in places it might try to.

Quoted 5 lines

I thought about what to do about calls to read the registers once the guest state has become encrypted. I think it would take a lot of changes to make Qemu "protected state aware" for what I see as little gain. Qemu is likely to see a lot of zeroes or actual register values from the GHCB protocol for previous VMGEXITs that took place.

Yep, that's fair enough - I was curious if we'll hit anything accidentally still reading it.

How does SEV-ES interact with the 'NODBG' flag of the guest policy - if that's 0, and 'debugging of the guest' is allowed, what can you actually do?

Dave

Quoted 55 lines

Thanks,
Tom

Quoted 51 lines

Dave

Quoted 48 lines

[1] https://nam11.safelinks.protection.outlook.com/?url=https%3A%2F%2Fwww.amd.com%2Fsystem%2Ffiles%2FTechDocs%2F24593.pdf&amp;data=02%7C01%7Cthomas.lendacky%40amd.com%7Cecf88d6f7bd0494d1b0e08d85bb9c19b%7C3dd8961fe4884e608e11a82d994e183d%7C0%7C0%7C637360201727448609&amp;sdata=e6CbpjDDvCUG2q9pk6OSsty0QB5HuhueVAM4t8iygT8%3D&amp;reserved=0
[2] https://nam11.safelinks.protection.outlook.com/?url=https%3A%2F%2Fdeveloper.amd.com%2Fwp-content%2Fresources%2F56421.pdf&amp;data=02%7C01%7Cthomas.lendacky%40amd.com%7Cecf88d6f7bd0494d1b0e08d85bb9c19b%7C3dd8961fe4884e608e11a82d994e183d%7C0%7C0%7C637360201727448609&amp;sdata=%2FUzJB5K%2F8rOMF%2B%2BVPXjg%2BJBLgD4uLW6U82Wvf8pXq%2BA%3D&amp;reserved=0
[3] 30937f2f98c4 ("OvmfPkg: Use the SEV-ES work area for the SEV-ES AP reset vector")

      https://nam11.safelinks.protection.outlook.com/?url=https%3A%2F%2Fgithub.com%2Ftianocore%2Fedk2%2Fcommit%2F30937f2f98c42496f2f143fe8374ae7f7e684847&data=02%7C01%7Cthomas.lendacky%40amd.com%7Cecf88d6f7bd0494d1b0e08d85bb9c19b%7C3dd8961fe4884e608e11a82d994e183d%7C0%7C0%7C637360201727458605&sdata=0FmiYEdIEtDjw1VIGaeeRrto%2FZpvH1esIgE93gXyagM%3D&reserved=0

---

These patches are based on commit:
d0ed6a69d3 ("Update version for v5.1.0 release")

(I tried basing on the latest Qemu commit, but I was having build issues that level)

A version of the tree can be found at:
https://nam11.safelinks.protection.outlook.com/?url=https%3A%2F%2Fgithub.com%2FAMDESE%2Fqemu%2Ftree%2Fsev-es-v11&amp;data=02%7C01%7Cthomas.lendacky%40amd.com%7Cecf88d6f7bd0494d1b0e08d85bb9c19b%7C3dd8961fe4884e608e11a82d994e183d%7C0%7C0%7C637360201727458605&amp;sdata=w1tfrMDgruZBDxNHgKLhpKtQ50Ua%2FMy9IfkSXfne2xg%3D&amp;reserved=0

Changes since v2:
- Add in-kernel irqchip requirement for SEV-ES guests

Changes since v1:
- Fixed checkpatch.pl errors/warnings

Tom Lendacky (5):
sev/i386: Add initial support for SEV-ES
sev/i386: Require in-kernel irqchip support for SEV-ES guests sev/i386: Allow AP booting under SEV-ES
sev/i386: Don't allow a system reset under an SEV-ES guest sev/i386: Enable an SEV-ES guest based on SEV policy

   accel/kvm/kvm-all.c       |  73 ++++++++++++++++++++++++++
   accel/stubs/kvm-stub.c    |   5 ++
   hw/i386/pc_sysfw.c        |  10 +++-
   include/sysemu/cpus.h     |   2 +
   include/sysemu/hw_accel.h |   5 ++
   include/sysemu/kvm.h      |  18 +++++++
   include/sysemu/sev.h      |   3 ++
   softmmu/cpus.c            |   5 ++
   softmmu/vl.c              |   5 +-
   target/i386/cpu.c         |   1 +
   target/i386/kvm.c         |   2 +
   target/i386/sev-stub.c    |   5 ++
   target/i386/sev.c         | 105 +++++++++++++++++++++++++++++++++++++-
   target/i386/sev_i386.h    |   1 +
   14 files changed, 236 insertions(+), 4 deletions(-)

Signature · 1 line

2.28.0

Quoted 74 lines · Dr. David Alan Gilbert

Quoted 67 lines · * Tom Lendacky ([email protected]) wrote:

Quoted 47 lines · Dr. David Alan Gilbert

Quoted 34 lines · * Tom Lendacky ([email protected]) wrote:

From: Tom Lendacky <[email protected]>

This patch series provides support for launching an SEV-ES guest.

Secure Encrypted Virtualization - Encrypted State (SEV-ES) expands on the SEV support to protect the guest register state from the hypervisor. See "AMD64 Architecture Programmer's Manual Volume 2: System Programming", section "15.35 Encrypted State (SEV-ES)" [1].

In order to allow a hypervisor to perform functions on behalf of a guest, there is architectural support for notifying a guest's operating system when certain types of VMEXITs are about to occur. This allows the guest to selectively share information with the hypervisor to satisfy the requested function. The notification is performed using a new exception, the VMM Communication exception (#VC). The information is shared through the Guest-Hypervisor Communication Block (GHCB) using the VMGEXIT instruction. The GHCB format and the protocol for using it is documented in "SEV-ES Guest-Hypervisor Communication Block Standardization" [2].

The main areas of the Qemu code that are updated to support SEV-ES are around the SEV guest launch process and AP booting in order to support booting multiple vCPUs.

There are no new command line switches required. Instead, the desire for SEV-ES is presented using the SEV policy object. Bit 2 of the SEV policy object indicates that SEV-ES is required.

The SEV launch process is updated in two ways. The first is that a the KVM_SEV_ES_INIT ioctl is used to initialize the guest instead of the standard KVM_SEV_INIT ioctl. The second is that before the SEV launch measurement is calculated, the LAUNCH_UPDATE_VMSA SEV API is invoked for each vCPU that Qemu has created. Once the LAUNCH_UPDATE_VMSA API has been invoked, no direct changes to the guest register state can be made.

Quoted 11 lines

AP booting poses some interesting challenges. The INIT-SIPI-SIPI sequence is typically used to boot the APs. However, the hypervisor is not allowed to update the guest registers. For the APs, the reset vector must be known in advance. An OVMF method to provide a known reset vector address exists by providing an SEV information block, identified by UUID, near the end of the firmware [3]. OVMF will program the jump to the actual reset vector in this area of memory. Since the memory location is known in advance, an AP can be created with the known reset vector address as its starting CS:IP. The GHCB document [2] talks about how SMP booting under SEV-ES is performed. SEV-ES also requires the use of the in-kernel irqchip support in order to minimize the changes required to Qemu to support AP booting.

Quoted 2 lines

Some random thoughts:
a) Is there something that explicitly disallows SMM?

There isn't currently. Is there a way to know early on that SMM is enabled? Could I just call x86_machine_is_smm_enabled() to check that?

Quoted 6 lines

b) I think all the interfaces you're using are already defined in
Linux header files - even if the code to implement them isn't actually upstream in the kernel yet (the launch_update in particular) - we normally wait for the kernel interface to be accepted before taking the QEMU patches, but if the constants are in the headers already I'm not sure what the rule is.

Correct, everything was already present from a Linux header perspective.

Quoted 3 lines

c) What happens if QEMU reads the register values from the state if
the guest is paused - does it just see junk? I'm just wondering if you need to add checks in places it might try to.

Quoted 5 lines

I thought about what to do about calls to read the registers once the guest state has become encrypted. I think it would take a lot of changes to make Qemu "protected state aware" for what I see as little gain. Qemu is likely to see a lot of zeroes or actual register values from the GHCB protocol for previous VMGEXITs that took place.

Quoted 6 lines

Yep, that's fair enough - I was curious if we'll hit anything accidentally still reading it.

How does SEV-ES interact with the 'NODBG' flag of the guest policy - if that's 0, and 'debugging of the guest' is allowed, what can you actually do?

The SEV-ES KVM patches will disallow debugging of the guest, or at least setting breakpoints using the debug registers. Gdb can still break in, but you wont get anything reasonable with register dumps and memory dumps.

The NODBG policy bit enables or disables the DBG_DECRYPT and DBG_ENCRYPT APIs. So if the guest has allowed debugging, memory dumps could be done using those APIs (for encrypted pages). Registers are a different story because you simply can't update from the hypervisor side under SEV-ES.

Under SEV you could do actual debugging if the support was developed and in place.

Thanks,
Tom

Quoted 58 lines

Dave

Quoted 55 lines

Thanks,
Tom

Quoted 51 lines

Dave

Quoted 48 lines

[1] https://nam11.safelinks.protection.outlook.com/?url=https%3A%2F%2Fwww.amd.com%2Fsystem%2Ffiles%2FTechDocs%2F24593.pdf&amp;data=02%7C01%7Cthomas.lendacky%40amd.com%7Cecf88d6f7bd0494d1b0e08d85bb9c19b%7C3dd8961fe4884e608e11a82d994e183d%7C0%7C0%7C637360201727448609&amp;sdata=e6CbpjDDvCUG2q9pk6OSsty0QB5HuhueVAM4t8iygT8%3D&amp;reserved=0
[2] https://nam11.safelinks.protection.outlook.com/?url=https%3A%2F%2Fdeveloper.amd.com%2Fwp-content%2Fresources%2F56421.pdf&amp;data=02%7C01%7Cthomas.lendacky%40amd.com%7Cecf88d6f7bd0494d1b0e08d85bb9c19b%7C3dd8961fe4884e608e11a82d994e183d%7C0%7C0%7C637360201727448609&amp;sdata=%2FUzJB5K%2F8rOMF%2B%2BVPXjg%2BJBLgD4uLW6U82Wvf8pXq%2BA%3D&amp;reserved=0
[3] 30937f2f98c4 ("OvmfPkg: Use the SEV-ES work area for the SEV-ES AP reset vector")

      https://nam11.safelinks.protection.outlook.com/?url=https%3A%2F%2Fgithub.com%2Ftianocore%2Fedk2%2Fcommit%2F30937f2f98c42496f2f143fe8374ae7f7e684847&data=02%7C01%7Cthomas.lendacky%40amd.com%7Cecf88d6f7bd0494d1b0e08d85bb9c19b%7C3dd8961fe4884e608e11a82d994e183d%7C0%7C0%7C637360201727458605&sdata=0FmiYEdIEtDjw1VIGaeeRrto%2FZpvH1esIgE93gXyagM%3D&reserved=0

---

These patches are based on commit:
d0ed6a69d3 ("Update version for v5.1.0 release")

(I tried basing on the latest Qemu commit, but I was having build issues that level)

A version of the tree can be found at:
https://nam11.safelinks.protection.outlook.com/?url=https%3A%2F%2Fgithub.com%2FAMDESE%2Fqemu%2Ftree%2Fsev-es-v11&amp;data=02%7C01%7Cthomas.lendacky%40amd.com%7Cecf88d6f7bd0494d1b0e08d85bb9c19b%7C3dd8961fe4884e608e11a82d994e183d%7C0%7C0%7C637360201727458605&amp;sdata=w1tfrMDgruZBDxNHgKLhpKtQ50Ua%2FMy9IfkSXfne2xg%3D&amp;reserved=0

Changes since v2:
- Add in-kernel irqchip requirement for SEV-ES guests

Changes since v1:
- Fixed checkpatch.pl errors/warnings

Tom Lendacky (5):
sev/i386: Add initial support for SEV-ES
sev/i386: Require in-kernel irqchip support for SEV-ES guests sev/i386: Allow AP booting under SEV-ES
sev/i386: Don't allow a system reset under an SEV-ES guest sev/i386: Enable an SEV-ES guest based on SEV policy

   accel/kvm/kvm-all.c       |  73 ++++++++++++++++++++++++++
   accel/stubs/kvm-stub.c    |   5 ++
   hw/i386/pc_sysfw.c        |  10 +++-
   include/sysemu/cpus.h     |   2 +
   include/sysemu/hw_accel.h |   5 ++
   include/sysemu/kvm.h      |  18 +++++++
   include/sysemu/sev.h      |   3 ++
   softmmu/cpus.c            |   5 ++
   softmmu/vl.c              |   5 +-
   target/i386/cpu.c         |   1 +
   target/i386/kvm.c         |   2 +
   target/i386/sev-stub.c    |   5 ++
   target/i386/sev.c         | 105 +++++++++++++++++++++++++++++++++++++-
   target/i386/sev_i386.h    |   1 +
   14 files changed, 236 insertions(+), 4 deletions(-)

Signature · 1 line

2.28.0

Quoted 91 lines · * Tom Lendacky ([email protected]) wrote:

Quoted 74 lines · Dr. David Alan Gilbert

Quoted 67 lines · * Tom Lendacky ([email protected]) wrote:

Quoted 47 lines · Dr. David Alan Gilbert

Quoted 34 lines · * Tom Lendacky ([email protected]) wrote:

From: Tom Lendacky <[email protected]>

This patch series provides support for launching an SEV-ES guest.

Secure Encrypted Virtualization - Encrypted State (SEV-ES) expands on the SEV support to protect the guest register state from the hypervisor. See "AMD64 Architecture Programmer's Manual Volume 2: System Programming", section "15.35 Encrypted State (SEV-ES)" [1].

In order to allow a hypervisor to perform functions on behalf of a guest, there is architectural support for notifying a guest's operating system when certain types of VMEXITs are about to occur. This allows the guest to selectively share information with the hypervisor to satisfy the requested function. The notification is performed using a new exception, the VMM Communication exception (#VC). The information is shared through the Guest-Hypervisor Communication Block (GHCB) using the VMGEXIT instruction. The GHCB format and the protocol for using it is documented in "SEV-ES Guest-Hypervisor Communication Block Standardization" [2].

The main areas of the Qemu code that are updated to support SEV-ES are around the SEV guest launch process and AP booting in order to support booting multiple vCPUs.

There are no new command line switches required. Instead, the desire for SEV-ES is presented using the SEV policy object. Bit 2 of the SEV policy object indicates that SEV-ES is required.

The SEV launch process is updated in two ways. The first is that a the KVM_SEV_ES_INIT ioctl is used to initialize the guest instead of the standard KVM_SEV_INIT ioctl. The second is that before the SEV launch measurement is calculated, the LAUNCH_UPDATE_VMSA SEV API is invoked for each vCPU that Qemu has created. Once the LAUNCH_UPDATE_VMSA API has been invoked, no direct changes to the guest register state can be made.

Quoted 11 lines

AP booting poses some interesting challenges. The INIT-SIPI-SIPI sequence is typically used to boot the APs. However, the hypervisor is not allowed to update the guest registers. For the APs, the reset vector must be known in advance. An OVMF method to provide a known reset vector address exists by providing an SEV information block, identified by UUID, near the end of the firmware [3]. OVMF will program the jump to the actual reset vector in this area of memory. Since the memory location is known in advance, an AP can be created with the known reset vector address as its starting CS:IP. The GHCB document [2] talks about how SMP booting under SEV-ES is performed. SEV-ES also requires the use of the in-kernel irqchip support in order to minimize the changes required to Qemu to support AP booting.

Quoted 2 lines

Some random thoughts:
a) Is there something that explicitly disallows SMM?

There isn't currently. Is there a way to know early on that SMM is enabled? Could I just call x86_machine_is_smm_enabled() to check that?

Quoted 6 lines

b) I think all the interfaces you're using are already defined in
Linux header files - even if the code to implement them isn't actually upstream in the kernel yet (the launch_update in particular) - we normally wait for the kernel interface to be accepted before taking the QEMU patches, but if the constants are in the headers already I'm not sure what the rule is.

Correct, everything was already present from a Linux header perspective.

Quoted 3 lines

c) What happens if QEMU reads the register values from the state if
the guest is paused - does it just see junk? I'm just wondering if you need to add checks in places it might try to.

Quoted 5 lines

I thought about what to do about calls to read the registers once the guest state has become encrypted. I think it would take a lot of changes to make Qemu "protected state aware" for what I see as little gain. Qemu is likely to see a lot of zeroes or actual register values from the GHCB protocol for previous VMGEXITs that took place.

Quoted 6 lines

Yep, that's fair enough - I was curious if we'll hit anything accidentally still reading it.

How does SEV-ES interact with the 'NODBG' flag of the guest policy - if that's 0, and 'debugging of the guest' is allowed, what can you actually do?

The SEV-ES KVM patches will disallow debugging of the guest, or at least setting breakpoints using the debug registers. Gdb can still break in, but you wont get anything reasonable with register dumps and memory dumps.

The NODBG policy bit enables or disables the DBG_DECRYPT and DBG_ENCRYPT APIs. So if the guest has allowed debugging, memory dumps could be done using those APIs (for encrypted pages). Registers are a different story because you simply can't update from the hypervisor side under SEV-ES.

Quoted 2 lines

Under SEV you could do actual debugging if the support was developed and in place.

Thanks for the explanation - it might be interesting to wire the DBG_DECRYPT support up to dump/dump.c for doing full guest memory dumps
- if the policy has it enabled.

Dave

Quoted 62 lines

Thanks,
Tom

Quoted 58 lines

Dave

Quoted 55 lines

Thanks,
Tom

Quoted 51 lines

Dave

Quoted 48 lines

[1] https://nam11.safelinks.protection.outlook.com/?url=https%3A%2F%2Fwww.amd.com%2Fsystem%2Ffiles%2FTechDocs%2F24593.pdf&amp;data=02%7C01%7Cthomas.lendacky%40amd.com%7C8287b3e99fdc41c682fe08d85e244ee0%7C3dd8961fe4884e608e11a82d994e183d%7C0%7C0%7C637362858215616687&amp;sdata=d9n0rRa2AvVS1dmKyBoCF%2BapDhV6UUc8mfGpWD2%2FxMU%3D&amp;reserved=0
[2] https://nam11.safelinks.protection.outlook.com/?url=https%3A%2F%2Fdeveloper.amd.com%2Fwp-content%2Fresources%2F56421.pdf&amp;data=02%7C01%7Cthomas.lendacky%40amd.com%7C8287b3e99fdc41c682fe08d85e244ee0%7C3dd8961fe4884e608e11a82d994e183d%7C0%7C0%7C637362858215616687&amp;sdata=84imjHi1kXbEf2omr1sS%2Brl7gKYeGFGKk%2BeTavyluN4%3D&amp;reserved=0
[3] 30937f2f98c4 ("OvmfPkg: Use the SEV-ES work area for the SEV-ES AP reset vector")

      https://nam11.safelinks.protection.outlook.com/?url=https%3A%2F%2Fgithub.com%2Ftianocore%2Fedk2%2Fcommit%2F30937f2f98c42496f2f143fe8374ae7f7e684847&data=02%7C01%7Cthomas.lendacky%40amd.com%7C8287b3e99fdc41c682fe08d85e244ee0%7C3dd8961fe4884e608e11a82d994e183d%7C0%7C0%7C637362858215616687&sdata=BTWlq%2BqgpjRtUrbEZE1wasyXxs3YFmfKioEvzvWlsYc%3D&reserved=0

---

These patches are based on commit:
d0ed6a69d3 ("Update version for v5.1.0 release")

(I tried basing on the latest Qemu commit, but I was having build issues that level)

A version of the tree can be found at:
https://nam11.safelinks.protection.outlook.com/?url=https%3A%2F%2Fgithub.com%2FAMDESE%2Fqemu%2Ftree%2Fsev-es-v11&amp;data=02%7C01%7Cthomas.lendacky%40amd.com%7C8287b3e99fdc41c682fe08d85e244ee0%7C3dd8961fe4884e608e11a82d994e183d%7C0%7C0%7C637362858215626678&amp;sdata=1o51C3GUrPwodt4yP6ZlkORcrtSfxLUytJtC66OSjSQ%3D&amp;reserved=0

Changes since v2:
- Add in-kernel irqchip requirement for SEV-ES guests

Changes since v1:
- Fixed checkpatch.pl errors/warnings

Tom Lendacky (5):
sev/i386: Add initial support for SEV-ES
sev/i386: Require in-kernel irqchip support for SEV-ES guests sev/i386: Allow AP booting under SEV-ES
sev/i386: Don't allow a system reset under an SEV-ES guest sev/i386: Enable an SEV-ES guest based on SEV policy

   accel/kvm/kvm-all.c       |  73 ++++++++++++++++++++++++++
   accel/stubs/kvm-stub.c    |   5 ++
   hw/i386/pc_sysfw.c        |  10 +++-
   include/sysemu/cpus.h     |   2 +
   include/sysemu/hw_accel.h |   5 ++
   include/sysemu/kvm.h      |  18 +++++++
   include/sysemu/sev.h      |   3 ++
   softmmu/cpus.c            |   5 ++
   softmmu/vl.c              |   5 +-
   target/i386/cpu.c         |   1 +
   target/i386/kvm.c         |   2 +
   target/i386/sev-stub.c    |   5 ++
   target/i386/sev.c         | 105 +++++++++++++++++++++++++++++++++++++-
   target/i386/sev_i386.h    |   1 +
   14 files changed, 236 insertions(+), 4 deletions(-)

Signature · 1 line

2.28.0