Skip to contentSkip to conversation

Enable notify VM exit

PATCH Superseded Chenyi Qiang 2 participants 6 messages Mar 10, 2022 KVM

v1 of 8 Superseded Latest is v8 →

lore
Share & tools
lore
#0 lore raw

cover Enable notify VM exit

Notify VM exit is introduced to mitigate the potential DOS attach from malicious VM. This series is the userspace part to enable this feature through a new KVM capability KVM_CAP_X86_NOTIFY_VMEXIT. The corresponding KVM patch series is available at

https://lore.kernel.org/lkml/[email protected]/

Chenyi Qiang (2):
linux-headers: Sync the linux headers
i386: Add notify VM exit support

 hw/i386/x86.c               | 24 +++++++++++++++
 include/hw/i386/x86.h       |  3 ++
 linux-headers/asm-x86/kvm.h |  4 +++
 linux-headers/linux/kvm.h   | 29 +++++++++++++++----
 target/i386/kvm/kvm.c       | 58 ++++++++++++++++++++++++-------------
 5 files changed, 93 insertions(+), 25 deletions(-)
Signature · 1 line
2.17.1
#1 lore raw

PATCH 1/2 linux-headers: Sync the linux headers

---
 linux-headers/asm-x86/kvm.h |  4 ++++
 linux-headers/linux/kvm.h   | 29 ++++++++++++++++++++++++-----
 2 files changed, 28 insertions(+), 5 deletions(-)
linux-headers/asm-x86/kvm.h +4 -0
index 2da3316bb5..44757bd612 100644
@@ -325,6 +325,7 @@ struct kvm_reinject_control {
 #define KVM_VCPUEVENT_VALID_SHADOW	0x00000004
 #define KVM_VCPUEVENT_VALID_SMM		0x00000008
 #define KVM_VCPUEVENT_VALID_PAYLOAD	0x00000010
+#define KVM_VCPUEVENT_SHUTDOWN		0x00000020
 
 /* Interrupt shadow states */
 #define KVM_X86_SHADOW_INT_MOV_SS	0x01
@@ -452,6 +453,9 @@ struct kvm_sync_regs {
 
 #define KVM_STATE_VMX_PREEMPTION_TIMER_DEADLINE	0x00000001
 
+/* attributes for system fd (group 0) */
+#define KVM_X86_XCOMP_GUEST_SUPP	0
+
 struct kvm_vmx_nested_state_data {
 	__u8 vmcs12[KVM_STATE_NESTED_VMX_VMCS_SIZE];
 	__u8 shadow_vmcs12[KVM_STATE_NESTED_VMX_VMCS_SIZE];
linux-headers/linux/kvm.h +24 -5
index 00af3bc333..9579b10a46 100644
@@ -270,6 +270,7 @@ struct kvm_xen_exit {
 #define KVM_EXIT_X86_BUS_LOCK     33
 #define KVM_EXIT_XEN              34
 #define KVM_EXIT_RISCV_SBI        35
+#define KVM_EXIT_NOTIFY           36
 
 /* For KVM_EXIT_INTERNAL_ERROR */
 /* Emulate instruction failed. */
@@ -487,6 +488,11 @@ struct kvm_run {
 			unsigned long args[6];
 			unsigned long ret[2];
 		} riscv_sbi;
+		/* KVM_EXIT_NOTIFY */
+		struct {
+#define KVM_NOTIFY_CONTEXT_INVALID	(1 << 0)
+			__u32 data;
+		} notify;
 		/* Fix the size of the union. */
 		char padding[256];
 	};
@@ -562,9 +568,12 @@ struct kvm_s390_mem_op {
 	__u32 op;		/* type of operation */
 	__u64 buf;		/* buffer in userspace */
 	union {
-		__u8 ar;	/* the access register number */
+		struct {
+			__u8 ar;	/* the access register number */
+			__u8 key;	/* access key, ignored if flag unset */
+		};
 		__u32 sida_offset; /* offset into the sida */
-		__u8 reserved[32]; /* should be set to 0 */
+		__u8 reserved[32]; /* ignored */
 	};
 };
 /* types for kvm_s390_mem_op->op */
@@ -572,9 +581,12 @@ struct kvm_s390_mem_op {
 #define KVM_S390_MEMOP_LOGICAL_WRITE	1
 #define KVM_S390_MEMOP_SIDA_READ	2
 #define KVM_S390_MEMOP_SIDA_WRITE	3
+#define KVM_S390_MEMOP_ABSOLUTE_READ	4
+#define KVM_S390_MEMOP_ABSOLUTE_WRITE	5
 /* flags for kvm_s390_mem_op->flags */
 #define KVM_S390_MEMOP_F_CHECK_ONLY		(1ULL << 0)
 #define KVM_S390_MEMOP_F_INJECT_EXCEPTION	(1ULL << 1)
+#define KVM_S390_MEMOP_F_SKEY_PROTECTION	(1ULL << 2)
 
 /* for KVM_INTERRUPT */
 struct kvm_interrupt {
@@ -1133,6 +1145,11 @@ struct kvm_ppc_resize_hpt {
 #define KVM_CAP_VM_MOVE_ENC_CONTEXT_FROM 206
 #define KVM_CAP_VM_GPA_BITS 207
 #define KVM_CAP_XSAVE2 208
+#define KVM_CAP_SYS_ATTRIBUTES 209
+#define KVM_CAP_PPC_AIL_MODE_3 210
+#define KVM_CAP_S390_MEM_OP_EXTENSION 211
+#define KVM_CAP_PMU_CAPABILITY 212
+#define KVM_CAP_X86_NOTIFY_VMEXIT 213
 
 #ifdef KVM_CAP_IRQ_ROUTING
 
@@ -1623,9 +1640,6 @@ struct kvm_enc_region {
 #define KVM_S390_NORMAL_RESET	_IO(KVMIO,   0xc3)
 #define KVM_S390_CLEAR_RESET	_IO(KVMIO,   0xc4)
 
-/* Available with KVM_CAP_XSAVE2 */
-#define KVM_GET_XSAVE2		  _IOR(KVMIO,  0xcf, struct kvm_xsave)
-
 struct kvm_s390_pv_sec_parm {
 	__u64 origin;
 	__u64 length;
@@ -1972,6 +1986,8 @@ struct kvm_dirty_gfn {
 #define KVM_BUS_LOCK_DETECTION_OFF             (1 << 0)
 #define KVM_BUS_LOCK_DETECTION_EXIT            (1 << 1)
 
+#define KVM_PMU_CAP_DISABLE                    (1 << 0)
+
 /**
  * struct kvm_stats_header - Header of per vm/vcpu binary statistics data.
  * @flags: Some extra information for header, always 0 for now.
@@ -2047,4 +2063,7 @@ struct kvm_stats_desc {
 
 #define KVM_GET_STATS_FD  _IO(KVMIO,  0xce)
 
+/* Available with KVM_CAP_XSAVE2 */
+#define KVM_GET_XSAVE2		  _IOR(KVMIO,  0xcf, struct kvm_xsave)
+
 #endif /* __LINUX_KVM_H */
Signature · 1 line
2.17.1
#2 lore raw

PATCH 2/2 i386: Add notify VM exit support

In reply to Chenyi Qiang

There are cases that malicious virtual machine can cause CPU stuck (due to event windows don't open up), e.g., infinite loop in microcode when nested #AC (CVE-2015-5307). No event window means no event (NMI, SMI and IRQ) can be delivered. It leads the CPU to be unavailable to host or other VMs. Notify VM exit is introduced to mitigate such kind of attacks, which will generate a VM exit if no event window occurs in VM non-root mode for a specified amount of time (notify window).

A new KVM capability KVM_CAP_X86_NOTIFY_VMEXIT is exposed to user space so that the user can query the capability and set the expected notify window when creating VMs.

If notify VM exit happens with VM_INVALID_CONTEXT, hypervisor should exit to user space with the exit reason KVM_EXIT_NOTIFY to inform the fatal case. Then user space can inject a SHUTDOWN event to the target vcpu. This is implemented by defining a new bit in flags field of kvm_vcpu_event in KVM_SET_VCPU_EVENTS ioctl.

---
 hw/i386/x86.c         | 24 ++++++++++++++++++
 include/hw/i386/x86.h |  3 +++
 target/i386/kvm/kvm.c | 58 ++++++++++++++++++++++++++++---------------
 3 files changed, 65 insertions(+), 20 deletions(-)
hw/i386/x86.c +24 -0
index b84840a1bb..25e6c50b1e 100644
@@ -1309,6 +1309,23 @@ static void machine_set_sgx_epc(Object *obj, Visitor *v, const char *name,
     qapi_free_SgxEPCList(list);
 }
 
+static void x86_machine_get_notify_window(Object *obj, Visitor *v,
+                                const char *name, void *opaque, Error **errp)
+{
+    X86MachineState *x86ms = X86_MACHINE(obj);
+    int32_t notify_window = x86ms->notify_window;
+
+    visit_type_int32(v, name, &notify_window, errp);
+}
+
+static void x86_machine_set_notify_window(Object *obj, Visitor *v,
+                               const char *name, void *opaque, Error **errp)
+{
+    X86MachineState *x86ms = X86_MACHINE(obj);
+
+    visit_type_int32(v, name, &x86ms->notify_window, errp);
+}
+
 static void x86_machine_initfn(Object *obj)
 {
     X86MachineState *x86ms = X86_MACHINE(obj);
@@ -1319,6 +1336,7 @@ static void x86_machine_initfn(Object *obj)
     x86ms->oem_id = g_strndup(ACPI_BUILD_APPNAME6, 6);
     x86ms->oem_table_id = g_strndup(ACPI_BUILD_APPNAME8, 8);
     x86ms->bus_lock_ratelimit = 0;
+    x86ms->notify_window = -1;
 }
 
 static void x86_machine_class_init(ObjectClass *oc, void *data)
@@ -1375,6 +1393,12 @@ static void x86_machine_class_init(ObjectClass *oc, void *data)
         NULL, NULL);
     object_class_property_set_description(oc, "sgx-epc",
         "SGX EPC device");
+
+    object_class_property_add(oc, X86_MACHINE_NOTIFY_WINDOW, "int32_t",
+                                x86_machine_get_notify_window,
+                                x86_machine_set_notify_window, NULL, NULL);
+    object_class_property_set_description(oc, X86_MACHINE_NOTIFY_WINDOW,
+            "Set the notify window required by notify VM exit");
 }
 
 static const TypeInfo x86_machine_info = {
include/hw/i386/x86.h +3 -0
index a145a30370..2a4ca21a94 100644
@@ -82,6 +82,8 @@ struct X86MachineState {
      * which means no limitation on the guest's bus locks.
      */
     uint64_t bus_lock_ratelimit;
+
+    int32_t notify_window;
 };
 
 #define X86_MACHINE_SMM              "smm"
@@ -89,6 +91,7 @@ struct X86MachineState {
 #define X86_MACHINE_OEM_ID           "x-oem-id"
 #define X86_MACHINE_OEM_TABLE_ID     "x-oem-table-id"
 #define X86_MACHINE_BUS_LOCK_RATELIMIT  "bus-lock-ratelimit"
+#define X86_MACHINE_NOTIFY_WINDOW     "notify-window"
 
 #define TYPE_X86_MACHINE   MACHINE_TYPE_NAME("x86")
 OBJECT_DECLARE_TYPE(X86MachineState, X86MachineClass, X86_MACHINE)
target/i386/kvm/kvm.c +38 -20

89 lines not on this page · raw message

Signature · 1 line
2.17.1
#3 lore raw

Re: [PATCH 2/2] i386: Add notify VM exit support

Quoted 54 lines · Chenyi Qiang

There are cases that malicious virtual machine can cause CPU stuck (due to event windows don't open up), e.g., infinite loop in microcode when nested #AC (CVE-2015-5307). No event window means no event (NMI, SMI and IRQ) can be delivered. It leads the CPU to be unavailable to host or other VMs. Notify VM exit is introduced to mitigate such kind of attacks, which will generate a VM exit if no event window occurs in VM non-root mode for a specified amount of time (notify window).

A new KVM capability KVM_CAP_X86_NOTIFY_VMEXIT is exposed to user space so that the user can query the capability and set the expected notify window when creating VMs.

If notify VM exit happens with VM_INVALID_CONTEXT, hypervisor should exit to user space with the exit reason KVM_EXIT_NOTIFY to inform the fatal case. Then user space can inject a SHUTDOWN event to the target vcpu. This is implemented by defining a new bit in flags field of kvm_vcpu_event in KVM_SET_VCPU_EVENTS ioctl.

---
 hw/i386/x86.c         | 24 ++++++++++++++++++
 include/hw/i386/x86.h |  3 +++
 target/i386/kvm/kvm.c | 58 ++++++++++++++++++++++++++++---------------
 3 files changed, 65 insertions(+), 20 deletions(-)
hw/i386/x86.c +17 -0
index b84840a1bb..25e6c50b1e 100644
@@ -1309,6 +1309,23 @@ static void machine_set_sgx_epc(Object *obj, Visitor *v, const char *name,
     qapi_free_SgxEPCList(list);
 }
 
+static void x86_machine_get_notify_window(Object *obj, Visitor *v,
+                                const char *name, void *opaque, Error **errp)
+{
+    X86MachineState *x86ms = X86_MACHINE(obj);
+    int32_t notify_window = x86ms->notify_window;
+
+    visit_type_int32(v, name, &notify_window, errp);
+}
+
+static void x86_machine_set_notify_window(Object *obj, Visitor *v,
+                               const char *name, void *opaque, Error **errp)
+{
+    X86MachineState *x86ms = X86_MACHINE(obj);
+
+    visit_type_int32(v, name, &x86ms->notify_window, errp);
+}
+
 static void x86_machine_initfn(Object *obj)
 {
     X86MachineState *x86ms = X86_MACHINE(obj);

hw/i386/x86.c static void x86_machine_initfn(Object *obj) · line 1340

Quoted 5 lines
hw/i386/x86.c +1 -0
@@ -1319,6 +1336,7 @@ static void x86_machine_initfn(Object *obj)
     x86ms->oem_id = g_strndup(ACPI_BUILD_APPNAME6, 6);
     x86ms->oem_table_id = g_strndup(ACPI_BUILD_APPNAME8, 8);
     x86ms->bus_lock_ratelimit = 0;
+    x86ms->notify_window = -1;
 }

IIUC from the kernel patch, this negative value leaves the protection disabled, and thus the host remains vulnerable to the CVE. I would expect this ought to set a suitable default value to fix the flaw.

Regards,
Daniel

Signature · 3 lines
|: https://berrange.com      -o-    https://www.flickr.com/photos/dberrange :|
|: https://libvirt.org         -o-            https://fstop138.berrange.com :|
|: https://entangle-photo.org    -o-    https://www.instagram.com/dberrange :|
#4 lore raw

Re: [PATCH 2/2] i386: Add notify VM exit support

Quoted 56 lines · Daniel P. Berrangé
Quoted 55 lines · Chenyi Qiang

There are cases that malicious virtual machine can cause CPU stuck (due to event windows don't open up), e.g., infinite loop in microcode when nested #AC (CVE-2015-5307). No event window means no event (NMI, SMI and IRQ) can be delivered. It leads the CPU to be unavailable to host or other VMs. Notify VM exit is introduced to mitigate such kind of attacks, which will generate a VM exit if no event window occurs in VM non-root mode for a specified amount of time (notify window).

A new KVM capability KVM_CAP_X86_NOTIFY_VMEXIT is exposed to user space so that the user can query the capability and set the expected notify window when creating VMs.

If notify VM exit happens with VM_INVALID_CONTEXT, hypervisor should exit to user space with the exit reason KVM_EXIT_NOTIFY to inform the fatal case. Then user space can inject a SHUTDOWN event to the target vcpu. This is implemented by defining a new bit in flags field of kvm_vcpu_event in KVM_SET_VCPU_EVENTS ioctl.

---
  hw/i386/x86.c         | 24 ++++++++++++++++++
  include/hw/i386/x86.h |  3 +++
  target/i386/kvm/kvm.c | 58 ++++++++++++++++++++++++++++---------------
  3 files changed, 65 insertions(+), 20 deletions(-)
hw/i386/x86.c +17 -0
index b84840a1bb..25e6c50b1e 100644
@@ -1309,6 +1309,23 @@ static void machine_set_sgx_epc(Object *obj, Visitor *v, const char *name,
      qapi_free_SgxEPCList(list);
  }
 
+static void x86_machine_get_notify_window(Object *obj, Visitor *v,
+                                const char *name, void *opaque, Error **errp)
+{
+    X86MachineState *x86ms = X86_MACHINE(obj);
+    int32_t notify_window = x86ms->notify_window;
+
+    visit_type_int32(v, name, &notify_window, errp);
+}
+
+static void x86_machine_set_notify_window(Object *obj, Visitor *v,
+                               const char *name, void *opaque, Error **errp)
+{
+    X86MachineState *x86ms = X86_MACHINE(obj);
+
+    visit_type_int32(v, name, &x86ms->notify_window, errp);
+}
+
  static void x86_machine_initfn(Object *obj)
  {
      X86MachineState *x86ms = X86_MACHINE(obj);
@@ -1319,6 +1336,7 @@ static void x86_machine_initfn(Object *obj)
      x86ms->oem_id = g_strndup(ACPI_BUILD_APPNAME6, 6);
Quoted 9 lines

hw/i386/x86.c static void x86_machine_initfn(Object *obj) · line 1340

Quoted 4 lines
hw/i386/x86.c +1 -0
      x86ms->oem_table_id = g_strndup(ACPI_BUILD_APPNAME8, 8);
      x86ms->bus_lock_ratelimit = 0;
+    x86ms->notify_window = -1;
  }

IIUC from the kernel patch, this negative value leaves the protection disabled, and thus the host remains vulnerable to the CVE. I would expect this ought to set a suitable default value to fix the flaw.

Hum, I missed some explanation in commit message.
We had some discussion about the default behavior of this feature. There are some concerns. e.g. There's a possibility, however small, that a notify VM exit happens
with VM_CONTEXT_INVALID set in exit qualification, which means VM
context is corrupted. To avoid the false positive and a well-behaved
guest gets killed, we decide to make this feature opt-in.

Quoted 2 lines

Regards,
Daniel

#5 lore raw

Re: [PATCH 2/2] i386: Add notify VM exit support

Quoted 68 lines · Chenyi Qiang
Quoted 55 lines · Daniel P. Berrangé
Quoted 54 lines · Chenyi Qiang

There are cases that malicious virtual machine can cause CPU stuck (due to event windows don't open up), e.g., infinite loop in microcode when nested #AC (CVE-2015-5307). No event window means no event (NMI, SMI and IRQ) can be delivered. It leads the CPU to be unavailable to host or other VMs. Notify VM exit is introduced to mitigate such kind of attacks, which will generate a VM exit if no event window occurs in VM non-root mode for a specified amount of time (notify window).

A new KVM capability KVM_CAP_X86_NOTIFY_VMEXIT is exposed to user space so that the user can query the capability and set the expected notify window when creating VMs.

If notify VM exit happens with VM_INVALID_CONTEXT, hypervisor should exit to user space with the exit reason KVM_EXIT_NOTIFY to inform the fatal case. Then user space can inject a SHUTDOWN event to the target vcpu. This is implemented by defining a new bit in flags field of kvm_vcpu_event in KVM_SET_VCPU_EVENTS ioctl.

---
  hw/i386/x86.c         | 24 ++++++++++++++++++
  include/hw/i386/x86.h |  3 +++
  target/i386/kvm/kvm.c | 58 ++++++++++++++++++++++++++++---------------
  3 files changed, 65 insertions(+), 20 deletions(-)
hw/i386/x86.c +17 -0
index b84840a1bb..25e6c50b1e 100644
@@ -1309,6 +1309,23 @@ static void machine_set_sgx_epc(Object *obj, Visitor *v, const char *name,
      qapi_free_SgxEPCList(list);
  }
+static void x86_machine_get_notify_window(Object *obj, Visitor *v,
+                                const char *name, void *opaque, Error **errp)
+{
+    X86MachineState *x86ms = X86_MACHINE(obj);
+    int32_t notify_window = x86ms->notify_window;
+
+    visit_type_int32(v, name, &notify_window, errp);
+}
+
+static void x86_machine_set_notify_window(Object *obj, Visitor *v,
+                               const char *name, void *opaque, Error **errp)
+{
+    X86MachineState *x86ms = X86_MACHINE(obj);
+
+    visit_type_int32(v, name, &x86ms->notify_window, errp);
+}
+
  static void x86_machine_initfn(Object *obj)
  {
      X86MachineState *x86ms = X86_MACHINE(obj);
@@ -1319,6 +1336,7 @@ static void x86_machine_initfn(Object *obj)
      x86ms->oem_id = g_strndup(ACPI_BUILD_APPNAME6, 6);
Quoted 9 lines

hw/i386/x86.c static void x86_machine_initfn(Object *obj) · line 1340

Quoted 4 lines
hw/i386/x86.c +1 -0
      x86ms->oem_table_id = g_strndup(ACPI_BUILD_APPNAME8, 8);
      x86ms->bus_lock_ratelimit = 0;
+    x86ms->notify_window = -1;
  }

IIUC from the kernel patch, this negative value leaves the protection disabled, and thus the host remains vulnerable to the CVE. I would expect this ought to set a suitable default value to fix the flaw.

Quoted 7 lines

Hum, I missed some explanation in commit message.
We had some discussion about the default behavior of this feature. There are some concerns. e.g.
There's a possibility, however small, that a notify VM exit happens with VM_CONTEXT_INVALID set in exit qualification, which means VM context is corrupted. To avoid the false positive and a well-behaved guest gets killed, we decide to make this feature opt-in.

That is unfortunate. It is not going to be much benefit to add this feature, if users are discouraged from using it because of the danger of it killing non-malicious guests :-(

Regards,
Daniel

Signature · 3 lines
|: https://berrange.com      -o-    https://www.flickr.com/photos/dberrange :|
|: https://libvirt.org         -o-            https://fstop138.berrange.com :|
|: https://entangle-photo.org    -o-    https://www.instagram.com/dberrange :|