Skip to contentSkip to conversation

Extensible VCPU state IOCTL

PATCH New Jan Kiszka 1 participant 7 messages Nov 6, 2009 – Nov 9, 2009

lore
Share & tools
lore

Jan Kiszka

author
#0 lore raw

cover Extensible VCPU state IOCTL

This series adds support for the upcoming new KVM IOCTL to get/set VPCU states as a set up substates. See [1] for more details on this KVM extension.

While I did not fully convert the corresponding qemu-kvm part (it is expected to morph into upstream soon), QEMU should receive a complete conversion. So this series first fixes, cleans up, and refactors some KVM code, and then adds KVM_GET/SET_VCPU_STATE as an alternative to the classic multi-IOCTL state getting and setting. I did this full conversion with all its #ifdefs in order to be able to drop legacy support at some (still distant) point in the future.

Along with the conversion a few fragile migration, save/restore and reset cases are fixed. We now have access to all missing exception, interrupt, and NMI states KVM kept private so far. See [2] for details.

Find this series also at git://git.kiszka.org/qemu.git queues/kvm

Jan Kiszka (5):
kvm: x86: Fix merge artifact of f8d926e9 about mp_state
kvm: Add arch reset handler
kvm: x86: Refactor use of interrupt_bitmap
kvm: x86: Add support for KVM_GET/PUT_VCPU_STATE
kvm: x86: Add support for event states

 kvm-all.c             |   13 ++
 kvm.h                 |    3 +
 target-i386/cpu.h     |    8 +-
 target-i386/kvm.c     |  514 ++++++++++++++++++++++++++++++-------------------
 target-i386/machine.c |   29 +--
 target-ppc/kvm.c      |    4 +
 6 files changed, 346 insertions(+), 225 deletions(-)

[1] http://thread.gmane.org/gmane.comp.emulators.kvm.devel/42699
[2] http://thread.gmane.org/gmane.comp.emulators.kvm.devel/42699/focus=42926

author
#1 lore raw

PATCH 1/5 kvm: x86: Fix merge artifact of f8d926e9 about mp_state

---

 target-i386/kvm.c |    8 ++++----
 1 files changed, 4 insertions(+), 4 deletions(-)
target-i386/kvm.c +4 -4
index 7010999..aa90eff 100644
@@ -683,10 +683,6 @@ int kvm_arch_put_registers(CPUState *env)
     if (ret < 0)
         return ret;
 
-    ret = kvm_get_mp_state(env);
-    if (ret < 0)
-        return ret;
-
     return 0;
 }
 
@@ -710,6 +706,10 @@ int kvm_arch_get_registers(CPUState *env)
     if (ret < 0)
         return ret;
 
+    ret = kvm_get_mp_state(env);
+    if (ret < 0)
+        return ret;
+
     return 0;
 }
 
author
#2 lore raw

PATCH 2/5 kvm: Add arch reset handler

In reply to Jan Kiszka

Will be required by succeeding changes.

---

 kvm-all.c         |    2 ++
 kvm.h             |    2 ++
 target-i386/kvm.c |    4 ++++
 target-ppc/kvm.c  |    4 ++++
 4 files changed, 12 insertions(+), 0 deletions(-)
kvm-all.c +2 -0
index 48ae26c..31bc2f8 100644
@@ -150,6 +150,7 @@ static void kvm_reset_vcpu(void *opaque)
 {
     CPUState *env = opaque;
 
+    kvm_arch_reset_vcpu(env);
     if (kvm_arch_put_registers(env)) {
         fprintf(stderr, "Fatal: kvm vcpu reset failed\n");
         abort();
@@ -201,6 +202,7 @@ int kvm_init_vcpu(CPUState *env)
     ret = kvm_arch_init_vcpu(env);
     if (ret == 0) {
         qemu_register_reset(kvm_reset_vcpu, env);
+        kvm_arch_reset_vcpu(env);
         ret = kvm_arch_put_registers(env);
     }
 err:
kvm.h +2 -0
index e7d5beb..6a82f6a 100644
@@ -93,6 +93,8 @@ int kvm_arch_init(KVMState *s, int smp_cpus);
 
 int kvm_arch_init_vcpu(CPUState *env);
 
+void kvm_arch_reset_vcpu(CPUState *env);
+
 struct kvm_guest_debug;
 struct kvm_debug_exit_arch;
 
target-i386/kvm.c +4 -0
index aa90eff..ca99cc7 100644
@@ -221,6 +221,10 @@ int kvm_arch_init_vcpu(CPUState *env)
     return kvm_vcpu_ioctl(env, KVM_SET_CPUID2, &cpuid_data);
 }
 
+void kvm_arch_reset_vcpu(CPUState *env)
+{
+}
+
 static int kvm_has_msr_star(CPUState *env)
 {
     static int has_msr_star;
target-ppc/kvm.c +4 -0
index b53d6e9..4e1c65f 100644
@@ -53,6 +53,10 @@ int kvm_arch_init_vcpu(CPUState *cenv)
     return ret;
 }
 
+void kvm_arch_reset_vcpu(CPUState *env)
+{
+}
+
 int kvm_arch_put_registers(CPUState *env)
 {
     struct kvm_regs regs;
author
#3 lore raw

PATCH 3/5 kvm: x86: Refactor use of interrupt_bitmap

In reply to Jan Kiszka

Drop interrupt_bitmap from the cpustate and solely rely on the integer interupt_injected. This prepares us for the new injected-interrupt interface, which will deprecate the bitmap, while preserving compatibility.

---

 target-i386/cpu.h     |    3 +--
 target-i386/kvm.c     |   25 ++++++++++++++++++-------
 target-i386/machine.c |   24 ++----------------------
 3 files changed, 21 insertions(+), 31 deletions(-)
target-i386/cpu.h +1 -2
index 5929d28..eb9532a 100644
@@ -691,8 +691,8 @@ typedef struct CPUX86State {
     MTRRVar mtrr_var[8];
 
     /* For KVM */
-    uint64_t interrupt_bitmap[256 / 64];
     uint32_t mp_state;
+    int32_t interrupt_injected;
 
     /* in order to simplify APIC support, we leave this pointer to the
        user */
@@ -709,7 +709,6 @@ typedef struct CPUX86State {
     uint16_t fpus_vmstate;
     uint16_t fptag_vmstate;
     uint16_t fpregs_format_vmstate;
-    int32_t pending_irq_vmstate;
 } CPUX86State;
 
 CPUX86State *cpu_x86_init(const char *cpu_model);
target-i386/kvm.c +18 -7
index ca99cc7..c769d70 100644
@@ -23,6 +23,7 @@
 #include "kvm.h"
 #include "cpu.h"
 #include "gdbstub.h"
+#include "host-utils.h"
 
 //#define DEBUG_KVM
 
@@ -223,6 +224,7 @@ int kvm_arch_init_vcpu(CPUState *env)
 
 void kvm_arch_reset_vcpu(CPUState *env)
 {
+    env->interrupt_injected = -1;
 }
 
 static int kvm_has_msr_star(CPUState *env)
@@ -411,9 +413,11 @@ static int kvm_put_sregs(CPUState *env)
 {
     struct kvm_sregs sregs;
 
-    memcpy(sregs.interrupt_bitmap,
-           env->interrupt_bitmap,
-           sizeof(sregs.interrupt_bitmap));
+    memset(sregs.interrupt_bitmap, 0, sizeof(sregs.interrupt_bitmap));
+    if (env->interrupt_injected >= 0) {
+        sregs.interrupt_bitmap[env->interrupt_injected / 64] |=
+                (uint64_t)1 << (env->interrupt_injected % 64);
+    }
 
     if ((env->eflags & VM_MASK)) {
 	    set_v8086_seg(&sregs.cs, &env->segs[R_CS]);
@@ -520,15 +524,22 @@ static int kvm_get_sregs(CPUState *env)
 {
     struct kvm_sregs sregs;
     uint32_t hflags;
-    int ret;
+    int bit, i, ret;
 
     ret = kvm_vcpu_ioctl(env, KVM_GET_SREGS, &sregs);
     if (ret < 0)
         return ret;
 
-    memcpy(env->interrupt_bitmap, 
-           sregs.interrupt_bitmap,
-           sizeof(sregs.interrupt_bitmap));
+    /* There can only be one pending IRQ set in the bitmap at a time, so try
+       to find it and save its number instead (-1 for none). */
+    env->interrupt_injected = -1;
+    for (i = 0; i < ARRAY_SIZE(sregs.interrupt_bitmap); i++) {
+        if (sregs.interrupt_bitmap[i]) {
+            bit = ctz64(sregs.interrupt_bitmap[i]);
+            env->interrupt_injected = i * 64 + bit;
+            break;
+        }
+    }
 
     get_seg(&env->segs[R_CS], &sregs.cs);
     get_seg(&env->segs[R_DS], &sregs.ds);
target-i386/machine.c +2 -22
index 869c681..c09b049 100644
@@ -2,7 +2,6 @@
 #include "hw/boards.h"
 #include "hw/pc.h"
 #include "hw/isa.h"
-#include "host-utils.h"
 
 #include "exec-all.h"
 #include "kvm.h"
@@ -320,7 +319,7 @@ static const VMStateInfo vmstate_hack_uint64_as_uint32 = {
 static void cpu_pre_save(void *opaque)
 {
     CPUState *env = opaque;
-    int i, bit;
+    int i;
 
     cpu_synchronize_state(env);
 
@@ -336,17 +335,6 @@ static void cpu_pre_save(void *opaque)
 #else
     env->fpregs_format_vmstate = 1;
 #endif
-
-    /* There can only be one pending IRQ set in the bitmap at a time, so try
-       to find it and save its number instead (-1 for none). */
-    env->pending_irq_vmstate = -1;
-    for (i = 0; i < ARRAY_SIZE(env->interrupt_bitmap); i++) {
-        if (env->interrupt_bitmap[i]) {
-            bit = ctz64(env->interrupt_bitmap[i]);
-            env->pending_irq_vmstate = i * 64 + bit;
-            break;
-        }
-    }
 }
 
 static int cpu_pre_load(void *opaque)
@@ -375,14 +363,6 @@ static int cpu_post_load(void *opaque, int version_id)
     for (i = 0; i < 4; i++)
         hw_breakpoint_insert(env, i);
 
-    if (version_id >= 9) {
-        memset(&env->interrupt_bitmap, 0, sizeof(env->interrupt_bitmap));
-        if (env->pending_irq_vmstate >= 0) {
-            env->interrupt_bitmap[env->pending_irq_vmstate / 64] |=
-                (uint64_t)1 << (env->pending_irq_vmstate % 64);
-        }
-    }
-
     tlb_flush(env, 1);
     return 0;
 }
@@ -465,7 +445,7 @@ static const VMStateDescription vmstate_cpu = {
         VMSTATE_UINT64_V(mtrr_deftype, CPUState, 8),
         VMSTATE_MTRR_VARS(mtrr_var, CPUState, 8, 8),
         /* KVM-related states */
-        VMSTATE_INT32_V(pending_irq_vmstate, CPUState, 9),
+        VMSTATE_INT32_V(interrupt_injected, CPUState, 9),
         VMSTATE_UINT32_V(mp_state, CPUState, 9),
         VMSTATE_UINT64_V(tsc, CPUState, 9),
         /* MCE */
author
#4 lore raw

PATCH 4/5 kvm: x86: Add support for KVM_GET/PUT_VCPU_STATE

In reply to Jan Kiszka

This patch adds support for the new KVM IOCTLs KVM_GET/SET_VCPU_STATE. They are supposed to be introduced with 2.6.33 or 34 and allow to extend the VCPU state exchange between kernel and user space without adding new IOCTLs and to acquire/set a complete state via a single systemcall.

In order to remain backward-compatible to existing kernels, the old state synchronization interface is of course kept as a fall-back.

---

 kvm-all.c         |   11 +
 kvm.h             |    1 
 target-i386/kvm.c |  451 +++++++++++++++++++++++++++++------------------------
 3 files changed, 260 insertions(+), 203 deletions(-)
kvm-all.c +11 -0
index 31bc2f8..0d22e42 100644
@@ -61,6 +61,7 @@ struct KVMState
     int coalesced_mmio;
     int broken_set_mem_region;
     int migration_log;
+    int vcpu_state;
 #ifdef KVM_CAP_SET_GUEST_DEBUG
     struct kvm_sw_breakpoint_head kvm_sw_breakpoints;
 #endif
@@ -499,6 +500,11 @@ int kvm_init(int smp_cpus)
     }
 #endif
 
+    s->vcpu_state = 0;
+#ifdef KVM_CAP_VCPU_STATE
+    s->vcpu_state = kvm_check_extension(s, KVM_CAP_VCPU_STATE);
+#endif
+
     ret = kvm_arch_init(s, smp_cpus);
     if (ret < 0)
         goto err;
@@ -888,6 +894,11 @@ int kvm_has_sync_mmu(void)
 #endif
 }
 
+int kvm_has_vcpu_state(void)
+{
+    return kvm_state->vcpu_state;
+}
+
 void kvm_setup_guest_memory(void *start, size_t size)
 {
     if (!kvm_has_sync_mmu()) {
kvm.h +1 -0
index 6a82f6a..9e237d1 100644
@@ -47,6 +47,7 @@ int kvm_log_stop(target_phys_addr_t phys_addr, ram_addr_t size);
 int kvm_set_migration_log(int enable);
 
 int kvm_has_sync_mmu(void);
+int kvm_has_vcpu_state(void);
 
 void kvm_setup_guest_memory(void *start, size_t size);
 
target-i386/kvm.c +248 -203

566 lines not on this page · raw message

author
#5 lore raw

PATCH v2 4/5 kvm: x86: Add support for KVM_GET/PUT_VCPU_STATE

This patch adds support for the new KVM IOCTLs KVM_GET/SET_VCPU_STATE. They are supposed to be introduced with 2.6.33 or 34 and allow to extend the VCPU state exchange between kernel and user space without adding new IOCTLs and to acquire/set a complete state via a single systemcall.

In order to remain backward-compatible to existing kernels, the old state synchronization interface is of course kept as a fall-back.

---

Changes in v2:
- A classic: Let's make it cleaner, use ARRAY_SIZE, no need to test.
IOW: fix request.header.nsubstates initialization (queue on kiszka.org updated)

 kvm-all.c         |   11 +
 kvm.h             |    1 
 target-i386/kvm.c |  451 +++++++++++++++++++++++++++++------------------------
 3 files changed, 260 insertions(+), 203 deletions(-)
kvm-all.c +11 -0
index 31bc2f8..0d22e42 100644
@@ -61,6 +61,7 @@ struct KVMState
     int coalesced_mmio;
     int broken_set_mem_region;
     int migration_log;
+    int vcpu_state;
 #ifdef KVM_CAP_SET_GUEST_DEBUG
     struct kvm_sw_breakpoint_head kvm_sw_breakpoints;
 #endif
@@ -499,6 +500,11 @@ int kvm_init(int smp_cpus)
     }
 #endif
 
+    s->vcpu_state = 0;
+#ifdef KVM_CAP_VCPU_STATE
+    s->vcpu_state = kvm_check_extension(s, KVM_CAP_VCPU_STATE);
+#endif
+
     ret = kvm_arch_init(s, smp_cpus);
     if (ret < 0)
         goto err;
@@ -888,6 +894,11 @@ int kvm_has_sync_mmu(void)
 #endif
 }
 
+int kvm_has_vcpu_state(void)
+{
+    return kvm_state->vcpu_state;
+}
+
 void kvm_setup_guest_memory(void *start, size_t size)
 {
     if (!kvm_has_sync_mmu()) {
kvm.h +1 -0
index 6a82f6a..9e237d1 100644
@@ -47,6 +47,7 @@ int kvm_log_stop(target_phys_addr_t phys_addr, ram_addr_t size);
 int kvm_set_migration_log(int enable);
 
 int kvm_has_sync_mmu(void);
+int kvm_has_vcpu_state(void);
 
 void kvm_setup_guest_memory(void *start, size_t size);
 
target-i386/kvm.c +248 -203

566 lines not on this page · raw message

author
#6 lore raw

PATCH 5/5 kvm: x86: Add support for event states

In reply to Jan Kiszka

This patch extends the KVM state sync logic with the event substate from the new VCPU state interface, giving access to yet missing exception, interrupt and NMI states.

---

 target-i386/cpu.h     |    5 ++++
 target-i386/kvm.c     |   56 +++++++++++++++++++++++++++++++++++++++++++++++--
 target-i386/machine.c |    5 ++++
 3 files changed, 64 insertions(+), 2 deletions(-)
target-i386/cpu.h +5 -0
index eb9532a..9c3e905 100644
@@ -693,6 +693,11 @@ typedef struct CPUX86State {
     /* For KVM */
     uint32_t mp_state;
     int32_t interrupt_injected;
+    uint8_t soft_interrupt;
+    uint8_t nmi_injected;
+    uint8_t nmi_pending;
+    uint8_t has_error_code;
+    uint32_t sipi_vector;
 
     /* in order to simplify APIC support, we leave this pointer to the
        user */
target-i386/kvm.c +54 -2

108 lines not on this page · raw message

target-i386/machine.c +5 -0
index c09b049..54d324e 100644
@@ -448,6 +448,11 @@ static const VMStateDescription vmstate_cpu = {
         VMSTATE_INT32_V(interrupt_injected, CPUState, 9),
         VMSTATE_UINT32_V(mp_state, CPUState, 9),
         VMSTATE_UINT64_V(tsc, CPUState, 9),
+        VMSTATE_UINT8_V(soft_interrupt, CPUState, 11),
+        VMSTATE_UINT8_V(nmi_injected, CPUState, 11),
+        VMSTATE_UINT8_V(nmi_pending, CPUState, 11),
+        VMSTATE_UINT8_V(has_error_code, CPUState, 11),
+        VMSTATE_UINT32_V(sipi_vector, CPUState, 11),
         /* MCE */
         VMSTATE_UINT64_V(mcg_cap, CPUState, 10),
         VMSTATE_UINT64_V(mcg_status, CPUState, 10),