Skip to contentSkip to conversation

Complete cpu initialization before signaling main thread.

PATCH Applied Gleb Natapov 4 participants 5 messages Oct 14, 2009 – Nov 13, 2009 KVM

lore
Share & tools
lore
#0 lore raw

PATCH v2 1/2 Complete cpu initialization before signaling main thread.

Otherwise some cpus may start executing code before others are fully initialized.

---
v1->v2:
- reinit cpu_single_env after qemu_cond_wait()

 qemu-kvm.c |   29 +++++++++++++++--------------
 1 files changed, 15 insertions(+), 14 deletions(-)
qemu-kvm.c +15 -14
index 62ca050..a104ab8 100644
@@ -1954,18 +1954,6 @@ static void process_irqchip_events(CPUState *env)
 
 static int kvm_main_loop_cpu(CPUState *env)
 {
-    setup_kernel_sigmask(env);
-
-    pthread_mutex_lock(&qemu_mutex);
-
-    kvm_arch_init_vcpu(env);
-#ifdef TARGET_I386
-    kvm_tpr_vcpu_start(env);
-#endif
-
-    cpu_single_env = env;
-    kvm_arch_load_regs(env);
-
     while (1) {
         int run_cpu = !is_cpu_stopped(env);
         if (run_cpu && !kvm_irqchip_in_kernel(kvm_context)) {
@@ -2003,15 +1991,28 @@ static void *ap_main_loop(void *_env)
         on_vcpu(env, kvm_arch_do_ioperm, data);
 #endif
 
-    /* signal VCPU creation */
+    setup_kernel_sigmask(env);
+
     pthread_mutex_lock(&qemu_mutex);
+    cpu_single_env = env;
+
+    kvm_arch_init_vcpu(env);
+#ifdef TARGET_I386
+    kvm_tpr_vcpu_start(env);
+#endif
+
+    kvm_arch_load_regs(env);
+
+    /* signal VCPU creation */
     current_env->created = 1;
     pthread_cond_signal(&qemu_vcpu_cond);
 
     /* and wait for machine initialization */
     while (!qemu_system_ready)
         qemu_cond_wait(&qemu_system_cond);
-    pthread_mutex_unlock(&qemu_mutex);
+
+    /* re-initialize cpu_single_env after re-acquiring qemu_mutex */
+    cpu_single_env = env;
 
     kvm_main_loop_cpu(env);
     return NULL;
Signature · 1 line
1.6.3.3
#1 lore raw

PATCH 2/2 Don't sync mpstate to/from kernel when unneeded.

mp_state, unlike other cpu state, can be changed not only from vcpu context it belongs to, but by other vcpus too. That makes its loading from kernel/saving back not safe if mp_state value is changed inside kernel between load and save. For example vcpu 1 loads mp_sate into user-space and the state is RUNNING, vcpu 0 sends INIT/SIPI to vcpu 1 so in-kernel mp_sate becomes SIPI, vcpu 1 save user-space copy into kernel and calls vcpu_run(). SIPI sate is lost.

The patch copies mp_sate into kernel only when it is knows that int-kernel value is outdated. This happens on reset and vmload.

---
 hw/apic.c             |    1 +
 monitor.c             |    2 ++
 qemu-kvm.c            |    9 ++++-----
 qemu-kvm.h            |    1 -
 target-i386/machine.c |    3 +++
 5 files changed, 10 insertions(+), 6 deletions(-)
hw/apic.c +1 -0
index 2952675..7244449 100644
@@ -512,6 +512,7 @@ void apic_init_reset(CPUState *env)
     if (kvm_enabled() && qemu_kvm_irqchip_in_kernel()) {
         env->mp_state
             = env->halted ? KVM_MP_STATE_UNINITIALIZED : KVM_MP_STATE_RUNNABLE;
+        kvm_load_mpstate(env);
     }
 #endif
 }
monitor.c +2 -0
index 7f0f5a9..dd8f2ca 100644
@@ -350,6 +350,7 @@ static CPUState *mon_get_cpu(void)
         mon_set_cpu(0);
     }
     cpu_synchronize_state(cur_mon->mon_cpu);
+    kvm_save_mpstate(cur_mon->mon_cpu);
     return cur_mon->mon_cpu;
 }
 
@@ -377,6 +378,7 @@ static void do_info_cpus(Monitor *mon)
 
     for(env = first_cpu; env != NULL; env = env->next_cpu) {
         cpu_synchronize_state(env);
+        kvm_save_mpstate(env);
         monitor_printf(mon, "%c CPU #%d:",
                        (env == mon->mon_cpu) ? '*' : ' ',
                        env->cpu_index);
qemu-kvm.c +4 -5
index a104ab8..267222d 100644
@@ -1609,11 +1609,6 @@ static void on_vcpu(CPUState *env, void (*func)(void *data), void *data)
 void kvm_arch_get_registers(CPUState *env)
 {
 	kvm_arch_save_regs(env);
-	kvm_arch_save_mpstate(env);
-#ifdef KVM_CAP_MP_STATE
-	if (kvm_irqchip_in_kernel(kvm_context))
-		env->halted = (env->mp_state == KVM_MP_STATE_HALTED);
-#endif
 }
 
 static void do_kvm_cpu_synchronize_state(void *_env)
@@ -1707,6 +1702,10 @@ static void kvm_do_save_mpstate(void *_env)
     CPUState *env = _env;
 
     kvm_arch_save_mpstate(env);
+#ifdef KVM_CAP_MP_STATE
+    if (kvm_irqchip_in_kernel(kvm_context))
+        env->halted = (env->mp_state == KVM_MP_STATE_HALTED);
+#endif
 }
 
 void kvm_save_mpstate(CPUState *env)
qemu-kvm.h +0 -1
index d6748c7..e2a87b8 100644
@@ -1186,7 +1186,6 @@ void kvm_arch_get_registers(CPUState *env);
 static inline void kvm_arch_put_registers(CPUState *env)
 {
     kvm_load_registers(env);
-    kvm_load_mpstate(env);
 }
 
 void kvm_cpu_synchronize_state(CPUState *env);
target-i386/machine.c +3 -0
index e640dad..16d9c57 100644
@@ -324,6 +324,7 @@ static void cpu_pre_save(void *opaque)
     int i, bit;
 
     cpu_synchronize_state(env);
+    kvm_save_mpstate(env);
 
     /* FPU */
     env->fpus_vmstate = (env->fpus & ~0x3800) | (env->fpstt & 0x7) << 11;
@@ -385,6 +386,8 @@ static int cpu_post_load(void *opaque, int version_id)
     }
 
     tlb_flush(env, 1);
+    kvm_load_mpstate(env);
+
     return 0;
 }
 
Signature · 1 line
1.6.3.3
#2 lore raw

Re: [PATCH 2/2] Don't sync mpstate to/from kernel when unneeded.

Quoted 10 lines · Gleb Natapov

mp_state, unlike other cpu state, can be changed not only from vcpu context it belongs to, but by other vcpus too. That makes its loading from kernel/saving back not safe if mp_state value is changed inside kernel between load and save. For example vcpu 1 loads mp_sate into user-space and the state is RUNNING, vcpu 0 sends INIT/SIPI to vcpu 1 so in-kernel mp_sate becomes SIPI, vcpu 1 save user-space copy into kernel and calls vcpu_run(). SIPI sate is lost.

The patch copies mp_sate into kernel only when it is knows that int-kernel value is outdated. This happens on reset and vmload.

Just stumbled over this commit as it breaks kvm-disabled builds (layering violations...). I guess we need this upstream too once the irqchips go in. Or do your patches, Glauber, already include this? If not, anyone already thought about how to move the kvm logic behind qemu's state sync curtain?

Jan

Quoted 102 lines
---
 hw/apic.c             |    1 +
 monitor.c             |    2 ++
 qemu-kvm.c            |    9 ++++-----
 qemu-kvm.h            |    1 -
 target-i386/machine.c |    3 +++
 5 files changed, 10 insertions(+), 6 deletions(-)
hw/apic.c +1 -0
index 2952675..7244449 100644
@@ -512,6 +512,7 @@ void apic_init_reset(CPUState *env)
     if (kvm_enabled() && qemu_kvm_irqchip_in_kernel()) {
         env->mp_state
             = env->halted ? KVM_MP_STATE_UNINITIALIZED : KVM_MP_STATE_RUNNABLE;
+        kvm_load_mpstate(env);
     }
 #endif
 }
monitor.c +2 -0
index 7f0f5a9..dd8f2ca 100644
@@ -350,6 +350,7 @@ static CPUState *mon_get_cpu(void)
         mon_set_cpu(0);
     }
     cpu_synchronize_state(cur_mon->mon_cpu);
+    kvm_save_mpstate(cur_mon->mon_cpu);
     return cur_mon->mon_cpu;
 }
 
@@ -377,6 +378,7 @@ static void do_info_cpus(Monitor *mon)
 
     for(env = first_cpu; env != NULL; env = env->next_cpu) {
         cpu_synchronize_state(env);
+        kvm_save_mpstate(env);
         monitor_printf(mon, "%c CPU #%d:",
                        (env == mon->mon_cpu) ? '*' : ' ',
                        env->cpu_index);
qemu-kvm.c +4 -5
index a104ab8..267222d 100644
@@ -1609,11 +1609,6 @@ static void on_vcpu(CPUState *env, void (*func)(void *data), void *data)
 void kvm_arch_get_registers(CPUState *env)
 {
 	kvm_arch_save_regs(env);
-	kvm_arch_save_mpstate(env);
-#ifdef KVM_CAP_MP_STATE
-	if (kvm_irqchip_in_kernel(kvm_context))
-		env->halted = (env->mp_state == KVM_MP_STATE_HALTED);
-#endif
 }
 
 static void do_kvm_cpu_synchronize_state(void *_env)
@@ -1707,6 +1702,10 @@ static void kvm_do_save_mpstate(void *_env)
     CPUState *env = _env;
 
     kvm_arch_save_mpstate(env);
+#ifdef KVM_CAP_MP_STATE
+    if (kvm_irqchip_in_kernel(kvm_context))
+        env->halted = (env->mp_state == KVM_MP_STATE_HALTED);
+#endif
 }
 
 void kvm_save_mpstate(CPUState *env)
qemu-kvm.h +0 -1
index d6748c7..e2a87b8 100644
@@ -1186,7 +1186,6 @@ void kvm_arch_get_registers(CPUState *env);
 static inline void kvm_arch_put_registers(CPUState *env)
 {
     kvm_load_registers(env);
-    kvm_load_mpstate(env);
 }
 
 void kvm_cpu_synchronize_state(CPUState *env);
target-i386/machine.c +3 -0
index e640dad..16d9c57 100644
@@ -324,6 +324,7 @@ static void cpu_pre_save(void *opaque)
     int i, bit;
 
     cpu_synchronize_state(env);
+    kvm_save_mpstate(env);
 
     /* FPU */
     env->fpus_vmstate = (env->fpus & ~0x3800) | (env->fpstt & 0x7) << 11;
@@ -385,6 +386,8 @@ static int cpu_post_load(void *opaque, int version_id)
     }
 
     tlb_flush(env, 1);
+    kvm_load_mpstate(env);
+
     return 0;
 }
 
#3 lore raw

Re: [PATCH 2/2] Don't sync mpstate to/from kernel when unneeded.

Quoted 9 lines · Jan Kiszka
Quoted 8 lines · Gleb Natapov

mp_state, unlike other cpu state, can be changed not only from vcpu context it belongs to, but by other vcpus too. That makes its loading from kernel/saving back not safe if mp_state value is changed inside kernel between load and save. For example vcpu 1 loads mp_sate into user-space and the state is RUNNING, vcpu 0 sends INIT/SIPI to vcpu 1 so in-kernel mp_sate becomes SIPI, vcpu 1 save user-space copy into kernel and calls vcpu_run(). SIPI sate is lost.

Quoted 8 lines
Quoted 2 lines

The patch copies mp_sate into kernel only when it is knows that int-kernel value is outdated. This happens on reset and vmload.

Just stumbled over this commit as it breaks kvm-disabled builds (layering violations...). I guess we need this upstream too once the irqchips go in. Or do your patches, Glauber, already include this? If not, anyone already thought about how to move the kvm logic behind qemu's state sync curtain?

It does not, since upstream qemu still does not support smp.

I basically did not worried about it.

Quoted 106 lines

Jan

Quoted 102 lines
---
 hw/apic.c             |    1 +
 monitor.c             |    2 ++
 qemu-kvm.c            |    9 ++++-----
 qemu-kvm.h            |    1 -
 target-i386/machine.c |    3 +++
 5 files changed, 10 insertions(+), 6 deletions(-)
hw/apic.c +1 -0
index 2952675..7244449 100644
@@ -512,6 +512,7 @@ void apic_init_reset(CPUState *env)
     if (kvm_enabled() && qemu_kvm_irqchip_in_kernel()) {
         env->mp_state
             = env->halted ? KVM_MP_STATE_UNINITIALIZED : KVM_MP_STATE_RUNNABLE;
+        kvm_load_mpstate(env);
     }
 #endif
 }
monitor.c +2 -0
index 7f0f5a9..dd8f2ca 100644
@@ -350,6 +350,7 @@ static CPUState *mon_get_cpu(void)
         mon_set_cpu(0);
     }
     cpu_synchronize_state(cur_mon->mon_cpu);
+    kvm_save_mpstate(cur_mon->mon_cpu);
     return cur_mon->mon_cpu;
 }
 
@@ -377,6 +378,7 @@ static void do_info_cpus(Monitor *mon)
 
     for(env = first_cpu; env != NULL; env = env->next_cpu) {
         cpu_synchronize_state(env);
+        kvm_save_mpstate(env);
         monitor_printf(mon, "%c CPU #%d:",
                        (env == mon->mon_cpu) ? '*' : ' ',
                        env->cpu_index);
qemu-kvm.c +4 -5
index a104ab8..267222d 100644
@@ -1609,11 +1609,6 @@ static void on_vcpu(CPUState *env, void (*func)(void *data), void *data)
 void kvm_arch_get_registers(CPUState *env)
 {
 	kvm_arch_save_regs(env);
-	kvm_arch_save_mpstate(env);
-#ifdef KVM_CAP_MP_STATE
-	if (kvm_irqchip_in_kernel(kvm_context))
-		env->halted = (env->mp_state == KVM_MP_STATE_HALTED);
-#endif
 }
 
 static void do_kvm_cpu_synchronize_state(void *_env)
@@ -1707,6 +1702,10 @@ static void kvm_do_save_mpstate(void *_env)
     CPUState *env = _env;
 
     kvm_arch_save_mpstate(env);
+#ifdef KVM_CAP_MP_STATE
+    if (kvm_irqchip_in_kernel(kvm_context))
+        env->halted = (env->mp_state == KVM_MP_STATE_HALTED);
+#endif
 }
 
 void kvm_save_mpstate(CPUState *env)
qemu-kvm.h +0 -1
index d6748c7..e2a87b8 100644
@@ -1186,7 +1186,6 @@ void kvm_arch_get_registers(CPUState *env);
 static inline void kvm_arch_put_registers(CPUState *env)
 {
     kvm_load_registers(env);
-    kvm_load_mpstate(env);
 }
 
 void kvm_cpu_synchronize_state(CPUState *env);
target-i386/machine.c +3 -0
index e640dad..16d9c57 100644
@@ -324,6 +324,7 @@ static void cpu_pre_save(void *opaque)
     int i, bit;
 
     cpu_synchronize_state(env);
+    kvm_save_mpstate(env);
 
     /* FPU */
     env->fpus_vmstate = (env->fpus & ~0x3800) | (env->fpstt & 0x7) << 11;
@@ -385,6 +386,8 @@ static int cpu_post_load(void *opaque, int version_id)
     }
 
     tlb_flush(env, 1);
+    kvm_load_mpstate(env);
+
     return 0;
 }