Skip to contentSkip to conversation

queue_work proposal

RFC In review Glauber Costa 2 participants 4 messages Sep 3, 2009 – Oct 22, 2009 1 message not in the archive

lore
Share & tools
lore
#0 lore raw

RFC v2 1/1 queue_work proposal

Hi guys

In this patch, I am attaching an early version of a new "on_vcpu" mechanism (after making it generic, I saw no reason to keep its name). It allows us to guarantee that a piece of code will be executed in a certain vcpu, indicated by a CPUState.

I am sorry for the big patch, I just dumped what I had so we can have early directions. When it comes to submission state, I'll split it accordingly.

As we discussed these days at qemu-devel, I am using pthread_set/get_specific for dealing with thread-local variables. Note that they are not used from signal handlers. A first optimization would be to use TLS variables where available.

In vl.c, I am providing a version of queue_work for the IO-thread, and other for normal operation. The "normal" one should fix the problems Jan is having, since it does nothing more than just issuing the function we want to execute.

The io-thread version is tested with both tcg and kvm, and works (to the extent they were working before, which in kvm case, is not much)

Changes from v1:
* Don't open the possibility of asynchronous calling queue_work, suggested by Avi "Peter Parker" Kivity
* Use a local mutex, suggested by Paolo Bonzini

---
 cpu-all.h  |    3 ++
 cpu-defs.h |   15 ++++++++++++
 exec.c     |    1 +
 kvm-all.c  |   58 +++++++++++++++++++---------------------------
 kvm.h      |    7 +++++
 vl.c       |   75 ++++++++++++++++++++++++++++++++++++++++++++++++++++++++++++
 6 files changed, 125 insertions(+), 34 deletions(-)
cpu-all.h +3 -0
index 1a6a812..529479e 100644
@@ -763,6 +763,9 @@ extern CPUState *cpu_single_env;
 extern int64_t qemu_icount;
 extern int use_icount;
 
+void qemu_queue_work(CPUState *env, void (*func)(void *data), void *data);
+void qemu_flush_work(CPUState *env);
+
 #define CPU_INTERRUPT_HARD   0x02 /* hardware interrupt pending */
 #define CPU_INTERRUPT_EXITTB 0x04 /* exit the current TB (use for x86 a20 case) */
 #define CPU_INTERRUPT_TIMER  0x08 /* internal timer exception pending */
cpu-defs.h +15 -0
index b6c8b95..20f83e3 100644
@@ -31,6 +31,8 @@
 #include "sys-queue.h"
 #include "targphys.h"
 
+#include "qemu-thread.h"
+
 #ifndef TARGET_LONG_BITS
 #error TARGET_LONG_BITS must be defined before including this header
 #endif
@@ -134,6 +136,14 @@ typedef struct CPUWatchpoint {
     TAILQ_ENTRY(CPUWatchpoint) entry;
 } CPUWatchpoint;
 
+typedef struct QemuWorkItem {
+    void (*func)(void *data);
+    void *data;
+    int done;
+    TAILQ_ENTRY(QemuWorkItem) entry;
+} QemuWorkItem;
+
+
 #define CPU_TEMP_BUF_NLONGS 128
 #define CPU_COMMON                                                      \
     struct TranslationBlock *current_tb; /* currently executing TB  */  \
@@ -175,6 +185,10 @@ typedef struct CPUWatchpoint {
     TAILQ_HEAD(watchpoints_head, CPUWatchpoint) watchpoints;            \
     CPUWatchpoint *watchpoint_hit;                                      \
                                                                         \
+    TAILQ_HEAD(queued_work_head, QemuWorkItem) queued_work;             \
+    uint64_t queued_local, queued_total;                                \
+    struct QemuMutex queue_lock;                                        \
+                                                                        \
     struct GDBRegisterState *gdb_regs;                                  \
                                                                         \
     /* Core interrupt code */                                           \
@@ -194,6 +208,7 @@ typedef struct CPUWatchpoint {
     uint32_t created;                                                   \
     struct QemuThread *thread;                                          \
     struct QemuCond *halt_cond;                                         \
+    struct QemuCond work_cond;                                          \
     const char *cpu_model_str;                                          \
     struct KVMState *kvm_state;                                         \
     struct kvm_run *kvm_run;                                            \
exec.c +1 -0
index 21f10b9..2db5e5b 100644
@@ -577,6 +577,7 @@ void cpu_exec_init(CPUState *env)
     env->numa_node = 0;
     TAILQ_INIT(&env->breakpoints);
     TAILQ_INIT(&env->watchpoints);
+    TAILQ_INIT(&env->queued_work);
     *penv = env;
 #if defined(CONFIG_USER_ONLY)
     cpu_list_unlock();
kvm-all.c +24 -34

92 lines not on this page · raw message

kvm.h +7 -0
index dbe825f..7595141 100644
@@ -139,4 +139,11 @@ static inline void cpu_synchronize_state(CPUState *env)
     }
 }
 
+typedef struct KVMIoctl {
+    int fd;
+    int type;
+    int ret;
+    void *data;
+} KVMIoctl;
+
 #endif
vl.c +75 -0

117 lines not on this page · raw message

Signature · 1 line
1.6.2.2

A message that is not in the archive (sent off-list or not yet received): [email protected]

#2 lore raw
Quoted 21 lines · Juan Quintela

Glauber Costa <[email protected]> wrote:

Hi Glauber

Quoted 17 lines
kvm-all.c +6 -3
+int kvm_vcpu_ioctl(CPUState *env, int type, ...)
+{
     void *arg;
     va_list ap;
+    KVMIoctl data;
 
     va_start(ap, type);
     arg = va_arg(ap, void *);
     va_end(ap);
 
-    ret = ioctl(env->kvm_fd, type, arg);
-    if (ret == -1)
-        ret = -errno;
+    data.type = type;
+    data.data = arg;
+    data.fd = env->kvm_fd;
Quoted 6 lines

kvm-all.c int kvm_vm_ioctl(KVMState *s, int type, ...) · line 890

Quoted 4 lines
kvm-all.c +3 -1
-    return ret;
+    qemu_queue_work(env, kvm_remote_ioctl, (void *)&data);
+
+    return data.ret;

Does this work? giving the address of a local variable is a no-no.

Yes, because it synchronous. The stack will keep existing until queue_work returns.

Actually, this was one of the reasons Avi pushed so hardly against the async mechanism

#3 lore raw
In reply to Glauber Costa
Quoted 69 lines · Glauber Costa

Hi guys

In this patch, I am attaching an early version of a new "on_vcpu" mechanism (after making it generic, I saw no reason to keep its name). It allows us to guarantee that a piece of code will be executed in a certain vcpu, indicated by a CPUState.

I am sorry for the big patch, I just dumped what I had so we can have early directions. When it comes to submission state, I'll split it accordingly.

As we discussed these days at qemu-devel, I am using pthread_set/get_specific for dealing with thread-local variables. Note that they are not used from signal handlers. A first optimization would be to use TLS variables where available.

In vl.c, I am providing a version of queue_work for the IO-thread, and other for normal operation. The "normal" one should fix the problems Jan is having, since it does nothing more than just issuing the function we want to execute.

The io-thread version is tested with both tcg and kvm, and works (to the extent they were working before, which in kvm case, is not much)

Changes from v1:
* Don't open the possibility of asynchronous calling queue_work, suggested by Avi "Peter Parker" Kivity
* Use a local mutex, suggested by Paolo Bonzini

---
 cpu-all.h  |    3 ++
 cpu-defs.h |   15 ++++++++++++
 exec.c     |    1 +
 kvm-all.c  |   58 +++++++++++++++++++---------------------------
 kvm.h      |    7 +++++
 vl.c       |   75 ++++++++++++++++++++++++++++++++++++++++++++++++++++++++++++
 6 files changed, 125 insertions(+), 34 deletions(-)
cpu-all.h +18 -0
index 1a6a812..529479e 100644
@@ -763,6 +763,9 @@ extern CPUState *cpu_single_env;
 extern int64_t qemu_icount;
 extern int use_icount;
 
+void qemu_queue_work(CPUState *env, void (*func)(void *data), void *data);
+void qemu_flush_work(CPUState *env);
+
 #define CPU_INTERRUPT_HARD   0x02 /* hardware interrupt pending */
 #define CPU_INTERRUPT_EXITTB 0x04 /* exit the current TB (use for x86 a20 case) */
 #define CPU_INTERRUPT_TIMER  0x08 /* internal timer exception pending */
@@ -3808,6 +3835,50 @@ void qemu_cpu_kick(void *_env)
         qemu_thread_signal(env->thread, SIGUSR1);
 }
 
+void qemu_queue_work(CPUState *env, void (*func)(void *data), void *data)
+{
+    QemuWorkItem wii;
+
+    env->queued_total++;
+
+    if (env == qemu_get_current_env()) {
+        env->queued_local++;
+        func(data);
+        return;
+    }
+
+    wii.func = func;
+    wii.data = data;
+    qemu_mutex_lock(&env->queue_lock);

cpu-all.h void qemu_cpu_kick(void *_env) · line 3862

Quoted 10 lines
cpu-all.h +10 -0
+    TAILQ_INSERT_TAIL(&env->queued_work, &wii, entry);
+    qemu_mutex_unlock(&env->queue_lock);
+
+    qemu_thread_signal(env->thread, SIGUSR1);
+
+    qemu_mutex_lock(&env->queue_lock);
+    while (!wii.done) {
+        qemu_cond_wait(&env->work_cond, &qemu_global_mutex);
+    }
+    qemu_mutex_unlock(&env->queue_lock);

How's qemu_flush_work supposed to execute if env->queue_lock is held here?

qemu_cond_wait() should work with env->queue_lock, and qemu_global_mutex should be dropped before waiting and reacquired on return.

#4 lore raw
Quoted 84 lines · Marcelo Tosatti
Quoted 69 lines · Glauber Costa

Hi guys

In this patch, I am attaching an early version of a new "on_vcpu" mechanism (after making it generic, I saw no reason to keep its name). It allows us to guarantee that a piece of code will be executed in a certain vcpu, indicated by a CPUState.

I am sorry for the big patch, I just dumped what I had so we can have early directions. When it comes to submission state, I'll split it accordingly.

As we discussed these days at qemu-devel, I am using pthread_set/get_specific for dealing with thread-local variables. Note that they are not used from signal handlers. A first optimization would be to use TLS variables where available.

In vl.c, I am providing a version of queue_work for the IO-thread, and other for normal operation. The "normal" one should fix the problems Jan is having, since it does nothing more than just issuing the function we want to execute.

The io-thread version is tested with both tcg and kvm, and works (to the extent they were working before, which in kvm case, is not much)

Changes from v1:
* Don't open the possibility of asynchronous calling queue_work, suggested by Avi "Peter Parker" Kivity
* Use a local mutex, suggested by Paolo Bonzini

---
 cpu-all.h  |    3 ++
 cpu-defs.h |   15 ++++++++++++
 exec.c     |    1 +
 kvm-all.c  |   58 +++++++++++++++++++---------------------------
 kvm.h      |    7 +++++
 vl.c       |   75 ++++++++++++++++++++++++++++++++++++++++++++++++++++++++++++
 6 files changed, 125 insertions(+), 34 deletions(-)
cpu-all.h +18 -0
index 1a6a812..529479e 100644
@@ -763,6 +763,9 @@ extern CPUState *cpu_single_env;
 extern int64_t qemu_icount;
 extern int use_icount;
 
+void qemu_queue_work(CPUState *env, void (*func)(void *data), void *data);
+void qemu_flush_work(CPUState *env);
+
 #define CPU_INTERRUPT_HARD   0x02 /* hardware interrupt pending */
 #define CPU_INTERRUPT_EXITTB 0x04 /* exit the current TB (use for x86 a20 case) */
 #define CPU_INTERRUPT_TIMER  0x08 /* internal timer exception pending */
@@ -3808,6 +3835,50 @@ void qemu_cpu_kick(void *_env)
         qemu_thread_signal(env->thread, SIGUSR1);
 }
 
+void qemu_queue_work(CPUState *env, void (*func)(void *data), void *data)
+{
+    QemuWorkItem wii;
+
+    env->queued_total++;
+
+    if (env == qemu_get_current_env()) {
+        env->queued_local++;
+        func(data);
+        return;
+    }
+
+    wii.func = func;
+    wii.data = data;
+    qemu_mutex_lock(&env->queue_lock);

cpu-all.h void qemu_cpu_kick(void *_env) · line 3862

Quoted 10 lines
cpu-all.h +10 -0
+    TAILQ_INSERT_TAIL(&env->queued_work, &wii, entry);
+    qemu_mutex_unlock(&env->queue_lock);
+
+    qemu_thread_signal(env->thread, SIGUSR1);
+
+    qemu_mutex_lock(&env->queue_lock);
+    while (!wii.done) {
+        qemu_cond_wait(&env->work_cond, &qemu_global_mutex);
+    }
+    qemu_mutex_unlock(&env->queue_lock);

How's qemu_flush_work supposed to execute if env->queue_lock is held here?

Quoted 2 lines

qemu_cond_wait() should work with env->queue_lock, and qemu_global_mutex should be dropped before waiting and reacquired on return.

After some thinking, I don't plan to introduce this until it is absolutely needed. I believe we can refactor a lot of code to actually run on the vcpu it should, instead of triggering a remove event.