Skip to contentSkip to conversation

separate thread for io v2

PATCH In review [email protected] 4 participants 19 messages Mar 19, 2009 – Mar 24, 2009 Kbuild

lore
Share & tools
lore
#0 lore raw

cover separate thread for io v2

Move IO event processing to a separate thread, with a global mutex serializing access to QEMU data structures. This opens the possibility to execute certain operations in parallel, and is necessary to integrate KVM SMP support.

Future changes:
- make the cpu_single_env history nicer
- use signalfd and block IO event signals in the io thread
- optimize tcg interaction
- implement kvm smp

#1 lore raw

PATCH 1/7 qemu: mutex/thread/cond wrappers

Makefile.target +1 -1
Index: qemu/Makefile.target
===================================================================
@@ -500,7 +500,7 @@ endif #CONFIG_BSD_USER
 # System emulator target
 ifndef CONFIG_USER_ONLY
 
-OBJS=vl.o osdep.o monitor.o pci.o loader.o isa_mmio.o machine.o
+OBJS=vl.o qemu-thread.o osdep.o monitor.o pci.o loader.o isa_mmio.o machine.o
 # virtio has to be here due to weird dependency between PCI and virtio-net.
 # need to fix this properly
 OBJS+=virtio.o virtio-blk.o virtio-balloon.o virtio-net.o virtio-console.o
vl.c +13 -0
Index: qemu/vl.c
===================================================================
@@ -36,6 +36,7 @@
 #include "gdbstub.h"
 #include "qemu-timer.h"
 #include "qemu-char.h"
+#include "qemu-thread.h"
 #include "cache-utils.h"
 #include "block.h"
 #include "audio/audio.h"
@@ -263,6 +264,8 @@ static QEMUTimer *nographic_timer;
 
 uint8_t qemu_uuid[16];
 
+QemuMutex qemu_global_mutex;
+
 /***********************************************************/
 /* x86 ISA bus support */
 
@@ -3650,7 +3653,14 @@ void main_loop_wait(int timeout)
         slirp_select_fill(&nfds, &rfds, &wfds, &xfds);
     }
 #endif
+
+    /*
+     * main_loop_wait() *must* not assume any global state is consistent across
+     * select() invocations.
+     */
+    qemu_mutex_unlock(&qemu_global_mutex);
     ret = select(nfds + 1, &rfds, &wfds, &xfds, &tv);
+    qemu_mutex_lock(&qemu_global_mutex);
     if (ret > 0) {
         IOHandlerRecord **pioh;
 
@@ -3708,6 +3718,9 @@ static int main_loop(void)
 #endif
     CPUState *env;
 
+    qemu_mutex_init(&qemu_global_mutex);
+    qemu_mutex_lock(&qemu_global_mutex);
+
     cur_cpu = first_cpu;
     next_cpu = cur_cpu->next_cpu ?: first_cpu;
     for(;;) {
qemu-thread.c new file +99 -0

99 lines not on this page · raw message

qemu-thread.h new file +38 -0
Index: qemu/qemu-thread.h
===================================================================
@@ -0,0 +1,38 @@
+#include "semaphore.h"
+#include "pthread.h"
+
+struct QemuMutex {
+    pthread_mutex_t lock;
+};
+
+struct QemuCond {
+    pthread_cond_t cond;
+};
+
+struct QemuThread {
+    pthread_t thread;
+};
+
+typedef struct QemuMutex QemuMutex;
+typedef struct QemuCond QemuCond;
+typedef struct QemuThread QemuThread;
+
+int qemu_mutex_init(QemuMutex *mutex);
+void qemu_mutex_lock(QemuMutex *mutex);
+int qemu_mutex_trylock(QemuMutex *mutex);
+int qemu_mutex_timedlock(QemuMutex *mutex, unsigned int msecs);
+void qemu_mutex_unlock(QemuMutex *mutex);
+
+void qemu_cond_init(QemuCond *cond);
+void qemu_cond_signal(QemuCond *cond);
+void qemu_cond_broadcast(QemuCond *cond);
+int qemu_cond_wait(QemuCond *cond, QemuMutex *mutex);
+int qemu_cond_timedwait(QemuCond *cond, QemuMutex *mutex, unsigned int msecs);
+
+int qemu_thread_create(QemuThread *thread,
+                       void *(*start_routine)(void*),
+                       void *arg);
+int qemu_thread_signal(QemuThread *thread, int sig);
+void qemu_thread_self(QemuThread *thread);
+int qemu_thread_equal(QemuThread *thread1, QemuThread *thread2);
+
#2 lore raw

Re: [Qemu-devel] [patch 1/7] qemu: mutex/thread/cond wrappers

Quoted 36 lines · [email protected] wrote:
vl.c +10 -0
@@ -36,6 +36,7 @@
 #include "gdbstub.h"
 #include "qemu-timer.h"
 #include "qemu-char.h"
+#include "qemu-thread.h"
 #include "cache-utils.h"
 #include "block.h"
 #include "audio/audio.h"
@@ -263,6 +264,8 @@ static QEMUTimer *nographic_timer;
 
 uint8_t qemu_uuid[16];
 
+QemuMutex qemu_global_mutex;
+
 /***********************************************************/
 /* x86 ISA bus support */
 
@@ -3650,7 +3653,14 @@ void main_loop_wait(int timeout)
         slirp_select_fill(&nfds, &rfds, &wfds, &xfds);
     }
 #endif
+
+    /*
+     * main_loop_wait() *must* not assume any global state is consistent across
+     * select() invocations.
+     */
+    qemu_mutex_unlock(&qemu_global_mutex);
     ret = select(nfds + 1, &rfds, &wfds, &xfds, &tv);
+    qemu_mutex_lock(&qemu_global_mutex);
     if (ret > 0) {
         IOHandlerRecord **pioh;
 

vl.c static int main_loop(void) · line 3726

Quoted 9 lines
vl.c +3 -0
@@ -3708,6 +3718,9 @@ static int main_loop(void)
 #endif
     CPUState *env;
 
+    qemu_mutex_init(&qemu_global_mutex);
+    qemu_mutex_lock(&qemu_global_mutex);
+
     cur_cpu = first_cpu;
     next_cpu = cur_cpu->next_cpu ?: first_cpu;
     for(;;) {

All the bits above seem unnecessary for the this patch.

vl.c

Quoted 6 lines
vl.c +5 -0
+
+int qemu_mutex_init(QemuMutex *mutex)
+{
+    return pthread_mutex_init(&mutex->lock, NULL);
+}

No one will check the return code, so better to check it here and abort() if it's bad.

vl.c

Quoted 10 lines
vl.c +9 -0
+static void add_to_timespec(struct timespec *ts, unsigned int msecs)
+{
+    ts->tv_sec = ts->tv_sec + (long)(msecs / 1000);
+    ts->tv_nsec = (ts->tv_nsec + ((long)msecs % 1000) * 1000000);
+    if (ts->tv_nsec >= 1000000000) {
+        ts->tv_nsec -= 1000000000;
+        ts->tv_sec++;
+    }
+}

timespec_add_msec()? and why milliseconds? Also, maybe uint64_t is a better type.

vl.c

Quoted 11 lines
vl.c +10 -0
+
+int qemu_mutex_timedlock(QemuMutex *mutex, unsigned int msecs)
+{
+    struct timespec ts;
+
+    clock_gettime(CLOCK_REALTIME, &ts);
+    add_to_timespec(&ts, msecs);
+
+    return pthread_mutex_timedlock(&mutex->lock, &ts);
+}

I would have preferred a deadline instead of a timeout, but we'll see on the next patches.

vl.c

Quoted 8 lines
vl.c +7 -0
+
+int qemu_thread_create(QemuThread *thread,
+                       void *(*start_routine)(void*),
+                       void *arg)
+{
+    return pthread_create(&thread->thread, NULL, start_routine, arg);
+}

Don't return an error that no one will check.

vl.c

Quoted 8 lines
vl.c +7 -0
+
+int qemu_thread_signal(QemuThread *thread, int sig)
+{
+    if (thread->thread != 0)
+        return pthread_kill(thread->thread, sig);
+    return -1; /* XXX: ESCHR */
+}

Ditto. If the thread dies, qemu dies.

vl.c

Quoted 5 lines
vl.c +4 -0
+int qemu_thread_equal(QemuThread *thread1, QemuThread *thread2)
+{
+   return (thread1->thread == thread2->thread);
+}

Can compare thing1 to thing2 instead of ->thread. Not that it matters.

Signature · 1 line
error compiling committee.c: too many arguments to function
#3 lore raw

Re: [Qemu-devel] [patch 1/7] qemu: mutex/thread/cond wrappers

Quoted 7 lines · Avi Kivity

qemu-thread.c line 9

Quoted 4 lines
qemu-thread.c new file +4 -0
+int qemu_mutex_init(QemuMutex *mutex)
+{
+    return pthread_mutex_init(&mutex->lock, NULL);
+}

No one will check the return code, so better to check it here and abort() if it's bad.

I agree.

Same applies to pthread_mutex_lock, etc., all of which return an error code, can fail if misused, and nobody checks.

-- Jamie

#4 lore raw

Re: [Qemu-devel] [patch 1/7] qemu: mutex/thread/cond wrappers

In reply to Avi Kivity
Quoted 13 lines · Avi Kivity

qemu-thread.c line 29

Quoted 10 lines
qemu-thread.c new file +9 -0
+static void add_to_timespec(struct timespec *ts, unsigned int msecs)
+{
+    ts->tv_sec = ts->tv_sec + (long)(msecs / 1000);
+    ts->tv_nsec = (ts->tv_nsec + ((long)msecs % 1000) * 1000000);
+    if (ts->tv_nsec >= 1000000000) {
+        ts->tv_nsec -= 1000000000;
+        ts->tv_sec++;
+    }
+}

timespec_add_msec()?

Quoted 1 line

and why milliseconds?

Well, its suitable for the current purposes. Any other suggestion?

Quoted 16 lines

Also, maybe uint64_t is a better type.

qemu-thread.c line 39

Quoted 11 lines
qemu-thread.c new file +10 -0
+
+int qemu_mutex_timedlock(QemuMutex *mutex, unsigned int msecs)
+{
+    struct timespec ts;
+
+    clock_gettime(CLOCK_REALTIME, &ts);
+    add_to_timespec(&ts, msecs);
+
+    return pthread_mutex_timedlock(&mutex->lock, &ts);
+}

I would have preferred a deadline instead of a timeout, but we'll see on the next patches.

Please expand?

Fixed the other comments, thanks.

#5 lore raw

Re: [Qemu-devel] [patch 1/7] qemu: mutex/thread/cond wrappers

Quoted 6 lines · Marcelo Tosatti
Quoted 2 lines

and why milliseconds?

Well, its suitable for the current purposes. Any other suggestion?

uint64_t nanoseconds. It's standard timekeeping in qemu:

  #define QEMU_TIMER_BASE 1000000000LL

and it's of course best to limit the number of standards.

Quoted 19 lines
Quoted 4 lines

Also, maybe uint64_t is a better type.

Quoted 14 lines

qemu-thread.c line 39

Quoted 12 lines
qemu-thread.c new file +10 -0
+
+int qemu_mutex_timedlock(QemuMutex *mutex, unsigned int msecs)
+{
+    struct timespec ts;
+
+    clock_gettime(CLOCK_REALTIME, &ts);
+    add_to_timespec(&ts, msecs);
+
+    return pthread_mutex_timedlock(&mutex->lock, &ts);
+}

I would have preferred a deadline instead of a timeout, but we'll see on the next patches.

Quoted 2 lines

Please expand?

Instead of 'sleep until n nanoseconds have elapsed', 'sleep until get_clock() >= deadline'. I think it simplifies things somewhat.

Signature · 2 lines
I have a truly marvellous patch that fixes the bug which this
signature is too narrow to contain.
#6 lore raw

PATCH 2/7 qemu: separate thread for io

In reply to [email protected]
exec.c +14 -0
Index: qemu/exec.c
===================================================================
@@ -1513,6 +1513,20 @@ void cpu_interrupt(CPUState *env, int ma
     /* FIXME: This is probably not threadsafe.  A different thread could
        be in the middle of a read-modify-write operation.  */
     env->interrupt_request |= mask;
+
+    switch(mask) {
+    case CPU_INTERRUPT_HARD:
+    case CPU_INTERRUPT_SMI:
+    case CPU_INTERRUPT_NMI:
+    case CPU_INTERRUPT_EXIT:
+    /*
+     * only unlink the TB's if we're called from cpu thread context,
+     * otherwise signal cpu thread to do it.
+     */
+        if (qemu_notify_event(env))
+               return;
+    }
+
 #if defined(USE_NPTL)
     /* FIXME: TB unchaining isn't SMP safe.  For now just ignore the
        problem and hope the cpu will stop of its own accord.  For userspace
qemu-common.h +2 -0
Index: qemu/qemu-common.h
===================================================================
@@ -190,6 +190,8 @@ int cpu_load(QEMUFile *f, void *opaque, 
 
 /* Force QEMU to stop what it's doing and service IO */
 void qemu_service_io(void);
+void main_loop_break(void);
+int qemu_notify_event(void *env);
 
 typedef struct QEMUIOVector {
     struct iovec *iov;
vl.c +164 -12

239 lines not on this page · raw message

#7 lore raw

[Qemu-devel] Re: [patch 2/7] qemu: separate thread for io

Quoted 132 lines · [email protected] wrote:

Index: qemu/exec.c
===================================================================

exec.c +14 -0
@@ -1513,6 +1513,20 @@ void cpu_interrupt(CPUState *env, int ma
     /* FIXME: This is probably not threadsafe.  A different thread could
        be in the middle of a read-modify-write operation.  */
     env->interrupt_request |= mask;
+
+    switch(mask) {
+    case CPU_INTERRUPT_HARD:
+    case CPU_INTERRUPT_SMI:
+    case CPU_INTERRUPT_NMI:
+    case CPU_INTERRUPT_EXIT:
+    /*
+     * only unlink the TB's if we're called from cpu thread context,
+     * otherwise signal cpu thread to do it.
+     */
+        if (qemu_notify_event(env))
+               return;
+    }
+
 #if defined(USE_NPTL)
     /* FIXME: TB unchaining isn't SMP safe.  For now just ignore the
        problem and hope the cpu will stop of its own accord.  For userspace

Index: qemu/qemu-common.h
===================================================================

qemu-common.h +2 -0
@@ -190,6 +190,8 @@ int cpu_load(QEMUFile *f, void *opaque, 
 
 /* Force QEMU to stop what it's doing and service IO */
 void qemu_service_io(void);
+void main_loop_break(void);
+int qemu_notify_event(void *env);
 
 typedef struct QEMUIOVector {
     struct iovec *iov;

Index: qemu/vl.c
===================================================================

vl.c +34 -10
@@ -155,6 +155,7 @@
 //#define DEBUG_NET
 //#define DEBUG_SLIRP
 
+#define SIG_IPI (SIGRTMIN+4)
 
 #ifdef DEBUG_IOPORT
 #  define LOG_IOPORT(...) qemu_log_mask(CPU_LOG_IOPORT, ## __VA_ARGS__)
@@ -265,6 +266,10 @@ static QEMUTimer *nographic_timer;
 uint8_t qemu_uuid[16];
 
 QemuMutex qemu_global_mutex;
+QemuMutex qemu_fair_mutex;
+
+QemuThread io_thread;
+QemuThread cpus_thread;
 
 /***********************************************************/
 /* x86 ISA bus support */
@@ -1328,7 +1333,6 @@ static void host_alarm_handler(int host_
                                qemu_get_clock(vm_clock))) ||
         qemu_timer_expired(active_timers[QEMU_TIMER_REALTIME],
                            qemu_get_clock(rt_clock))) {
-        CPUState *env = next_cpu;
 
 #ifdef _WIN32
         struct qemu_alarm_win32 *data = ((struct qemu_alarm_timer*)dwUser)->priv;
@@ -1339,15 +1343,6 @@ static void host_alarm_handler(int host_
 #endif
         alarm_timer->flags |= ALARM_FLAG_EXPIRED;
 
-        if (env) {
-            /* stop the currently executing cpu because a timer occured */
-            cpu_interrupt(env, CPU_INTERRUPT_EXIT);
-#ifdef USE_KQEMU
-            if (env->kqemu_enabled) {
-                kqemu_cpu_interrupt(env);
-            }
-#endif
-        }
         event_pending = 1;
     }
 }
@@ -2878,6 +2873,7 @@ int qemu_set_fd_handler2(int fd,
         ioh->opaque = opaque;
         ioh->deleted = 0;
     }
+    main_loop_break();
     return 0;
 }
 
@@ -3334,6 +3330,7 @@ void qemu_bh_schedule(QEMUBH *bh)
     if (env) {
         cpu_interrupt(env, CPU_INTERRUPT_EXIT);
     }
+    main_loop_break();
 }
 
 void qemu_bh_cancel(QEMUBH *bh)
@@ -3611,6 +3608,155 @@ static void host_main_loop_wait(int *tim
 }
 #endif
 
+static int wait_signal(int timeout)
+{
+    struct timespec ts;
+    sigset_t waitset;
+
+    ts.tv_sec = timeout / 1000;
+    ts.tv_nsec = (timeout % 1000) * 1000000;
+    sigemptyset(&waitset);
+    sigaddset(&waitset, SIGUSR1);
+
+    return sigtimedwait(&waitset, NULL, &ts);
+}
+
+static int has_work(CPUState *env)
+{
+    int r = 0;
+    if (!env->halted)
+        r = 1;
+    if (env->interrupt_request & (CPU_INTERRUPT_HARD | CPU_INTERRUPT_NMI))
+        r = 1;
+    return r;
+}
+
+static void qemu_wait_io_event(CPUState *env, int timeout)
+{
+    qemu_mutex_unlock(&qemu_global_mutex);
+

vl.c static void host_main_loop_wait(int *tim · line 3647

Quoted 10 lines
vl.c +9 -0
+    if (timeout && !has_work(env))
+        wait_signal(timeout);
+    /*
+     * Users of qemu_global_mutex can be starved, having no chance
+     * to acquire it since this path will get to it first.
+     * So use another lock to provide fairness.
+     */
+    qemu_mutex_lock(&qemu_fair_mutex);
+    qemu_mutex_unlock(&qemu_fair_mutex);
 

This is extremely subtle. I think it can be made a bit more explicit via something like:

int generation = qemu_io_generation() + 1;

qemu_mutex_unlock(&qemu_global_mutex);

if (timeout && !has_work(env))

    wait_signal(timeout);

qemu_mutex_lock(&qemu_global_mutex)

while (qemu_io_generation() < generation)
qemu_cond_wait(&qemu_io_generation_cond, &qemu_global_mutex);

Then, in main_loop_wait():

vl.c void main_loop_wait(int timeout) · line 3809

Quoted 9 lines
vl.c +0 -1
 void main_loop_wait(int timeout)
 {
     IOHandlerRecord *ioh;
@@ -3660,7 +3806,7 @@ void main_loop_wait(int timeout)
      */
     qemu_mutex_unlock(&qemu_global_mutex);
     ret = select(nfds + 1, &rfds, &wfds, &xfds, &tv);
-    qemu_mutex_lock(&qemu_global_mutex);
 

qemu_mutex_lock(&qemu_global_mutex);
qemu_io_generation_add();
qemu_cond_signal(&qemu_io_generation_cond);

This will ensure that the TCG loop backs off of qemu_global_mutex for at least one main_loop_wait() iteration.

Regards,

Anthony Liguori

#8 lore raw

[Qemu-devel] Re: [patch 2/7] qemu: separate thread for io

Quoted 35 lines · Anthony Liguori

Review

Quoted 3 lines
(quoted diff) +2 -0
+    qemu_mutex_lock(&qemu_fair_mutex);
+    qemu_mutex_unlock(&qemu_fair_mutex);

This is extremely subtle. I think it can be made a bit more explicit via something like:

int generation = qemu_io_generation() + 1;

qemu_mutex_unlock(&qemu_global_mutex);

if (timeout && !has_work(env))

    wait_signal(timeout);

qemu_mutex_lock(&qemu_global_mutex)

while (qemu_io_generation() < generation)
qemu_cond_wait(&qemu_io_generation_cond, &qemu_global_mutex);

Then, in main_loop_wait():

vl.c void main_loop_wait(int timeout) · line 3809

Quoted 9 lines

void main_loop_wait(int timeout)

 {
     IOHandlerRecord *ioh;
vl.c +0 -1
@@ -3660,7 +3806,7 @@ void main_loop_wait(int timeout)
      */
     qemu_mutex_unlock(&qemu_global_mutex);
     ret = select(nfds + 1, &rfds, &wfds, &xfds, &tv);
-    qemu_mutex_lock(&qemu_global_mutex);
 

qemu_mutex_lock(&qemu_global_mutex);
qemu_io_generation_add();
qemu_cond_signal(&qemu_io_generation_cond);

Quoted 2 lines

This will ensure that the TCG loop backs off of qemu_global_mutex for at least one main_loop_wait() iteration.

I don't see much benefit. It has the disadvantage of introducing more state (the generation counter, the conditional), and the potential problems associated with this state such as:

- If there is no event for the iothread to process, TCG will throttle unnecessarily (i can't see how that could happen, but is a possibility) until some event breaks it out of select() thus increasing the generation counter.

- Its possible to miss signals (again, I can't see in happening in the scheme you suggest), but..

Also note there is no need to be completly fair. It is fine to eventually be unfair.

Do you worry about interaction between the two locks? Can make the lock ordering documented.

Will spend some time thinking about this, need to take multiple "starved" threads into account.

Thanks.

#9 lore raw

[Qemu-devel] Re: [patch 2/7] qemu: separate thread for io

Quoted 40 lines · Marcelo Tosatti
Quoted 35 lines · Anthony Liguori

Review

Quoted 4 lines
(quoted diff) +2 -0
+    qemu_mutex_lock(&qemu_fair_mutex);
+    qemu_mutex_unlock(&qemu_fair_mutex);

This is extremely subtle. I think it can be made a bit more explicit via something like:

int generation = qemu_io_generation() + 1;

qemu_mutex_unlock(&qemu_global_mutex);

if (timeout && !has_work(env))

    wait_signal(timeout);

qemu_mutex_lock(&qemu_global_mutex)

while (qemu_io_generation() < generation)
qemu_cond_wait(&qemu_io_generation_cond, &qemu_global_mutex);

Then, in main_loop_wait():

vl.c void main_loop_wait(int timeout) · line 3810

Quoted 10 lines

void main_loop_wait(int timeout)

 {
     IOHandlerRecord *ioh;
vl.c +0 -1
@@ -3660,7 +3806,7 @@ void main_loop_wait(int timeout)
      */
     qemu_mutex_unlock(&qemu_global_mutex);
     ret = select(nfds + 1, &rfds, &wfds, &xfds, &tv);
-    qemu_mutex_lock(&qemu_global_mutex);
 
 

qemu_mutex_lock(&qemu_global_mutex);
qemu_io_generation_add();
qemu_cond_signal(&qemu_io_generation_cond);

Quoted 2 lines

This will ensure that the TCG loop backs off of qemu_global_mutex for at least one main_loop_wait() iteration.

Quoted 9 lines

I don't see much benefit. It has the disadvantage of introducing more state (the generation counter, the conditional), and the potential problems associated with this state such as:

- If there is no event for the iothread to process, TCG will throttle unnecessarily (i can't see how that could happen, but is a possibility) until some event breaks it out of select() thus increasing the generation counter.

Right, you have to couple this with a signal sent from the TCG thread to the io thread. And yeah, signals are not 100% reliable.

Quoted 9 lines

- Its possible to miss signals (again, I can't see in happening in the scheme you suggest), but..

Also note there is no need to be completly fair. It is fine to eventually be unfair.

Do you worry about interaction between the two locks? Can make the lock ordering documented.

I need to think about this a bit more. I understand the idea behind qemu_signal_lock() a little better now.

Since we know that the IO thread is trying to run in TCG (because we sent a signal to it), I wonder if we can use that as an indicator that we have to let the IO thread run for a bit.

BTW, your patches lack commit messages and Signed-off-bys. Are you not ready for having them committed? I know that we still have to figure out Windows support but do you know of any other show stoppers?

Have you thought about how this is going to affect kvm-userspace? Do you think we can eliminate the io threading code in kvm-userspace after this goes in?

Regards,

Anthony Liguori

Quoted 6 lines

Will spend some time thinking about this, need to take multiple "starved" threads into account.

Thanks.

#10 lore raw

[Qemu-devel] Re: [patch 2/7] qemu: separate thread for io

Quoted 22 lines · Anthony Liguori
Quoted 5 lines

- If there is no event for the iothread to process, TCG will throttle unnecessarily (i can't see how that could happen, but is a possibility) until some event breaks it out of select() thus increasing the generation counter.

Right, you have to couple this with a signal sent from the TCG thread to the io thread. And yeah, signals are not 100% reliable.

Quoted 9 lines

- Its possible to miss signals (again, I can't see in happening in
the scheme you suggest), but..

Also note there is no need to be completly fair. It is fine to eventually be unfair.

Do you worry about interaction between the two locks? Can make the lock ordering documented.

I need to think about this a bit more. I understand the idea behind qemu_signal_lock() a little better now.

Quoted 7 lines

Since we know that the IO thread is trying to run in TCG (because we sent a signal to it), I wonder if we can use that as an indicator that we have to let the IO thread run for a bit.

BTW, your patches lack commit messages and Signed-off-bys. Are you not ready for having them committed? I know that we still have to figure out Windows support but do you know of any other show stoppers?

There was a significant (25% IIRC) reduction in iperf performance. This is sort of expected, since there are no optimizations at all (should collapse the signals sent to TCG context, for one). But my thinking is to merge the iothread (along the lines of this patchset), stabilize and then optimize.

How about that?

Quoted 1 line

Have you thought about how this is going to affect kvm-userspace?

Oops, no. But I can be held accountable for kvm-userspace iothread until it can be fully replaced by upstream.

Quoted 2 lines

Do you think we can eliminate the io threading code in kvm-userspace after this goes in?

Not immediately, need to generalize some of the changes introduced by the patchset and merge the remaining logic of kvm-userspace's qemu-kvm.c.

#11 lore raw

[Qemu-devel] Re: [patch 2/7] qemu: separate thread for io

Quoted 8 lines · Marcelo Tosatti

There was a significant (25% IIRC) reduction in iperf performance. This is sort of expected, since there are no optimizations at all (should collapse the signals sent to TCG context, for one). But my thinking is to merge the iothread (along the lines of this patchset), stabilize and then optimize.

How about that?

I don't mind that but I'll need to check out how things behave with TCG and with other guest architectures. I'm willing to accept a temporary performance regression in something like iperf because I don't think people are relying on QEMU network performance that much today (outside of KVM/Xen) but if we have significant performance regressions with non-x86 boards with things like basic boot time, we'll have to fix those first.

We also need to not completely break the Windows build before merging.

Quoted 6 lines
Quoted 2 lines

Have you thought about how this is going to affect kvm-userspace?

Oops, no. But I can be held accountable for kvm-userspace iothread until it can be fully replaced by upstream.

Before merging into QEMU, we should at least make sure that it's not going to create a nightmare for Avi :-)

Quoted 8 lines
Quoted 3 lines

Do you think we can eliminate the io threading code in kvm-userspace after this goes in?

Not immediately, need to generalize some of the changes introduced by the patchset and merge the remaining logic of kvm-userspace's qemu-kvm.c.

Yeah, I thought it would take some work. Great work Marcelo!

Regards,

Anthony Liguori

#12 lore raw

[Qemu-devel] Re: [patch 2/7] qemu: separate thread for io

Quoted 1 line · Anthony Liguori

Before merging into QEMU, we should at least make sure that it's not going to create a nightmare for Avi :-)

While it's likely going to be bad, we shouldn't make too much effort in making the merge easy. We should aim to get the cleanest implementation in qemu, and then rework kvm-userspace to match that. That's the whole point of the merge, no?

Of course, if it happens to be easy I won't complain too much.

Signature · 1 line
error compiling committee.c: too many arguments to function
#13 lore raw

[Qemu-devel] Re: [patch 2/7] qemu: separate thread for io

Quoted 4 lines · Avi Kivity
Quoted 1 line · Anthony Liguori

Before merging into QEMU, we should at least make sure that it's not going to create a nightmare for Avi :-)

While it's likely going to be bad, we shouldn't make too much effort in making the merge easy. We should aim to get the cleanest implementation in qemu, and then rework kvm-userspace to match that. That's the whole point of the merge, no?

The effort to make merging easy should be in kvm-userspace, not in qemu. Thinking through how it would integrate with kvm-userspace is a good exercise in determining whether all of the requirements are being met for KVM too.

Regards,

Anthony Liguori

Quoted 2 lines

Of course, if it happens to be easy I won't complain too much.

#14 lore raw

PATCH 3/7 qemu: main thread does io and cpu thread is spawned

In reply to [email protected]
vl.c +30 -23
Index: qemu/vl.c
===================================================================
@@ -3623,12 +3623,13 @@ static int wait_signal(int timeout)
 
 static int has_work(CPUState *env)
 {
-    int r = 0;
+    if (!vm_running)
+        return 0;
     if (!env->halted)
-        r = 1;
+        return 1;
     if (env->interrupt_request & (CPU_INTERRUPT_HARD | CPU_INTERRUPT_NMI))
-        r = 1;
-    return r;
+        return 1;
+    return 0;
 }
 
 static void qemu_wait_io_event(CPUState *env, int timeout)
@@ -3737,14 +3738,6 @@ void main_loop_break(void)
     qemu_thread_signal(&io_thread, SIG_IPI);
 }
 
-static void *io_thread_fn(void *arg)
-{
-    unblock_io_signals();
-    qemu_mutex_lock(&qemu_global_mutex);
-    while (1)
-        main_loop_wait(1000);
-}
-
 static void qemu_signal_lock(unsigned int msecs)
 {
     qemu_mutex_lock(&qemu_fair_mutex);
@@ -3842,9 +3835,11 @@ void main_loop_wait(int timeout)
 #endif
 
     /* vm time timers */
-    if (vm_running && likely(!(cur_cpu->singlestep_enabled & SSTEP_NOTIMER)))
-        qemu_run_timers(&active_timers[QEMU_TIMER_VIRTUAL],
-                        qemu_get_clock(vm_clock));
+    if (vm_running) {
+        if (cur_cpu && likely(!(cur_cpu->singlestep_enabled & SSTEP_NOTIMER)))
+            qemu_run_timers(&active_timers[QEMU_TIMER_VIRTUAL],
+                             qemu_get_clock(vm_clock));
+    }
 
     /* real time timers */
     qemu_run_timers(&active_timers[QEMU_TIMER_REALTIME],
@@ -3856,7 +3851,7 @@ void main_loop_wait(int timeout)
 
 }
 
-static int main_loop(void)
+static void *cpu_main_loop(void *arg)
 {
     int ret, timeout;
 #ifdef CONFIG_PROFILER
@@ -3864,15 +3859,11 @@ static int main_loop(void)
 #endif
     CPUState *env;
 
-    qemu_mutex_init(&qemu_fair_mutex);
-    qemu_mutex_init(&qemu_global_mutex);
-    qemu_mutex_lock(&qemu_global_mutex);
-
-    qemu_thread_create(&io_thread, io_thread_fn, NULL);
     block_io_signals();
-
     qemu_thread_self(&cpus_thread);
 
+    qemu_mutex_lock(&qemu_global_mutex);
+
     cur_cpu = first_cpu;
     next_cpu = cur_cpu->next_cpu ?: first_cpu;
     for(;;) {
@@ -4011,7 +4002,23 @@ static int main_loop(void)
 #endif
     }
     cpu_disable_ticks();
-    return ret;
+    return NULL;
+}
+
+static void main_loop(void)
+{
+    qemu_mutex_init(&qemu_fair_mutex);
+    qemu_mutex_init(&qemu_global_mutex);
+    qemu_mutex_lock(&qemu_global_mutex);
+
+    qemu_thread_self(&io_thread);
+
+    unblock_io_signals();
+
+    qemu_thread_create(&cpus_thread, cpu_main_loop, NULL);
+
+    while (1)
+        main_loop_wait(1000);
 }
 
 static void help(int exitcode)
#15 lore raw

PATCH 4/7 qemu: handle reset/poweroff/shutdown in iothread

In reply to [email protected]

Its simpler to handle these events from only one context.

vl.c +14 -30
Index: qemu/vl.c
===================================================================
@@ -3540,22 +3540,19 @@ void qemu_system_reset_request(void)
     } else {
         reset_requested = 1;
     }
-    if (cpu_single_env)
-        cpu_interrupt(cpu_single_env, CPU_INTERRUPT_EXIT);
+    main_loop_break();
 }
 
 void qemu_system_shutdown_request(void)
 {
     shutdown_requested = 1;
-    if (cpu_single_env)
-        cpu_interrupt(cpu_single_env, CPU_INTERRUPT_EXIT);
+    main_loop_break();
 }
 
 void qemu_system_powerdown_request(void)
 {
     powerdown_requested = 1;
-    if (cpu_single_env)
-        cpu_interrupt(cpu_single_env, CPU_INTERRUPT_EXIT);
+    main_loop_break();
 }
 
 #ifdef _WIN32
@@ -3921,25 +3918,6 @@ static void *cpu_main_loop(void *arg)
             }
             cur_cpu = env;
 
-            if (shutdown_requested) {
-                ret = EXCP_INTERRUPT;
-                if (no_shutdown) {
-                    vm_stop(0);
-                    no_shutdown = 0;
-                }
-                else
-                    break;
-            }
-            if (reset_requested) {
-                reset_requested = 0;
-                qemu_system_reset();
-                ret = EXCP_INTERRUPT;
-            }
-            if (powerdown_requested) {
-                powerdown_requested = 0;
-		qemu_system_powerdown();
-                ret = EXCP_INTERRUPT;
-            }
             if (unlikely(ret == EXCP_DEBUG)) {
                 gdb_set_stop_cpu(cur_cpu);
                 vm_stop(EXCP_DEBUG);
@@ -3987,10 +3965,6 @@ static void *cpu_main_loop(void *arg)
                 timeout = 0;
             }
         } else {
-            if (shutdown_requested) {
-                ret = EXCP_INTERRUPT;
-                break;
-            }
             timeout = 5000;
         }
 #ifdef CONFIG_PROFILER
@@ -4017,8 +3991,18 @@ static void main_loop(void)
 
     qemu_thread_create(&cpus_thread, cpu_main_loop, NULL);
 
-    while (1)
+    while (1) {
         main_loop_wait(1000);
+        if (qemu_shutdown_requested()) {
+            if (no_shutdown)
+                no_shutdown = 0;
+            else
+                    break;
+        } else if (qemu_powerdown_requested())
+            qemu_system_powerdown();
+        else if (qemu_reset_requested())
+            qemu_system_reset();
+    }
 }
 
 static void help(int exitcode)
#16 lore raw

PATCH 5/7 qemu: pause and resume cpu thread(s)

In reply to [email protected]

Since cpu emulation happens on a separate thread, it is necessary to pause/resume it upon certain events such as reset, debug exception, live migration, etc.

vl.c +81 -3
Index: qemu/vl.c
===================================================================
@@ -271,6 +271,11 @@ QemuMutex qemu_fair_mutex;
 QemuThread io_thread;
 QemuThread cpus_thread;
 
+QemuCond qemu_pause_cond;
+
+static void pause_all_vcpus(void);
+static void resume_all_vcpus(void);
+
 /***********************************************************/
 /* x86 ISA bus support */
 
@@ -3461,6 +3466,7 @@ void vm_start(void)
     if (!vm_running) {
         cpu_enable_ticks();
         vm_running = 1;
+        resume_all_vcpus();
         vm_state_notify(1, 0);
         qemu_rearm_alarm_timer(alarm_timer);
     }
@@ -3471,6 +3477,7 @@ void vm_stop(int reason)
     if (vm_running) {
         cpu_disable_ticks();
         vm_running = 0;
+        pause_all_vcpus();
         vm_state_notify(0, reason);
     }
 }
@@ -3620,7 +3627,9 @@ static int wait_signal(int timeout)
 
 static int has_work(CPUState *env)
 {
-    if (!vm_running)
+    if (env->stop)
+        return 1;
+    if (!vm_running || env->stopped)
         return 0;
     if (!env->halted)
         return 1;
@@ -3644,6 +3653,11 @@ static void qemu_wait_io_event(CPUState 
     qemu_mutex_unlock(&qemu_fair_mutex);
 
     qemu_mutex_lock(&qemu_global_mutex);
+    if (env->stop) {
+        env->stop = 0;
+        env->stopped = 1;
+        qemu_cond_signal(&qemu_pause_cond);
+    }
 }
 
 static void cpu_signal(int sig)
@@ -3848,6 +3862,21 @@ void main_loop_wait(int timeout)
 
 }
 
+static int vm_can_run(CPUState *env)
+{
+    if (env->stop)
+        return 0;
+    if (env->stopped)
+        return 0;
+    if (shutdown_requested)
+        return 0;
+    if (powerdown_requested)
+        return 0;
+    if (reset_requested)
+        return 0;
+    return 1;
+}
+
 static void *cpu_main_loop(void *arg)
 {
     int ret, timeout;
@@ -3887,7 +3916,9 @@ static void *cpu_main_loop(void *arg)
                     env->icount_decr.u16.low = decr;
                     env->icount_extra = count;
                 }
-                ret = cpu_exec(env);
+                ret = EXCP_HALTED;
+                if (vm_can_run(env))
+                    ret = cpu_exec(env);
 #ifdef CONFIG_PROFILER
                 qemu_time += profile_getclock() - ti;
 #endif
@@ -3965,6 +3996,7 @@ static void *cpu_main_loop(void *arg)
                 timeout = 0;
             }
         } else {
+            env = env->next_cpu ?: first_cpu;
             timeout = 5000;
         }
 #ifdef CONFIG_PROFILER
@@ -3979,11 +4011,53 @@ static void *cpu_main_loop(void *arg)
     return NULL;
 }
 
+static int all_vcpus_paused(void)
+{
+    CPUState *penv = first_cpu;
+
+    while (penv) {
+        if (penv->stop)
+            return 0;
+        penv = (CPUState *)penv->next_cpu;
+    }
+
+    return 1;
+}
+
+static void pause_all_vcpus(void)
+{
+    CPUState *penv = first_cpu;
+
+    while (penv) {
+        penv->stop = 1;
+        qemu_thread_signal(&cpus_thread, SIGUSR1);
+        penv = (CPUState *)penv->next_cpu;
+    }
+
+    while (!all_vcpus_paused()) {
+        qemu_cond_timedwait(&qemu_pause_cond, &qemu_global_mutex, 100);
+        qemu_thread_signal(&cpus_thread, SIGUSR1);
+    }
+}
+
+static void resume_all_vcpus(void)
+{
+    CPUState *penv = first_cpu;
+
+    while (penv) {
+        penv->stop = 0;
+        penv->stopped = 0;
+        qemu_thread_signal(&cpus_thread, SIGUSR1);
+        penv = (CPUState *)penv->next_cpu;
+    }
+}
+
 static void main_loop(void)
 {
     qemu_mutex_init(&qemu_fair_mutex);
     qemu_mutex_init(&qemu_global_mutex);
     qemu_mutex_lock(&qemu_global_mutex);
+    qemu_cond_init(&qemu_pause_cond);
 
     qemu_thread_self(&io_thread);
 
@@ -3994,14 +4068,18 @@ static void main_loop(void)
     while (1) {
         main_loop_wait(1000);
         if (qemu_shutdown_requested()) {
+            pause_all_vcpus();
             if (no_shutdown)
                 no_shutdown = 0;
             else
                     break;
         } else if (qemu_powerdown_requested())
             qemu_system_powerdown();
-        else if (qemu_reset_requested())
+        else if (qemu_reset_requested()) {
+            pause_all_vcpus();
             qemu_system_reset();
+            resume_all_vcpus();
+        }
     }
 }
 
cpu-defs.h +2 -0
Index: qemu/cpu-defs.h
===================================================================
@@ -169,6 +169,8 @@ typedef struct CPUWatchpoint {
     target_ulong mem_io_vaddr; /* target virtual addr at which the      \
                                      memory was accessed */             \
     uint32_t halted; /* Nonzero if the CPU is in suspend state */       \
+    uint32_t stop;   /* Stop request */                                 \
+    uint32_t stopped; /* Artificially stopped */                        \
     uint32_t interrupt_request;                                         \
     /* The meaning of the MMU modes is defined in the target code. */   \
     CPUTLBEntry tlb_table[NB_MMU_MODES][CPU_TLB_SIZE];                  \
#17 lore raw

PATCH 6/7 qemu: handle vmstop from cpu context

In reply to [email protected]

There are certain cases where cpu context requests a vm stop, such as -ENOSPC handling.

IMO its simpler to handle vmstop only through the iothread.

Note there is change in behaviour: now the cpu thread which requested vm_stop will actually only stop when it exits back to cpu_main_loop. It might cause further damage in between vmstop and cpu_main_loop.

CC Gleb Natapov <[email protected]>
vl.c +39 -3
Index: qemu/vl.c
===================================================================
@@ -3472,7 +3472,22 @@ void vm_start(void)
     }
 }
 
-void vm_stop(int reason)
+static int vmstop_requested;
+
+static int qemu_vmstop_requested(void)
+{
+    int r = vmstop_requested;
+    vmstop_requested = 0;
+    return r;
+}
+
+static void qemu_system_vmstop_request(int reason)
+{
+    vmstop_requested = reason;
+    main_loop_break();
+}
+
+static void __vm_stop(int reason)
 {
     if (vm_running) {
         cpu_disable_ticks();
@@ -3482,6 +3497,21 @@ void vm_stop(int reason)
     }
 }
 
+void vm_stop(int reason)
+{
+    QemuThread me;
+    qemu_thread_self(&me);
+
+    if (!qemu_thread_equal(&me, &io_thread)) {
+        qemu_system_vmstop_request(reason);
+        /* make sure we ourselves can't return to cpu_exec */
+        if (cpu_single_env)
+            cpu_single_env->stop = 1;
+        return;
+    }
+    __vm_stop(reason);
+}
+
 /* reset/shutdown handler */
 
 typedef struct QEMUResetEntry {
@@ -3874,6 +3904,8 @@ static int vm_can_run(CPUState *env)
         return 0;
     if (reset_requested)
         return 0;
+    if (vmstop_requested)
+        return 0;
     return 1;
 }
 
@@ -4054,6 +4086,8 @@ static void resume_all_vcpus(void)
 
 static void main_loop(void)
 {
+    int r;
+
     qemu_mutex_init(&qemu_fair_mutex);
     qemu_mutex_init(&qemu_global_mutex);
     qemu_mutex_lock(&qemu_global_mutex);
@@ -4073,12 +4107,14 @@ static void main_loop(void)
                 no_shutdown = 0;
             else
                     break;
-        } else if (qemu_powerdown_requested())
+        } else if (qemu_powerdown_requested()) {
             qemu_system_powerdown();
-        else if (qemu_reset_requested()) {
+        } else if (qemu_reset_requested()) {
             pause_all_vcpus();
             qemu_system_reset();
             resume_all_vcpus();
+        } else if ((r = qemu_vmstop_requested())) {
+            vm_stop(r);
         }
     }
 }
#18 lore raw

PATCH 7/7 qemu: use pipe to wakeup io thread

In reply to [email protected]

This avoids a wakeup being lost, in case the iothread handles the signal before select().

Copied and possibly damaged from kvm-userspace.

vl.c +61 -18
Index: qemu/vl.c
===================================================================
@@ -155,8 +155,6 @@
 //#define DEBUG_NET
 //#define DEBUG_SLIRP
 
-#define SIG_IPI (SIGRTMIN+4)
-
 #ifdef DEBUG_IOPORT
 #  define LOG_IOPORT(...) qemu_log_mask(CPU_LOG_IOPORT, ## __VA_ARGS__)
 #else
@@ -273,6 +271,8 @@ QemuThread cpus_thread;
 
 QemuCond qemu_pause_cond;
 
+static int io_thread_fd = -1;
+
 static void pause_all_vcpus(void);
 static void resume_all_vcpus(void);
 
@@ -3723,7 +3723,6 @@ static void block_io_signals(void)
     sigaddset(&set, SIGUSR2);
     sigaddset(&set, SIGIO);
     sigaddset(&set, SIGALRM);
-    sigaddset(&set, SIG_IPI);
     pthread_sigmask(SIG_BLOCK, &set, NULL);
 
     sigemptyset(&set);
@@ -3735,30 +3734,19 @@ static void block_io_signals(void)
     sigaction(SIGUSR1, &sigact, NULL);
 }
 
-/* used to wake up the io thread */
-static void sig_ipi_handler(int sig)
-{
-}
-
 static void unblock_io_signals(void)
 {
     sigset_t set;
-    struct sigaction sigact;
 
     sigemptyset(&set);
     sigaddset(&set, SIGUSR2);
     sigaddset(&set, SIGIO);
     sigaddset(&set, SIGALRM);
-    sigaddset(&set, SIG_IPI);
     pthread_sigmask(SIG_UNBLOCK, &set, NULL);
 
     sigemptyset(&set);
     sigaddset(&set, SIGUSR1);
     pthread_sigmask(SIG_BLOCK, &set, NULL);
-
-    memset(&sigact, 0, sizeof(sigact));
-    sigact.sa_handler = sig_ipi_handler;
-    sigaction(SIG_IPI, &sigact, NULL);
 }
 
 int qemu_notify_event(void *cpu_env)
@@ -3774,11 +3762,51 @@ int qemu_notify_event(void *cpu_env)
 
 void main_loop_break(void)
 {
-    QemuThread me;
-    qemu_thread_self(&me);
-    if (qemu_thread_equal(&io_thread, &me))
+    uint64_t value = 1;
+    char buffer[8];
+    size_t offset = 0;
+
+    if (io_thread_fd == -1)
         return;
-    qemu_thread_signal(&io_thread, SIG_IPI);
+
+    memcpy(buffer, &value, sizeof(value));
+
+    while (offset < 8) {
+        ssize_t len;
+
+        len = write(io_thread_fd, buffer + offset, 8 - offset);
+        if (len == -1 && errno == EINTR)
+            continue;
+
+        if (len <= 0)
+            break;
+
+        offset += len;
+    }
+
+    if (offset != 8)
+        fprintf(stderr, "failed to notify io thread\n");
+}
+
+/* Used to break IO thread out of select */
+static void io_thread_wakeup(void *opaque)
+{
+    int fd = (unsigned long)opaque;
+    char buffer[8];
+    size_t offset = 0;
+
+    while (offset < 8) {
+        ssize_t len;
+
+        len = read(fd, buffer + offset, 8 - offset);
+        if (len == -1 && errno == EINTR)
+            continue;
+
+        if (len <= 0)
+            break;
+
+        offset += len;
+    }
 }
 
 static void qemu_signal_lock(unsigned int msecs)
@@ -4086,6 +4114,20 @@ static void resume_all_vcpus(void)
     }
 }
 
+static void setup_iothread_fd(void)
+{
+    int fds[2];
+
+    if (pipe(fds) == -1) {
+        fprintf(stderr, "failed to create iothread pipe");
+        exit(0);
+    }
+
+    qemu_set_fd_handler2(fds[0], NULL, io_thread_wakeup, NULL,
+                         (void *)(unsigned long)fds[0]);
+    io_thread_fd = fds[1];
+}
+
 static void main_loop(void)
 {
     int r;
@@ -4096,6 +4138,7 @@ static void main_loop(void)
     qemu_cond_init(&qemu_pause_cond);
 
     qemu_thread_self(&io_thread);
+    setup_iothread_fd();
 
     unblock_io_signals();