dataplane: add query-iothreads QMP command
cover dataplane: add query-iothreads QMP command
This series applies on top of "[PATCH v4 0/6] dataplane: switch to N:M devices-per-thread model".
The new "query-iothreads" command allows QMP clients to fetch the thread IDs for dataplane threads. This will allow libvirt and friends to bind dataplane threads.
The approach is similar to "query-cpus" which is also used for vcpu thread binding (among other things).
Stefan Hajnoczi (2):
iothread: stash thread ID away
qmp: add query-iothreads command
iothread.c | 70 ++++++++++++++++++++++++++++++++++++++++++++++++++++++-- qapi-schema.json | 29 +++++++++++++++++++++++ qmp-commands.hx | 39 +++++++++++++++++++++++++++++++ 3 files changed, 136 insertions(+), 2 deletions(-)
Signature · 1 line
1.8.5.3
PATCH 1/2 iothread: stash thread ID away
Keep the thread ID around so we can report it via QMP.
There's only one problem: qemu_get_thread_id() (gettid() wrapper on Linux) must be called from the thread itself. There is no way to get the thread ID outside the thread.
This patch uses a condvar to wait for iothread_run() to populate the thread_id inside the thread.
--- iothread.c | 34 ++++++++++++++++++++++++++++++++-- 1 file changed, 32 insertions(+), 2 deletions(-)
iothread.c +32 -2
@@ -26,8 +26,15 @@ struct IOThread {QemuThread thread;AioContext *ctx;bool stopping; +int thread_id; }; +typedef struct { +QemuMutex init_done_lock; +QemuCond init_done_cond;/* is thread initialization done? */ +IOThread *iothread; +}ThreadInitInfo ; +#define IOTHREAD_GET_CLASS (obj) \ OBJECT_GET_CLASS(IOThreadClass, obj, TYPE_IOTHREAD)#define IOTHREAD_CLASS (klass) \
@@ -35,7 +42,15 @@ struct IOThread {static void *iothread_run (void *opaque) { -IOThread *iothread = opaque; +ThreadInitInfo *init_info = opaque; +IOThread *iothread = init_info->iothread; + + iothread->thread_id =qemu_get_thread_id (); + +/* Signal that initialization is done */ +qemu_mutex_lock (&init_info->init_done_lock); +qemu_cond_signal (&init_info->init_done_cond); +qemu_mutex_unlock (&init_info->init_done_lock);while (!iothread->stopping) {aio_context_acquire (iothread->ctx);
@@ -60,15 +75,30 @@ static void iothread_instance_finalize(Object *obj)static void iothread_complete (UserCreatable *obj,Error **errp) {IOThread *iothread =IOTHREAD (obj); +ThreadInitInfo init_info = { + .iothread = iothread, + }; iothread->stopping = false; iothread->ctx =aio_context_new (); +qemu_mutex_init (&init_info.init_done_lock); +qemu_cond_init (&init_info.init_done_cond); +/* This assumes we are called from a thread with useful CPU affinity for us * to inherit. */ qemu_thread_create (&iothread->thread, iothread_run, - iothread,QEMU_THREAD_JOINABLE ); + &init_info,QEMU_THREAD_JOINABLE ); + +/* Wait for initialization to complete */ +qemu_mutex_lock (&init_info.init_done_lock); +qemu_cond_wait (&init_info.init_done_cond, + &init_info.init_done_lock); +qemu_mutex_unlock (&init_info.init_done_lock); + +qemu_cond_destroy (&init_info.init_done_cond); +qemu_mutex_destroy (&init_info.init_done_lock); }static void iothread_class_init (ObjectClass *klass,void *class_data)
Signature · 1 line
1.8.5.3
Re: [Qemu-devel] [PATCH 1/2] iothread: stash thread ID away
Il 21/02/2014 15:51, Stefan Hajnoczi ha scritto:
Quoted 70 lines
Keep the thread ID around so we can report it via QMP.
There's only one problem:
qemu_get_thread_id()(gettid()wrapper on Linux) must be called from the thread itself. There is no way to get the thread ID outside the thread.This patch uses a condvar to wait for
iothread_run()to populate thethread_idinside the thread.Signed-off-by Stefan Hajnoczi <[email protected]>--- iothread.c | 34 ++++++++++++++++++++++++++++++++-- 1 file changed, 32 insertions(+), 2 deletions(-)iothread.c +22
-2@@ -26,8 +26,15 @@ struct IOThread {QemuThread thread;AioContext *ctx;bool stopping; +int thread_id; }; +typedef struct { +QemuMutex init_done_lock; +QemuCond init_done_cond;/* is thread initialization done? */ +IOThread *iothread; +}ThreadInitInfo ; +#define IOTHREAD_GET_CLASS (obj) \ OBJECT_GET_CLASS(IOThreadClass, obj, TYPE_IOTHREAD)#define IOTHREAD_CLASS (klass) \@@ -35,7 +42,15 @@ struct IOThread {static void *iothread_run (void *opaque) { -IOThread *iothread = opaque; +ThreadInitInfo *init_info = opaque; +IOThread *iothread = init_info->iothread; + + iothread->thread_id =qemu_get_thread_id (); + +/* Signal that initialization is done */ +qemu_mutex_lock (&init_info->init_done_lock); +qemu_cond_signal (&init_info->init_done_cond); +qemu_mutex_unlock (&init_info->init_done_lock);while (!iothread->stopping) {aio_context_acquire (iothread->ctx);@@ -60,15 +75,30 @@ static void iothread_instance_finalize(Object *obj)static void iothread_complete (UserCreatable *obj,Error **errp) {IOThread *iothread =IOTHREAD (obj); +ThreadInitInfo init_info = { + .iothread = iothread, + }; iothread->stopping = false; iothread->ctx =aio_context_new (); +qemu_mutex_init (&init_info.init_done_lock); +qemu_cond_init (&init_info.init_done_cond); +/* This assumes we are called from a thread with useful CPU affinity for us * to inherit. */ qemu_thread_create(&iothread->thread, iothread_run, - iothread,QEMU_THREAD_JOINABLE );
iothread.c static void iothread_instance_finalize(Object *obj) · line 101
Quoted 10 lines
iothread.c +10
-0+ &init_info, QEMU_THREAD_JOINABLE); + +/* Wait for initialization to complete */ +qemu_mutex_lock (&init_info.init_done_lock); +qemu_cond_wait (&init_info.init_done_cond, + &init_info.init_done_lock); +qemu_mutex_unlock (&init_info.init_done_lock); + +qemu_cond_destroy (&init_info.init_done_cond); +qemu_mutex_destroy (&init_info.init_done_lock);
Destroying the mutex here is racy. You need to keep it until the iothread is destroyed.
Paolo
Quoted 4 lines
iothread.c +0
-0}static void iothread_class_init (ObjectClass *klass,void *class_data)
Re: [Qemu-devel] [PATCH 1/2] iothread: stash thread ID away
Quoted 76 lines · Paolo Bonzini
Il 21/02/2014 15:51, Stefan Hajnoczi ha scritto:
Quoted 75 lines
Keep the thread ID around so we can report it via QMP.
There's only one problem:
qemu_get_thread_id()(gettid()wrapper on Linux) must be called from the thread itself. There is no way to get the thread ID outside the thread.This patch uses a condvar to wait for
iothread_run()to populate thethread_idinside the thread.Signed-off-by Stefan Hajnoczi <[email protected]>---
iothread.c| 34 ++++++++++++++++++++++++++++++++--1 file changed, 32 insertions(+), 2 deletions(-)iothread.c +27
-2@@ -26,8 +26,15 @@ struct IOThread {QemuThread thread;AioContext *ctx;bool stopping; +int thread_id; }; +typedef struct { +QemuMutex init_done_lock; +QemuCond init_done_cond;/* is thread initialization done? */ +IOThread *iothread; +}ThreadInitInfo ; +#define IOTHREAD_GET_CLASS (obj) \ OBJECT_GET_CLASS(IOThreadClass, obj, TYPE_IOTHREAD)#define IOTHREAD_CLASS (klass) \@@ -35,7 +42,15 @@ struct IOThread {static void *iothread_run (void *opaque) { -IOThread *iothread = opaque; +ThreadInitInfo *init_info = opaque; +IOThread *iothread = init_info->iothread; + + iothread->thread_id =qemu_get_thread_id (); + +/* Signal that initialization is done */ +qemu_mutex_lock (&init_info->init_done_lock); +qemu_cond_signal (&init_info->init_done_cond); +qemu_mutex_unlock (&init_info->init_done_lock);while (!iothread->stopping) {aio_context_acquire (iothread->ctx);@@ -60,15 +75,30 @@ static void iothread_instance_finalize(Object *obj)static void iothread_complete (UserCreatable *obj,Error **errp) {IOThread *iothread =IOTHREAD (obj); +ThreadInitInfo init_info = { + .iothread = iothread, + }; iothread->stopping = false; iothread->ctx =aio_context_new (); +qemu_mutex_init (&init_info.init_done_lock); +qemu_cond_init (&init_info.init_done_cond); +/* This assumes we are called from a thread with useful CPU affinity for us * to inherit. */ qemu_thread_create (&iothread->thread, iothread_run, - iothread,QEMU_THREAD_JOINABLE ); + &init_info,QEMU_THREAD_JOINABLE ); + +/* Wait for initialization to complete */ +qemu_mutex_lock (&init_info.init_done_lock); + qemu_cond_wait(&init_info.init_done_cond,
Quoted 8 lines
iothread.c static void iothread_instance_finalize(Object *obj) · line 101
Quoted 5 lines
iothread.c +5
-0+ &init_info.init_done_lock); +qemu_mutex_unlock (&init_info.init_done_lock); + +qemu_cond_destroy (&init_info.init_done_cond); +qemu_mutex_destroy (&init_info.init_done_lock);Destroying the mutex here is racy. You need to keep it until the iothread is destroyed.
I don't think so:
qemu_cond_signal() is called with the mutex held. Therefore, our qemu_cond_wait() followed by qemu_mutex_unlock() will only complete once the thread has released the mutex.
The thread will never touch the mutex again so it is safe to destroy it. There is no race condition.
Stefan
Re: [Qemu-devel] [PATCH 1/2] iothread: stash thread ID away
Il 24/02/2014 16:53, Stefan Hajnoczi ha scritto:
Quoted 5 lines
Quoted 5 lines
Quoted 5 lines
Quoted 2 lines
iothread.c static void iothread_instance_finalize(Object *obj) · line 101
Quoted 2 lines
iothread.c +2
-0+qemu_cond_destroy (&init_info.init_done_cond); +qemu_mutex_destroy (&init_info.init_done_lock);Destroying the mutex here is racy. You need to keep it until the iothread is destroyed.
Quoted 8 lines
I don't think so:
qemu_cond_signal()is called with the mutex held. Therefore, ourqemu_cond_wait()followed byqemu_mutex_unlock()will only complete once the thread has released the mutex.The thread will never touch the mutex again so it is safe to destroy it. There is no race condition.
Could qemu_mutex_destroy run while the other thread has already released the main thread, but before it returns? As far as I know, the only time when it is safe to destroy the "last" synchronization object (in this case the mutex is the last, the condvar is not) is after pthread_join.
Paolo
Re: [Qemu-devel] [PATCH 1/2] iothread: stash thread ID away
Quoted 12 lines · Paolo Bonzini
Il 24/02/2014 16:53, Stefan Hajnoczi ha scritto:
Quoted 11 lines
Quoted 5 lines
Quoted 5 lines
Quoted 2 lines
iothread.c static void iothread_instance_finalize(Object *obj) · line 101
Quoted 2 lines
iothread.c +2
-0+qemu_cond_destroy (&init_info.init_done_cond); +qemu_mutex_destroy (&init_info.init_done_lock);Destroying the mutex here is racy. You need to keep it until the iothread is destroyed.
I don't think so:
qemu_cond_signal()is called with the mutex held. Therefore, ourqemu_cond_wait()followed byqemu_mutex_unlock()will only complete once the thread has released the mutex.
Quoted 8 lines
Quoted 2 lines
The thread will never touch the mutex again so it is safe to destroy it. There is no race condition.
Could
qemu_mutex_destroyrun while the other thread has already released the main thread, but before it returns? As far as I know, the only time when it is safe to destroy the "last" synchronization object (in this case the mutex is the last, the condvar is not) is afterpthread_join.
I guess you're saying that while unlocking the mutex is atomic, that doesn't guarantee pthread won't access the mutex internal state some more after it has unlocked it. Therefore it's not safe for another thread to destroy the mutex even after it has acquired it.
POSIX does say that:
"It shall be safe to destroy an initialized mutex that is unlocked."
But maybe I am reading too much into that?
After poking around glibc a little I think you are right. I can't say for sure but it seems even after a futex call glibc might still mess with internal state. But if anyone knows for certain, please speak up.
Stefan
Re: [Qemu-devel] [PATCH 1/2] iothread: stash thread ID away
Il 25/02/2014 16:42, Stefan Hajnoczi ha scritto:
Quoted 4 lines
I guess you're saying that while unlocking the mutex is atomic, that doesn't guarantee pthread won't access the mutex internal state some more after it has unlocked it. Therefore it's not safe for another thread to destroy the mutex even after it has acquired it.
Yes.
Quoted 3 lines
POSIX does say that:
"It shall be safe to destroy an initialized mutex that is unlocked."
The question is what "unlocked" means... :)
Quoted 5 lines
But maybe I am reading too much into that?
After poking around glibc a little I think you are right. I can't say for sure but it seems even after a futex call glibc might still mess with internal state. But if anyone knows for certain, please speak up.
I think other races are possible. Let's look at the simple lock in nptl/lowlevellock.h:
/* Mutex lock counter:
bit 31 clear means unlocked;
bit 31 set means locked.
All code that looks at bit 31 first increases the 'number of interested threads' usage counter, which is in bits 0-30.
The comment is wrong, there is a fast path that does not do that; I'm not sure if this is why the problem can happen, I'm just pointing this out because it contradicts the code I'm posting now.
The file uses C code, but it's simpler to look at it in assembly. Unlocking is very simple:
lock; btcl $31, futex jz 2f ... do futex wake ... 2:
Locking has a fast path followed by preparing the slow path, re-checking the fastpath condition, and waiting if it fails still:
lock; btsl $31, futex jnc 9f lock; incl futex 1: lock; btsl $31, futex jnc 8f ... do futex wait ... jmp 1b 8: lock; decl futex 9:
It's possible, if futex is locked by CPU 0 and CPU 1 tries to grab it, that the following happens:
CPU 0 CPU 1
lock; btsl $31, futex (fails) lock; incl futex
lock; btcl %0 (not zero)
lock; btsl $31, futex (succeeds) lock; decl futex
destroy lock free(lock)
futex wake
If you get an EFAULT from the futex wakeup, this could be a problem.
Paolo
Re: [Qemu-devel] [PATCH 1/2] iothread: stash thread ID away
In reply to Paolo BonziniQuoted 12 lines · Paolo Bonzini
Il 24/02/2014 16:53, Stefan Hajnoczi ha scritto:
Quoted 11 lines
Quoted 5 lines
Quoted 5 lines
Quoted 2 lines
iothread.c static void iothread_instance_finalize(Object *obj) · line 101
Quoted 2 lines
iothread.c +2
-0+qemu_cond_destroy (&init_info.init_done_cond); +qemu_mutex_destroy (&init_info.init_done_lock);Destroying the mutex here is racy. You need to keep it until the iothread is destroyed.
I don't think so:
qemu_cond_signal()is called with the mutex held. Therefore, ourqemu_cond_wait()followed byqemu_mutex_unlock()will only complete once the thread has released the mutex.
Quoted 8 lines
Quoted 2 lines
The thread will never touch the mutex again so it is safe to destroy it. There is no race condition.
Could
qemu_mutex_destroyrun while the other thread has already released the main thread, but before it returns? As far as I know, the only time when it is safe to destroy the "last" synchronization object (in this case the mutex is the last, the condvar is not) is afterpthread_join.
For the default mutex type (PTHREAD_MUTEX_TIMED_NP) glibc looks safe to me. The other mutex types are trickier and I haven't audited them.
Anyway, I can just move the mutex into the IOThread object and destroy it after the thread is joined :).
Stefan
Re: [Qemu-devel] [PATCH 1/2] iothread: stash thread ID away
Il 25/02/2014 17:17, Stefan Hajnoczi ha scritto:
Quoted 2 lines
For the default mutex type (
PTHREAD_MUTEX_TIMED_NP) glibc looks safe to me. The other mutex types are trickier and I haven't audited them.
It also depends on the low-level lock implementation. I looked at the C one and it's not safe, the x86-optimized one is hard to follow.
I think I was looking at a different race, namely cases where the locking thread uses a fast path, while the unlocking thread plays it safe and uses the slow path instead. Then the slow path can run "asynchronously" from the locking thread, and the locking thread has time to unlock and destroy the mutex. See the other message.
Paolo
PATCH 2/2 qmp: add query-iothreads command
In reply to Stefan HajnocziThe "query-iothreads" command returns a list of information about iothreads. See the patch for API documentation.
--- iothread.c | 36 ++++++++++++++++++++++++++++++++++++ qapi-schema.json | 29 +++++++++++++++++++++++++++++ qmp-commands.hx | 39 +++++++++++++++++++++++++++++++++++++++ 3 files changed, 104 insertions(+)
iothread.c +36 -0
@@ -17,6 +17,7 @@#include "qemu/thread.h" #include "block/aio.h" #include "sysemu/iothread.h" +#include "qmp-commands.h" #define IOTHREADS_PATH "/objects"
@@ -151,3 +152,38 @@ AioContext *iothread_get_aio_context(IOThread *iothread) {return iothread->ctx; } + +static int query_one_iothread (Object *object,void *opaque) +{ +IOThreadInfoList ***prev = opaque; +IOThreadInfoList *elem; +IOThreadInfo *info; +IOThread *iothread; + + iothread = (IOThread *)object_dynamic_cast (object,TYPE_IOTHREAD ); +if (!iothread) { +return 0 ; + } + + info =g_new0 (IOThreadInfo,1 ); + info->id =iothread_get_id (iothread); + info->thread_id = iothread->thread_id; + + elem =g_new0 (IOThreadInfoList,1 ); + elem->value = info; + elem->next =NULL ; + + **prev = elem; + *prev = &elem->next; +return 0 ; +} + +IOThreadInfoList *qmp_query_iothreads (Error **errp) +{ +IOThreadInfoList *head =NULL ; +IOThreadInfoList **prev = &head; +Object *container =container_get (object_get_root (),IOTHREADS_PATH ); + +object_child_foreach (container, query_one_iothread, &prev); +return head; +}
qapi-schema.json +29 -0
@@ -884,6 +884,35 @@ { 'command': 'query-cpus', 'returns': ['CpuInfo'] } ## +# @IOThreadInfo: +# +# Information about an iothread +# +# @id: the identifier of the iothread +# +# @thread_id: ID of the underlying host thread +# +# Since: 2.0 +## +{ 'type': 'IOThreadInfo', + 'data': {'id': 'str', 'thread_id': 'int'} } + +## +# @query-iothreads: +# +# Returns a list of information about each iothread. +# +# Note this list excludes the QEMU main loop thread, which is not declared +# using the -object iothread command-line option. It is always the main thread +# of the process. +# +# Returns: a list of @IOThreadInfo for each iothread +# +# Since: 2.0 +## +{ 'command': 'query-iothreads', 'returns': ['IOThreadInfo'] } + +## # @BlockDeviceInfo: # # Information about the backing device for a block device.
qmp-commands.hx +39 -0
@@ -2304,6 +2304,45 @@ EQMP }, SQMP +query-iothreads +--------------- + +Returns a list of information about each iothread. + +Note this list excludes the QEMU main loop thread, which is not declared +using the -object iothread command-line option. It is always the main thread +of the process. + +Return a json-array. Each iothread is represented by a json-object, which contains: + +- "id": name of iothread (json-str) +- "thread_id": ID of the underlying host thread (json-int) + +Example: + +-> { "execute": "query-iothreads" } +<- { + "return":[ + { + "id":"iothread0", + "thread_id":3134 + }, + { + "id":"iothread1", + "thread_id":3135 + } + ] + } + +EQMP + + { + .name = "query-iothreads", + .args_type = "", + .mhandler.cmd_new = qmp_marshal_input_query_iothreads, + }, + +SQMP query-pci ---------
Signature · 1 line
1.8.5.3
Re: [Qemu-devel] [PATCH 2/2] qmp: add query-iothreads command
Quoted 8 lines · Stefan Hajnoczi
The "query-iothreads" command returns a list of information about iothreads. See the patch for API documentation.
Signed-off-by Stefan Hajnoczi <[email protected]>---
+++
b/qapi-schema.json
qapi-schema.json line 893
Quoted 10 lines
qapi-schema.json +7
-0@@ -884,6 +884,35 @@ { 'command': 'query-cpus', 'returns': ['CpuInfo'] } ## +# @IOThreadInfo: +# +# Information about an iothread +# +# @id: the identifier of the iothread +# +# @thread_id: ID of the underlying host thread
For new interfaces, we prefer thread-id (dash, not underscore).
But the idea definitely makes sense.
Signature · 2 lines
Eric Blake eblake redhat com +1-919-301-3266 Libvirt virtualization library http://libvirt.org
Re: [Qemu-devel] [PATCH 2/2] qmp: add query-iothreads command
Quoted 11 lines · Eric Blake
Quoted 10 lines · Stefan Hajnoczi
The "query-iothreads" command returns a list of information about iothreads. See the patch for API documentation.
Signed-off-by Stefan Hajnoczi <[email protected]>---
+++
b/qapi-schema.jsonqapi-schema.json +0
-0@@ -884,6 +884,35 @@ { 'command': 'query-cpus', 'returns': ['CpuInfo'] }
Quoted 10 lines
qapi-schema.json line 893
Quoted 8 lines
qapi-schema.json +7
-0## +# @IOThreadInfo: +# +# Information about an iothread +# +# @id: the identifier of the iothread +# +# @thread_id: ID of the underlying host threadFor new interfaces, we prefer thread-id (dash, not underscore).
Will fix in v2.
Stefan