Skip to contentSkip to conversation

dataplane: add query-iothreads QMP command

PATCH Superseded Stefan Hajnoczi 4 participants 12 messages Feb 21, 2014 – Feb 25, 2014

v1 of 3 Superseded Latest is v3 →

lore
Share & tools
lore
#0 lore raw

cover dataplane: add query-iothreads QMP command

This series applies on top of "[PATCH v4 0/6] dataplane: switch to N:M devices-per-thread model".

The new "query-iothreads" command allows QMP clients to fetch the thread IDs for dataplane threads. This will allow libvirt and friends to bind dataplane threads.

The approach is similar to "query-cpus" which is also used for vcpu thread binding (among other things).

Stefan Hajnoczi (2):
iothread: stash thread ID away
qmp: add query-iothreads command

 iothread.c       | 70 ++++++++++++++++++++++++++++++++++++++++++++++++++++++--
 qapi-schema.json | 29 +++++++++++++++++++++++
 qmp-commands.hx  | 39 +++++++++++++++++++++++++++++++
 3 files changed, 136 insertions(+), 2 deletions(-)
Signature · 1 line
1.8.5.3
#1 lore raw

PATCH 1/2 iothread: stash thread ID away

Keep the thread ID around so we can report it via QMP.

There's only one problem: qemu_get_thread_id() (gettid() wrapper on Linux) must be called from the thread itself. There is no way to get the thread ID outside the thread.

This patch uses a condvar to wait for iothread_run() to populate the thread_id inside the thread.

---
 iothread.c | 34 ++++++++++++++++++++++++++++++++--
 1 file changed, 32 insertions(+), 2 deletions(-)
iothread.c +32 -2
index 033de7f..1a0d1ec 100644
@@ -26,8 +26,15 @@ struct IOThread {
     QemuThread thread;
     AioContext *ctx;
     bool stopping;
+    int thread_id;
 };
 
+typedef struct {
+    QemuMutex init_done_lock;
+    QemuCond init_done_cond;    /* is thread initialization done? */
+    IOThread *iothread;
+} ThreadInitInfo;
+
 #define IOTHREAD_GET_CLASS(obj) \
    OBJECT_GET_CLASS(IOThreadClass, obj, TYPE_IOTHREAD)
 #define IOTHREAD_CLASS(klass) \
@@ -35,7 +42,15 @@ struct IOThread {
 
 static void *iothread_run(void *opaque)
 {
-    IOThread *iothread = opaque;
+    ThreadInitInfo *init_info = opaque;
+    IOThread *iothread = init_info->iothread;
+
+    iothread->thread_id = qemu_get_thread_id();
+
+    /* Signal that initialization is done */
+    qemu_mutex_lock(&init_info->init_done_lock);
+    qemu_cond_signal(&init_info->init_done_cond);
+    qemu_mutex_unlock(&init_info->init_done_lock);
 
     while (!iothread->stopping) {
         aio_context_acquire(iothread->ctx);
@@ -60,15 +75,30 @@ static void iothread_instance_finalize(Object *obj)
 static void iothread_complete(UserCreatable *obj, Error **errp)
 {
     IOThread *iothread = IOTHREAD(obj);
+    ThreadInitInfo init_info = {
+        .iothread = iothread,
+    };
 
     iothread->stopping = false;
     iothread->ctx = aio_context_new();
 
+    qemu_mutex_init(&init_info.init_done_lock);
+    qemu_cond_init(&init_info.init_done_cond);
+
     /* This assumes we are called from a thread with useful CPU affinity for us
      * to inherit.
      */
     qemu_thread_create(&iothread->thread, iothread_run,
-                       iothread, QEMU_THREAD_JOINABLE);
+                       &init_info, QEMU_THREAD_JOINABLE);
+
+    /* Wait for initialization to complete */
+    qemu_mutex_lock(&init_info.init_done_lock);
+    qemu_cond_wait(&init_info.init_done_cond,
+                   &init_info.init_done_lock);
+    qemu_mutex_unlock(&init_info.init_done_lock);
+
+    qemu_cond_destroy(&init_info.init_done_cond);
+    qemu_mutex_destroy(&init_info.init_done_lock);
 }
 
 static void iothread_class_init(ObjectClass *klass, void *class_data)
Signature · 1 line
1.8.5.3
#2 lore raw

Re: [Qemu-devel] [PATCH 1/2] iothread: stash thread ID away

Il 21/02/2014 15:51, Stefan Hajnoczi ha scritto:

Quoted 70 lines

Keep the thread ID around so we can report it via QMP.

There's only one problem: qemu_get_thread_id() (gettid() wrapper on Linux) must be called from the thread itself. There is no way to get the thread ID outside the thread.

This patch uses a condvar to wait for iothread_run() to populate the thread_id inside the thread.

---
 iothread.c | 34 ++++++++++++++++++++++++++++++++--
 1 file changed, 32 insertions(+), 2 deletions(-)
iothread.c +22 -2
index 033de7f..1a0d1ec 100644
@@ -26,8 +26,15 @@ struct IOThread {
     QemuThread thread;
     AioContext *ctx;
     bool stopping;
+    int thread_id;
 };
 
+typedef struct {
+    QemuMutex init_done_lock;
+    QemuCond init_done_cond;    /* is thread initialization done? */
+    IOThread *iothread;
+} ThreadInitInfo;
+
 #define IOTHREAD_GET_CLASS(obj) \
    OBJECT_GET_CLASS(IOThreadClass, obj, TYPE_IOTHREAD)
 #define IOTHREAD_CLASS(klass) \
@@ -35,7 +42,15 @@ struct IOThread {
 
 static void *iothread_run(void *opaque)
 {
-    IOThread *iothread = opaque;
+    ThreadInitInfo *init_info = opaque;
+    IOThread *iothread = init_info->iothread;
+
+    iothread->thread_id = qemu_get_thread_id();
+
+    /* Signal that initialization is done */
+    qemu_mutex_lock(&init_info->init_done_lock);
+    qemu_cond_signal(&init_info->init_done_cond);
+    qemu_mutex_unlock(&init_info->init_done_lock);
 
     while (!iothread->stopping) {
         aio_context_acquire(iothread->ctx);
@@ -60,15 +75,30 @@ static void iothread_instance_finalize(Object *obj)
 static void iothread_complete(UserCreatable *obj, Error **errp)
 {
     IOThread *iothread = IOTHREAD(obj);
+    ThreadInitInfo init_info = {
+        .iothread = iothread,
+    };
 
     iothread->stopping = false;
     iothread->ctx = aio_context_new();
 
+    qemu_mutex_init(&init_info.init_done_lock);
+    qemu_cond_init(&init_info.init_done_cond);
+
     /* This assumes we are called from a thread with useful CPU affinity for us
      * to inherit.
      */
     qemu_thread_create(&iothread->thread, iothread_run,
-                       iothread, QEMU_THREAD_JOINABLE);

iothread.c static void iothread_instance_finalize(Object *obj) · line 101

Quoted 10 lines
iothread.c +10 -0
+                       &init_info, QEMU_THREAD_JOINABLE);
+
+    /* Wait for initialization to complete */
+    qemu_mutex_lock(&init_info.init_done_lock);
+    qemu_cond_wait(&init_info.init_done_cond,
+                   &init_info.init_done_lock);
+    qemu_mutex_unlock(&init_info.init_done_lock);
+
+    qemu_cond_destroy(&init_info.init_done_cond);
+    qemu_mutex_destroy(&init_info.init_done_lock);

Destroying the mutex here is racy. You need to keep it until the iothread is destroyed.

Paolo

Quoted 4 lines
iothread.c +0 -0
 }
 
 static void iothread_class_init(ObjectClass *klass, void *class_data)
#3 lore raw

Re: [Qemu-devel] [PATCH 1/2] iothread: stash thread ID away

Quoted 76 lines · Paolo Bonzini

Il 21/02/2014 15:51, Stefan Hajnoczi ha scritto:

Quoted 75 lines

Keep the thread ID around so we can report it via QMP.

There's only one problem: qemu_get_thread_id() (gettid() wrapper on Linux) must be called from the thread itself. There is no way to get the thread ID outside the thread.

This patch uses a condvar to wait for iothread_run() to populate the thread_id inside the thread.

---
iothread.c | 34 ++++++++++++++++++++++++++++++++--

1 file changed, 32 insertions(+), 2 deletions(-)
iothread.c +27 -2
index 033de7f..1a0d1ec 100644
@@ -26,8 +26,15 @@ struct IOThread {
    QemuThread thread;
    AioContext *ctx;
    bool stopping;
+    int thread_id;
 };
 
+typedef struct {
+    QemuMutex init_done_lock;
+    QemuCond init_done_cond;    /* is thread initialization done? */
+    IOThread *iothread;
+} ThreadInitInfo;
+
 #define IOTHREAD_GET_CLASS(obj) \
   OBJECT_GET_CLASS(IOThreadClass, obj, TYPE_IOTHREAD)
 #define IOTHREAD_CLASS(klass) \
@@ -35,7 +42,15 @@ struct IOThread {
 
 static void *iothread_run(void *opaque)
 {
-    IOThread *iothread = opaque;
+    ThreadInitInfo *init_info = opaque;
+    IOThread *iothread = init_info->iothread;
+
+    iothread->thread_id = qemu_get_thread_id();
+
+    /* Signal that initialization is done */
+    qemu_mutex_lock(&init_info->init_done_lock);
+    qemu_cond_signal(&init_info->init_done_cond);
+    qemu_mutex_unlock(&init_info->init_done_lock);
 
    while (!iothread->stopping) {
        aio_context_acquire(iothread->ctx);
@@ -60,15 +75,30 @@ static void iothread_instance_finalize(Object *obj)
 static void iothread_complete(UserCreatable *obj, Error **errp)
 {
    IOThread *iothread = IOTHREAD(obj);
+    ThreadInitInfo init_info = {
+        .iothread = iothread,
+    };
 
    iothread->stopping = false;
    iothread->ctx = aio_context_new();
 
+    qemu_mutex_init(&init_info.init_done_lock);
+    qemu_cond_init(&init_info.init_done_cond);
+
    /* This assumes we are called from a thread with useful CPU affinity for us
     * to inherit.
     */
    qemu_thread_create(&iothread->thread, iothread_run,
-                       iothread, QEMU_THREAD_JOINABLE);
+                       &init_info, QEMU_THREAD_JOINABLE);
+
+    /* Wait for initialization to complete */
+    qemu_mutex_lock(&init_info.init_done_lock);
+    qemu_cond_wait(&init_info.init_done_cond,
Quoted 8 lines

iothread.c static void iothread_instance_finalize(Object *obj) · line 101

Quoted 5 lines
iothread.c +5 -0
+                   &init_info.init_done_lock);
+    qemu_mutex_unlock(&init_info.init_done_lock);
+
+    qemu_cond_destroy(&init_info.init_done_cond);
+    qemu_mutex_destroy(&init_info.init_done_lock);

Destroying the mutex here is racy. You need to keep it until the iothread is destroyed.

I don't think so:

qemu_cond_signal() is called with the mutex held. Therefore, our qemu_cond_wait() followed by qemu_mutex_unlock() will only complete once the thread has released the mutex.

The thread will never touch the mutex again so it is safe to destroy it. There is no race condition.

Stefan

#4 lore raw

Re: [Qemu-devel] [PATCH 1/2] iothread: stash thread ID away

Il 24/02/2014 16:53, Stefan Hajnoczi ha scritto:

Quoted 5 lines
Quoted 5 lines
Quoted 5 lines
Quoted 2 lines

iothread.c static void iothread_instance_finalize(Object *obj) · line 101

Quoted 2 lines
iothread.c +2 -0
+    qemu_cond_destroy(&init_info.init_done_cond);
+    qemu_mutex_destroy(&init_info.init_done_lock);

Destroying the mutex here is racy. You need to keep it until the iothread is destroyed.

Quoted 8 lines

I don't think so:

qemu_cond_signal() is called with the mutex held. Therefore, our qemu_cond_wait() followed by qemu_mutex_unlock() will only complete once the thread has released the mutex.

The thread will never touch the mutex again so it is safe to destroy it. There is no race condition.

Could qemu_mutex_destroy run while the other thread has already released the main thread, but before it returns? As far as I know, the only time when it is safe to destroy the "last" synchronization object (in this case the mutex is the last, the condvar is not) is after pthread_join.

Paolo

#5 lore raw

Re: [Qemu-devel] [PATCH 1/2] iothread: stash thread ID away

Quoted 12 lines · Paolo Bonzini

Il 24/02/2014 16:53, Stefan Hajnoczi ha scritto:

Quoted 11 lines
Quoted 5 lines
Quoted 5 lines
Quoted 2 lines

iothread.c static void iothread_instance_finalize(Object *obj) · line 101

Quoted 2 lines
iothread.c +2 -0
+    qemu_cond_destroy(&init_info.init_done_cond);
+    qemu_mutex_destroy(&init_info.init_done_lock);

Destroying the mutex here is racy. You need to keep it until the iothread is destroyed.

I don't think so:

qemu_cond_signal() is called with the mutex held. Therefore, our qemu_cond_wait() followed by qemu_mutex_unlock() will only complete once the thread has released the mutex.

Quoted 8 lines
Quoted 2 lines

The thread will never touch the mutex again so it is safe to destroy it. There is no race condition.

Could qemu_mutex_destroy run while the other thread has already released the main thread, but before it returns? As far as I know, the only time when it is safe to destroy the "last" synchronization object (in this case the mutex is the last, the condvar is not) is after pthread_join.

I guess you're saying that while unlocking the mutex is atomic, that doesn't guarantee pthread won't access the mutex internal state some more after it has unlocked it. Therefore it's not safe for another thread to destroy the mutex even after it has acquired it.

POSIX does say that:

"It shall be safe to destroy an initialized mutex that is unlocked."

But maybe I am reading too much into that?

After poking around glibc a little I think you are right. I can't say for sure but it seems even after a futex call glibc might still mess with internal state. But if anyone knows for certain, please speak up.

Stefan

#6 lore raw

Re: [Qemu-devel] [PATCH 1/2] iothread: stash thread ID away

Il 25/02/2014 16:42, Stefan Hajnoczi ha scritto:

Quoted 4 lines

I guess you're saying that while unlocking the mutex is atomic, that doesn't guarantee pthread won't access the mutex internal state some more after it has unlocked it. Therefore it's not safe for another thread to destroy the mutex even after it has acquired it.

Yes.

Quoted 3 lines

POSIX does say that:

"It shall be safe to destroy an initialized mutex that is unlocked."

The question is what "unlocked" means... :)

Quoted 5 lines

But maybe I am reading too much into that?

After poking around glibc a little I think you are right. I can't say for sure but it seems even after a futex call glibc might still mess with internal state. But if anyone knows for certain, please speak up.

I think other races are possible. Let's look at the simple lock in nptl/lowlevellock.h:

/* Mutex lock counter:
bit 31 clear means unlocked;
bit 31 set means locked.

All code that looks at bit 31 first increases the 'number of interested threads' usage counter, which is in bits 0-30.

The comment is wrong, there is a fast path that does not do that; I'm not sure if this is why the problem can happen, I'm just pointing this out because it contradicts the code I'm posting now.

The file uses C code, but it's simpler to look at it in assembly. Unlocking is very simple:

		lock; btcl $31, futex
	        jz 2f
		... do futex wake ...
	2:

Locking has a fast path followed by preparing the slow path, re-checking the fastpath condition, and waiting if it fails still:

		lock; btsl $31, futex
		jnc 9f
		lock; incl futex
	1:
		lock; btsl $31, futex
		jnc 8f
		... do futex wait ...
		jmp 1b
	8:
		lock; decl futex
	9:

It's possible, if futex is locked by CPU 0 and CPU 1 tries to grab it, that the following happens:

CPU 0 CPU 1
lock; btsl $31, futex (fails) lock; incl futex
lock; btcl %0 (not zero)
lock; btsl $31, futex (succeeds) lock; decl futex

destroy lock free(lock)
futex wake

If you get an EFAULT from the futex wakeup, this could be a problem.

Paolo

#7 lore raw

Re: [Qemu-devel] [PATCH 1/2] iothread: stash thread ID away

In reply to Paolo Bonzini
Quoted 12 lines · Paolo Bonzini

Il 24/02/2014 16:53, Stefan Hajnoczi ha scritto:

Quoted 11 lines
Quoted 5 lines
Quoted 5 lines
Quoted 2 lines

iothread.c static void iothread_instance_finalize(Object *obj) · line 101

Quoted 2 lines
iothread.c +2 -0
+    qemu_cond_destroy(&init_info.init_done_cond);
+    qemu_mutex_destroy(&init_info.init_done_lock);

Destroying the mutex here is racy. You need to keep it until the iothread is destroyed.

I don't think so:

qemu_cond_signal() is called with the mutex held. Therefore, our qemu_cond_wait() followed by qemu_mutex_unlock() will only complete once the thread has released the mutex.

Quoted 8 lines
Quoted 2 lines

The thread will never touch the mutex again so it is safe to destroy it. There is no race condition.

Could qemu_mutex_destroy run while the other thread has already released the main thread, but before it returns? As far as I know, the only time when it is safe to destroy the "last" synchronization object (in this case the mutex is the last, the condvar is not) is after pthread_join.

For the default mutex type (PTHREAD_MUTEX_TIMED_NP) glibc looks safe to me. The other mutex types are trickier and I haven't audited them.

Anyway, I can just move the mutex into the IOThread object and destroy it after the thread is joined :).

Stefan

#8 lore raw

Re: [Qemu-devel] [PATCH 1/2] iothread: stash thread ID away

Il 25/02/2014 17:17, Stefan Hajnoczi ha scritto:

Quoted 2 lines

For the default mutex type (PTHREAD_MUTEX_TIMED_NP) glibc looks safe to me. The other mutex types are trickier and I haven't audited them.

It also depends on the low-level lock implementation. I looked at the C one and it's not safe, the x86-optimized one is hard to follow.

I think I was looking at a different race, namely cases where the locking thread uses a fast path, while the unlocking thread plays it safe and uses the slow path instead. Then the slow path can run "asynchronously" from the locking thread, and the locking thread has time to unlock and destroy the mutex. See the other message.

Paolo

#9 lore raw

PATCH 2/2 qmp: add query-iothreads command

In reply to Stefan Hajnoczi

The "query-iothreads" command returns a list of information about iothreads. See the patch for API documentation.

---
 iothread.c       | 36 ++++++++++++++++++++++++++++++++++++
 qapi-schema.json | 29 +++++++++++++++++++++++++++++
 qmp-commands.hx  | 39 +++++++++++++++++++++++++++++++++++++++
 3 files changed, 104 insertions(+)
iothread.c +36 -0
index 1a0d1ec..7aa8463 100644
@@ -17,6 +17,7 @@
 #include "qemu/thread.h"
 #include "block/aio.h"
 #include "sysemu/iothread.h"
+#include "qmp-commands.h"
 
 #define IOTHREADS_PATH "/objects"
 
@@ -151,3 +152,38 @@ AioContext *iothread_get_aio_context(IOThread *iothread)
 {
     return iothread->ctx;
 }
+
+static int query_one_iothread(Object *object, void *opaque)
+{
+    IOThreadInfoList ***prev = opaque;
+    IOThreadInfoList *elem;
+    IOThreadInfo *info;
+    IOThread *iothread;
+
+    iothread = (IOThread *)object_dynamic_cast(object, TYPE_IOTHREAD);
+    if (!iothread) {
+        return 0;
+    }
+
+    info = g_new0(IOThreadInfo, 1);
+    info->id = iothread_get_id(iothread);
+    info->thread_id = iothread->thread_id;
+
+    elem = g_new0(IOThreadInfoList, 1);
+    elem->value = info;
+    elem->next = NULL;
+
+    **prev = elem;
+    *prev = &elem->next;
+    return 0;
+}
+
+IOThreadInfoList *qmp_query_iothreads(Error **errp)
+{
+    IOThreadInfoList *head = NULL;
+    IOThreadInfoList **prev = &head;
+    Object *container = container_get(object_get_root(), IOTHREADS_PATH);
+
+    object_child_foreach(container, query_one_iothread, &prev);
+    return head;
+}
qapi-schema.json +29 -0
index 473c096..6795a01 100644
@@ -884,6 +884,35 @@
 { 'command': 'query-cpus', 'returns': ['CpuInfo'] }
 
 ##
+# @IOThreadInfo:
+#
+# Information about an iothread
+#
+# @id: the identifier of the iothread
+#
+# @thread_id: ID of the underlying host thread
+#
+# Since: 2.0
+##
+{ 'type': 'IOThreadInfo',
+  'data': {'id': 'str', 'thread_id': 'int'} }
+
+##
+# @query-iothreads:
+#
+# Returns a list of information about each iothread.
+#
+# Note this list excludes the QEMU main loop thread, which is not declared
+# using the -object iothread command-line option.  It is always the main thread
+# of the process.
+#
+# Returns: a list of @IOThreadInfo for each iothread
+#
+# Since: 2.0
+##
+{ 'command': 'query-iothreads', 'returns': ['IOThreadInfo'] }
+
+##
 # @BlockDeviceInfo:
 #
 # Information about the backing device for a block device.
qmp-commands.hx +39 -0
index 8a0e832..ba16b61 100644
@@ -2304,6 +2304,45 @@ EQMP
     },
 
 SQMP
+query-iothreads
+---------------
+
+Returns a list of information about each iothread.
+
+Note this list excludes the QEMU main loop thread, which is not declared
+using the -object iothread command-line option.  It is always the main thread
+of the process.
+
+Return a json-array. Each iothread is represented by a json-object, which contains:
+
+- "id": name of iothread (json-str)
+- "thread_id": ID of the underlying host thread (json-int)
+
+Example:
+
+-> { "execute": "query-iothreads" }
+<- {
+      "return":[
+         {
+            "id":"iothread0",
+            "thread_id":3134
+         },
+         {
+            "id":"iothread1",
+            "thread_id":3135
+         }
+      ]
+   }
+
+EQMP
+
+    {
+        .name       = "query-iothreads",
+        .args_type  = "",
+        .mhandler.cmd_new = qmp_marshal_input_query_iothreads,
+    },
+
+SQMP
 query-pci
 ---------
 
Signature · 1 line
1.8.5.3
#10 lore raw

Re: [Qemu-devel] [PATCH 2/2] qmp: add query-iothreads command

Quoted 8 lines · Stefan Hajnoczi

The "query-iothreads" command returns a list of information about iothreads. See the patch for API documentation.

---

+++ b/qapi-schema.json

qapi-schema.json line 893

Quoted 10 lines
qapi-schema.json +7 -0
@@ -884,6 +884,35 @@
 { 'command': 'query-cpus', 'returns': ['CpuInfo'] }
 
 ##
+# @IOThreadInfo:
+#
+# Information about an iothread
+#
+# @id: the identifier of the iothread
+#
+# @thread_id: ID of the underlying host thread

For new interfaces, we prefer thread-id (dash, not underscore).

But the idea definitely makes sense.

Signature · 2 lines
Eric Blake   eblake redhat com    +1-919-301-3266
Libvirt virtualization library http://libvirt.org
#11 lore raw

Re: [Qemu-devel] [PATCH 2/2] qmp: add query-iothreads command

Quoted 11 lines · Eric Blake
Quoted 10 lines · Stefan Hajnoczi

The "query-iothreads" command returns a list of information about iothreads. See the patch for API documentation.

---

+++ b/qapi-schema.json

qapi-schema.json +0 -0
@@ -884,6 +884,35 @@
 { 'command': 'query-cpus', 'returns': ['CpuInfo'] }
 
Quoted 10 lines

qapi-schema.json line 893

Quoted 8 lines
qapi-schema.json +7 -0
 ##
+# @IOThreadInfo:
+#
+# Information about an iothread
+#
+# @id: the identifier of the iothread
+#
+# @thread_id: ID of the underlying host thread

For new interfaces, we prefer thread-id (dash, not underscore).

Will fix in v2.

Stefan