Skip to contentSkip to conversation

coroutine: Fix qemu_coroutine_yield()

PATCH Applied Kevin Wolf 4 participants 12 messages Feb 10, 2015 – Feb 20, 2015

lore
Share & tools
lore

Kevin Wolf

author
#0 lore raw

cover coroutine: Fix qemu_coroutine_yield()

Kevin Wolf (2):
coroutine: Fix use after free with qemu_coroutine_yield()
coroutine: Clean up qemu_coroutine_enter()

Stefan Hajnoczi (1):
test-coroutine: Regression test for yield bug

 include/block/coroutine_int.h |  1 +
 qemu-coroutine.c              | 38 ++++++++++++++++----------------------
 tests/test-coroutine.c        | 26 ++++++++++++++++++++++++++
 3 files changed, 43 insertions(+), 22 deletions(-)
Signature · 1 line
1.8.3.1
author
#1 lore raw

PATCH 1/3 coroutine: Fix use after free with qemu_coroutine_yield()

Instead of using the same function for entering and exiting coroutines, and hoping that it doesn't add any functionality that hurts with the parameters used for exiting, we can just directly call into the real task switch in qemu_coroutine_switch().

This fixes a use-after-free scenario where reentering a coroutine that has yielded still accesses the old parent coroutine (which may have meanwhile terminated) in the part of coroutine_swap() that follows qemu_coroutine_switch().

---
 qemu-coroutine.c | 2 +-
 1 file changed, 1 insertion(+), 1 deletion(-)
qemu-coroutine.c +1 -1
index 525247b..5019b81 100644
@@ -148,5 +148,5 @@ void coroutine_fn qemu_coroutine_yield(void)
     }
 
     self->caller = NULL;
-    coroutine_swap(self, to);
+    qemu_coroutine_switch(self, to, COROUTINE_YIELD);
 }
Signature · 1 line
1.8.3.1
author
#2 lore raw

Re: [Qemu-devel] [PATCH 1/3] coroutine: Fix use after free with qemu_coroutine_yield()

Am 10.02.2015 um 11:41 hat Kevin Wolf geschrieben:

Quoted 11 lines

Instead of using the same function for entering and exiting coroutines, and hoping that it doesn't add any functionality that hurts with the parameters used for exiting, we can just directly call into the real task switch in qemu_coroutine_switch().

This fixes a use-after-free scenario where reentering a coroutine that has yielded still accesses the old parent coroutine (which may have meanwhile terminated) in the part of coroutine_swap() that follows qemu_coroutine_switch().

Thanks to Peter for noticing that I forgot this.

Kevin

author
#3 lore raw

PATCH 2/3 coroutine: Clean up qemu_coroutine_enter()

In reply to Kevin Wolf

qemu_coroutine_enter() is now the only user of coroutine_swap(). Both functions are short, so inline it.

Also, using COROUTINE_YIELD is now even more confusing because this code is never called during qemu_coroutine_yield() any more. In fact, this value is never read back, so we can just introduce a new COROUTINE_ENTER which documents the purpose of the task switch better.

---
 include/block/coroutine_int.h |  1 +
 qemu-coroutine.c              | 36 +++++++++++++++---------------------
 2 files changed, 16 insertions(+), 21 deletions(-)
include/block/coroutine_int.h +1 -0
index f133d65..69b83db 100644
@@ -29,6 +29,7 @@
 #include "block/coroutine.h"
 
 typedef enum {
+    COROUTINE_ENTER = 0,
     COROUTINE_YIELD = 1,
     COROUTINE_TERMINATE = 2,
 } CoroutineAction;
qemu-coroutine.c +15 -21
index 5019b81..c17a92b 100644
@@ -99,29 +99,10 @@ static void coroutine_delete(Coroutine *co)
     qemu_coroutine_delete(co);
 }
 
-static void coroutine_swap(Coroutine *from, Coroutine *to)
-{
-    CoroutineAction ret;
-
-    ret = qemu_coroutine_switch(from, to, COROUTINE_YIELD);
-
-    qemu_co_queue_run_restart(to);
-
-    switch (ret) {
-    case COROUTINE_YIELD:
-        return;
-    case COROUTINE_TERMINATE:
-        trace_qemu_coroutine_terminate(to);
-        coroutine_delete(to);
-        return;
-    default:
-        abort();
-    }
-}
-
 void qemu_coroutine_enter(Coroutine *co, void *opaque)
 {
     Coroutine *self = qemu_coroutine_self();
+    CoroutineAction ret;
 
     trace_qemu_coroutine_enter(self, co, opaque);
 
@@ -132,7 +113,20 @@ void qemu_coroutine_enter(Coroutine *co, void *opaque)
 
     co->caller = self;
     co->entry_arg = opaque;
-    coroutine_swap(self, co);
+    ret = qemu_coroutine_switch(self, co, COROUTINE_ENTER);
+
+    qemu_co_queue_run_restart(co);
+
+    switch (ret) {
+    case COROUTINE_YIELD:
+        return;
+    case COROUTINE_TERMINATE:
+        trace_qemu_coroutine_terminate(co);
+        coroutine_delete(co);
+        return;
+    default:
+        abort();
+    }
 }
 
 void coroutine_fn qemu_coroutine_yield(void)
Signature · 1 line
1.8.3.1
#4 lore raw

Re: [Qemu-devel] [PATCH 2/3] coroutine: Clean up qemu_coroutine_enter()

qemu-coroutine.c void qemu_coroutine_enter(Coroutine *co, void *opaque) · line 127

Quoted 12 lines · Kevin Wolf
qemu-coroutine.c +12 -0
+    ret = qemu_coroutine_switch(self, co, COROUTINE_ENTER);
+
+    qemu_co_queue_run_restart(co);
+
+    switch (ret) {
+    case COROUTINE_YIELD:
+        return;
+    case COROUTINE_TERMINATE:
+        trace_qemu_coroutine_terminate(co);
+        coroutine_delete(co);
+        return;
+    default:

Say you have:

co1 co2
------------------------------------------------------------------------
1 qemu_co_mutex_lock(&m);
2 qemu_coroutine_yield();
3 qemu_co_mutex_lock(&m);
4 qemu_co_mutex_unlock(&m);
5 qemu_coroutine_yield();

Then you have:

1 mutex->locked = true;

2 coroutine_swap(co1, leader, COROUTINE_YIELD);

3 while (mutex->locked) {
     qemu_co_queue_wait(&mutex->queue);
           '--> QTAILQ_INSERT_TAIL(&queue->entries, self, co_queue_next);
                qemu_coroutine_yield();
                '--> coroutine_swap(co2, leader, COROUTINE_YIELD);

}

4 mutex->locked = false;
qemu_co_queue_next(&mutex->queue);

   '--> qemu_co_queue_do_restart(queue, true);
        '--> QTAILQ_REMOVE(&queue->entries, next, co_queue_next);
             QTAILQ_INSERT_TAIL(&self->co_queue_wakeup, next, co_queue_next);

5 coroutine_swap(co1, leader, COROUTINE_YIELD);

And co2 is never reentered until co1 terminates. Right?

Paolo

author
#5 lore raw

Re: [Qemu-devel] [PATCH 2/3] coroutine: Clean up qemu_coroutine_enter()

Am 10.02.2015 um 11:55 hat Paolo Bonzini geschrieben:

Quoted 40 lines · Kevin Wolf

qemu-coroutine.c void qemu_coroutine_enter(Coroutine *co, void *opaque) · line 127

Quoted 12 lines
qemu-coroutine.c +12 -0
+    ret = qemu_coroutine_switch(self, co, COROUTINE_ENTER);
+
+    qemu_co_queue_run_restart(co);
+
+    switch (ret) {
+    case COROUTINE_YIELD:
+        return;
+    case COROUTINE_TERMINATE:
+        trace_qemu_coroutine_terminate(co);
+        coroutine_delete(co);
+        return;
+    default:

Say you have:

co1 co2
------------------------------------------------------------------------
1 qemu_co_mutex_lock(&m);
2 qemu_coroutine_yield();
3 qemu_co_mutex_lock(&m);
4 qemu_co_mutex_unlock(&m);
5 qemu_coroutine_yield();

Then you have:

1 mutex->locked = true;

2 coroutine_swap(co1, leader, COROUTINE_YIELD);

3 while (mutex->locked) {
     qemu_co_queue_wait(&mutex->queue);
           '--> QTAILQ_INSERT_TAIL(&queue->entries, self, co_queue_next);
                qemu_coroutine_yield();
                '--> coroutine_swap(co2, leader, COROUTINE_YIELD);

}

4 mutex->locked = false;

Quoted 8 lines
  qemu_co_queue_next(&mutex->queue);
   '--> qemu_co_queue_do_restart(queue, true);
        '--> QTAILQ_REMOVE(&queue->entries, next, co_queue_next);
             QTAILQ_INSERT_TAIL(&self->co_queue_wakeup, next, co_queue_next);

5 coroutine_swap(co1, leader, COROUTINE_YIELD);

And co2 is never reentered until co1 terminates. Right?

No, co2 will be reentered during the yield in line 5. However, it's not the yielding coroutine that reenters it but the parent, which is resumed at exactly the line of code that you quoted above.

This is actually how it always worked, even with the bug. The bug caused it to access the queue of a random other coroutine, but that queue must have always been empty because it was already processed when that other coroutine yielded/terminated.

Kevin

Reviewed-by
#6 lore raw

Re: [Qemu-devel] [PATCH 2/3] coroutine: Clean up qemu_coroutine_enter()

Quoted 7 lines · Kevin Wolf
Quoted 7 lines
Quoted 7 lines

4 mutex->locked = false;

  qemu_co_queue_next(&mutex->queue);
   '--> qemu_co_queue_do_restart(queue, true);
        '--> QTAILQ_REMOVE(&queue->entries, next, co_queue_next);
             QTAILQ_INSERT_TAIL(&self->co_queue_wakeup, next, co_queue_next);
Quoted 6 lines
Quoted 3 lines
Quoted 3 lines

5 coroutine_swap(co1, leader, COROUTINE_YIELD);

And co2 is never reentered until co1 terminates. Right?

No, co2 will be reentered during the yield in line 5. However, it's not the yielding coroutine that reenters it but the parent, which is resumed at exactly the line of code that you quoted above.

So:

5 coroutine_swap(co1, leader, COROUTINE_YIELD);
'--> jumps back to qemu_coroutine_switch

       '--> returns to qemu_coroutine_enter
  qemu_co_queue_run_restart(co);
  '--> QTAILQ_REMOVE(&co->co_queue_wakeup, next, co_queue_next);
       qemu_coroutine_enter(next, NULL);

Thanks for the explanation. Series:

#7 lore raw

Re: [Qemu-devel] [PATCH 2/3] coroutine: Clean up qemu_coroutine_enter()

In reply to Kevin Wolf
Quoted 19 lines · Kevin Wolf

qemu_coroutine_enter() is now the only user of coroutine_swap(). Both functions are short, so inline it.

Also, using COROUTINE_YIELD is now even more confusing because this code is never called during qemu_coroutine_yield() any more. In fact, this value is never read back, so we can just introduce a new COROUTINE_ENTER which documents the purpose of the task switch better.

---
 include/block/coroutine_int.h |  1 +
 qemu-coroutine.c              | 36 +++++++++++++++---------------------
 2 files changed, 16 insertions(+), 21 deletions(-)
include/block/coroutine_int.h +0 -0
index f133d65..69b83db 100644

include/block/coroutine_int.h line 32

Quoted 4 lines
include/block/coroutine_int.h +1 -0
@@ -29,6 +29,7 @@
 #include "block/coroutine.h"
 
 typedef enum {
+    COROUTINE_ENTER = 0,

This makes the ucontext code harder to understand because CoroutineAction values are used with setjmp()/longjmp() in qemu_coroutine_switch().

The longjmp() man page says:

If longjmp() is invoked with a second argument of 0, 1 will be returned instead.

I haven't checked whether or not this causes problems, but the code would be simpler if we avoided using 0.

author
#8 lore raw

Re: [Qemu-devel] [PATCH 2/3] coroutine: Clean up qemu_coroutine_enter()

Am 18.02.2015 um 14:50 hat Stefan Hajnoczi geschrieben:

Quoted 29 lines · Kevin Wolf
Quoted 19 lines

qemu_coroutine_enter() is now the only user of coroutine_swap(). Both functions are short, so inline it.

Also, using COROUTINE_YIELD is now even more confusing because this code is never called during qemu_coroutine_yield() any more. In fact, this value is never read back, so we can just introduce a new COROUTINE_ENTER which documents the purpose of the task switch better.

---
 include/block/coroutine_int.h |  1 +
 qemu-coroutine.c              | 36 +++++++++++++++---------------------
 2 files changed, 16 insertions(+), 21 deletions(-)
include/block/coroutine_int.h +0 -0
index f133d65..69b83db 100644

include/block/coroutine_int.h line 32

Quoted 4 lines
include/block/coroutine_int.h +1 -0
@@ -29,6 +29,7 @@
 #include "block/coroutine.h"
 
 typedef enum {
+    COROUTINE_ENTER = 0,

This makes the ucontext code harder to understand because CoroutineAction values are used with setjmp()/longjmp() in qemu_coroutine_switch().

Quoted 7 lines

The longjmp() man page says:

If longjmp() is invoked with a second argument of 0, 1 will be returned instead.

I haven't checked whether or not this causes problems, but the code would be simpler if we avoided using 0.

It doesn't, the value is unused where we pass COROUTINE_ENTER. But I can make it 3 instead.

Kevin

#9 lore raw

Re: [Qemu-devel] [PATCH 2/3] coroutine: Clean up qemu_coroutine_enter()

Quoted 32 lines · Kevin Wolf

Am 18.02.2015 um 14:50 hat Stefan Hajnoczi geschrieben:

Quoted 31 lines · Kevin Wolf
Quoted 19 lines

qemu_coroutine_enter() is now the only user of coroutine_swap(). Both functions are short, so inline it.

Also, using COROUTINE_YIELD is now even more confusing because this code is never called during qemu_coroutine_yield() any more. In fact, this value is never read back, so we can just introduce a new COROUTINE_ENTER which documents the purpose of the task switch better.

---
 include/block/coroutine_int.h |  1 +
 qemu-coroutine.c              | 36 +++++++++++++++---------------------
 2 files changed, 16 insertions(+), 21 deletions(-)
include/block/coroutine_int.h +0 -0
index f133d65..69b83db 100644

include/block/coroutine_int.h line 32

Quoted 4 lines
include/block/coroutine_int.h +1 -0
@@ -29,6 +29,7 @@
 #include "block/coroutine.h"
 
 typedef enum {
+    COROUTINE_ENTER = 0,

This makes the ucontext code harder to understand because CoroutineAction values are used with setjmp()/longjmp() in qemu_coroutine_switch().

The longjmp() man page says:

Quoted 8 lines
Quoted 5 lines

If longjmp() is invoked with a second argument of 0, 1 will be returned instead.

I haven't checked whether or not this causes problems, but the code would be simpler if we avoided using 0.

It doesn't, the value is unused where we pass COROUTINE_ENTER. But I can make it 3 instead.

Thanks, that would be good.

Stefan

author
#10 lore raw

PATCH 3/3 test-coroutine: Regression test for yield bug

In reply to Kevin Wolf

From: Stefan Hajnoczi <[email protected]>

This adds a test for reentering a coroutine that previously yielded to a coroutine that has meanwhile terminated.

---
 tests/test-coroutine.c | 26 ++++++++++++++++++++++++++
 1 file changed, 26 insertions(+)
tests/test-coroutine.c +26 -0
index 27d1b6f..b552d9f 100644
@@ -13,6 +13,7 @@
 
 #include <glib.h>
 #include "block/coroutine.h"
+#include "block/coroutine_int.h"
 
 /*
  * Check that qemu_in_coroutine() works
@@ -122,6 +123,30 @@ static void test_yield(void)
     g_assert_cmpint(i, ==, 5); /* coroutine must yield 5 times */
 }
 
+static void coroutine_fn c2_fn(void *opaque)
+{
+    qemu_coroutine_yield();
+}
+
+static void coroutine_fn c1_fn(void *opaque)
+{
+    Coroutine *c2 = opaque;
+    qemu_coroutine_enter(c2, NULL);
+}
+
+static void test_co_queue(void)
+{
+    Coroutine *c1;
+    Coroutine *c2;
+
+    c1 = qemu_coroutine_create(c1_fn);
+    c2 = qemu_coroutine_create(c2_fn);
+
+    qemu_coroutine_enter(c1, c2);
+    memset(c1, 0xff, sizeof(Coroutine));
+    qemu_coroutine_enter(c2, NULL);
+}
+
 /*
  * Check that creation, enter, and return work
  */
@@ -343,6 +368,7 @@ static void perf_cost(void)
 int main(int argc, char **argv)
 {
     g_test_init(&argc, &argv, NULL);
+    g_test_add_func("/basic/co_queue", test_co_queue);
     g_test_add_func("/basic/lifecycle", test_lifecycle);
     g_test_add_func("/basic/yield", test_yield);
     g_test_add_func("/basic/nesting", test_nesting);
Signature · 1 line
1.8.3.1
author Applied
#11 lore raw
In reply to Kevin Wolf

Am 10.02.2015 um 11:41 hat Kevin Wolf geschrieben:

Quoted 11 lines

Kevin Wolf (2):
coroutine: Fix use after free with qemu_coroutine_yield()
coroutine: Clean up qemu_coroutine_enter()

Stefan Hajnoczi (1):
test-coroutine: Regression test for yield bug

 include/block/coroutine_int.h |  1 +
 qemu-coroutine.c              | 38 ++++++++++++++++----------------------
 tests/test-coroutine.c        | 26 ++++++++++++++++++++++++++
 3 files changed, 43 insertions(+), 22 deletions(-)

Applied to the block branch.

Kevin