Fix some jobs/drain/aio_poll related hangs

All callers in QEMU proper hold the AioContext lock when calling job_finish_sync(). The tests should do the same.

Signed-off-by Kevin Wolf <[email protected]>

---
 include/qemu/job.h      | 6 ++++++
 tests/test-bdrv-drain.c | 6 ++++++
 tests/test-blockjob.c   | 6 ++++++
 3 files changed, 18 insertions(+)

include/qemu/job.h +6 -0

index 0dae5b8481..8ac48dbd28 100644
@@ -520,6 +520,8 @@ void job_user_cancel(Job *job, bool force, Error **errp);
  *
  * Returns the return value from the job if the job actually completed
  * during the call, or -ECANCELED if it was canceled.
+ *
+ * Callers must hold the AioContext lock of job->aio_context.
  */
 int job_cancel_sync(Job *job);
 

@@ -537,6 +539,8 @@ void job_cancel_sync_all(void);
  * function).
  *
  * Returns the return value from the job.
+ *
+ * Callers must hold the AioContext lock of job->aio_context.
  */
 int job_complete_sync(Job *job, Error **errp);
 

@@ -579,6 +583,8 @@ void job_defer_to_main_loop(Job *job, JobDeferToMainLoopFn *fn, void *opaque);
  *
  * Returns 0 if the job is successfully completed, -ECANCELED if the job was
  * cancelled before completing, and -errno in other error cases.
+ *
+ * Callers must hold the AioContext lock of job->aio_context.
  */
 int job_finish_sync(Job *job, void (*finish)(Job *, Error **errp), Error **errp);
 

tests/test-bdrv-drain.c +6 -0

index 17bb8508ae..30294038ef 100644
@@ -795,6 +795,7 @@ static void test_blockjob_common(enum drain_type drain_type)
     BlockBackend *blk_src, *blk_target;
     BlockDriverState *src, *target;
     BlockJob *job;
+    AioContext *ctx;
     int ret;
 
     src = bdrv_new_open_driver(&bdrv_test, "source", BDRV_O_RDWR,

@@ -807,6 +808,9 @@ static void test_blockjob_common(enum drain_type drain_type)
     blk_target = blk_new(BLK_PERM_ALL, BLK_PERM_ALL);
     blk_insert_bs(blk_target, target, &error_abort);
 
+    ctx = qemu_get_aio_context();
+    aio_context_acquire(ctx);
+
     job = block_job_create("job0", &test_job_driver, NULL, src, 0, BLK_PERM_ALL,
                            0, 0, NULL, NULL, &error_abort);
     block_job_add_bdrv(job, "target", target, 0, BLK_PERM_ALL, &error_abort);

@@ -853,6 +857,8 @@ static void test_blockjob_common(enum drain_type drain_type)
     ret = job_complete_sync(&job->job, &error_abort);
     g_assert_cmpint(ret, ==, 0);
 
+    aio_context_release(ctx);
+
     blk_unref(blk_src);
     blk_unref(blk_target);
     bdrv_unref(src);

tests/test-blockjob.c +6 -0

index cb42f06e61..8c2babbe35 100644
@@ -230,6 +230,10 @@ static void cancel_common(CancelJob *s)
     BlockJob *job = &s->common;
     BlockBackend *blk = s->blk;
     JobStatus sts = job->job.status;
+    AioContext *ctx;
+
+    ctx = job->job.aio_context;
+    aio_context_acquire(ctx);
 
     job_cancel_sync(&job->job);
     if (sts != JOB_STATUS_CREATED && sts != JOB_STATUS_CONCLUDED) {

@@ -239,6 +243,8 @@ static void cancel_common(CancelJob *s)
     assert(job->job.status == JOB_STATUS_NULL);
     job_unref(&job->job);
     destroy_blk(blk);
+
+    aio_context_release(ctx);
 }
 
 static void test_cancel_created(void)

Signature · 1 line

2.13.6

Simimlar to AIO_WAIT_WHILE(), job_finish_sync() needs to release the AioContext lock of the job before calling aio_poll(). Otherwise, callbacks called by aio_poll() would possibly take the lock a second time and run into a deadlock with a nested AIO_WAIT_WHILE() call.

Signed-off-by Kevin Wolf <[email protected]>

---
 job.c | 3 +++
 1 file changed, 3 insertions(+)

job.c +0 -0

index a746bfe70b..6acf55bceb 100644

Quoted 19 lines

Simimlar to AIO_WAIT_WHILE(), job_finish_sync() needs to release the AioContext lock of the job before calling aio_poll(). Otherwise, callbacks called by aio_poll() would possibly take the lock a second time and run into a deadlock with a nested AIO_WAIT_WHILE() call.

Signed-off-by Kevin Wolf <[email protected]>

---
 job.c | 3 +++
 1 file changed, 3 insertions(+)

job.c +1 -0

index a746bfe70b..6acf55bceb 100644
@@ -1016,7 +1016,10 @@ int job_finish_sync(Job *job, void (*finish)(Job *, Error **errp), Error **errp)
         job_drain(job);
     }
     while (!job_is_completed(job)) {
+        AioContext *aio_context = job->aio_context;

job.c int job_finish_sync(Job *job, void (*finish)(Job *, Error **errp), Error **errp) · line 1025

Quoted 6 lines

job.c +2 -0

+        aio_context_release(aio_context);
         aio_poll(qemu_get_aio_context(), true);
+        aio_context_acquire(aio_context);
     }
     ret = (job_is_cancelled(job) && job->ret == 0) ? -ECANCELED : job->ret;
     job_unref(job);

Why doesn't this function just use AIO_WAIT_WHILE()?

Simimlar to AIO_WAIT_WHILE(), bdrv_drain_poll_top_level() needs to release the AioContext lock of the node to be drained before calling aio_poll(). Otherwise, callbacks called by aio_poll() would possibly take the lock a second time and run into a deadlock with a nested AIO_WAIT_WHILE() call.

Signed-off-by Kevin Wolf <[email protected]>

---
 block/io.c | 25 ++++++++++++++++++++++++-
 1 file changed, 24 insertions(+), 1 deletion(-)

block/io.c +24 -1

index 7100344c7b..832d2536bf 100644
@@ -268,9 +268,32 @@ bool bdrv_drain_poll(BlockDriverState *bs, bool recursive,
 static bool bdrv_drain_poll_top_level(BlockDriverState *bs, bool recursive,
                                       BdrvChild *ignore_parent)
 {
+    AioContext *ctx = bdrv_get_aio_context(bs);
+
+    /*
+     * We cannot easily release the lock unconditionally here because many
+     * callers of drain function (like qemu initialisation, tools, etc.) don't
+     * even hold the main context lock.
+     *
+     * This means that we fix potential deadlocks for the case where we are in
+     * the main context and polling a BDS in a different AioContext, but
+     * draining a BDS in the main context from a different I/O thread would
+     * still have this problem. Fortunately, this isn't supposed to happen
+     * anyway.
+     */
+    if (ctx != qemu_get_aio_context()) {
+        aio_context_release(ctx);
+    } else {
+        assert(qemu_get_current_aio_context() == qemu_get_aio_context());
+    }
+
     /* Execute pending BHs first and check everything else only after the BHs
      * have executed. */
-    while (aio_poll(bs->aio_context, false));
+    while (aio_poll(ctx, false));
+
+    if (ctx != qemu_get_aio_context()) {
+        aio_context_acquire(ctx);
+    }
 
     return bdrv_drain_poll(bs, recursive, ignore_parent, false);
 }

Signature · 1 line

2.13.6

Quoted 47 lines

Simimlar to AIO_WAIT_WHILE(), bdrv_drain_poll_top_level() needs to release the AioContext lock of the node to be drained before calling aio_poll(). Otherwise, callbacks called by aio_poll() would possibly take the lock a second time and run into a deadlock with a nested AIO_WAIT_WHILE() call.

Signed-off-by Kevin Wolf <[email protected]>

---
 block/io.c | 25 ++++++++++++++++++++++++-
 1 file changed, 24 insertions(+), 1 deletion(-)

block/io.c +24 -1

index 7100344c7b..832d2536bf 100644
@@ -268,9 +268,32 @@ bool bdrv_drain_poll(BlockDriverState *bs, bool recursive,
 static bool bdrv_drain_poll_top_level(BlockDriverState *bs, bool recursive,
                                       BdrvChild *ignore_parent)
 {
+    AioContext *ctx = bdrv_get_aio_context(bs);
+
+    /*
+     * We cannot easily release the lock unconditionally here because many
+     * callers of drain function (like qemu initialisation, tools, etc.) don't
+     * even hold the main context lock.
+     *
+     * This means that we fix potential deadlocks for the case where we are in
+     * the main context and polling a BDS in a different AioContext, but
+     * draining a BDS in the main context from a different I/O thread would
+     * still have this problem. Fortunately, this isn't supposed to happen
+     * anyway.
+     */
+    if (ctx != qemu_get_aio_context()) {
+        aio_context_release(ctx);
+    } else {
+        assert(qemu_get_current_aio_context() == qemu_get_aio_context());
+    }
+
     /* Execute pending BHs first and check everything else only after the BHs
      * have executed. */
-    while (aio_poll(bs->aio_context, false));
+    while (aio_poll(ctx, false));
+
+    if (ctx != qemu_get_aio_context()) {
+        aio_context_acquire(ctx);
+    }
 

block/io.c bool bdrv_drain_poll(BlockDriverState *bs, bool recursive, · line 299

Quoted 2 lines

block/io.c +0 -0

     return bdrv_drain_poll(bs, recursive, ignore_parent, false);
 }

The same question as patch 3: why not just use AIO_WAIT_WHILE() here? It takes care to not release any lock if both running and polling in the main context (taking the in_aio_context_home_thread() branch).

I'm running out of time and will be offline for the next two weeks, so I'm just sending out what I have right now. This is probably not ready to be merged yet, but if need be, someone else can pick it up. Otherwise I'll do that myself when I return.

This is related to the following bug reports:
https://bugzilla.redhat.com/show_bug.cgi?id=1614623
https://bugzilla.redhat.com/show_bug.cgi?id=1601212

Essentially it tries to fix a few deadlocks into which we run when running two commit jobs (probably just one example for problematic scenarios) on nodes that are in a non-mainloop AioContext (i.e. with dataplane).

Obviously, besides verifying that the code changes are actually correct as they are, we also want to add some test cases for this. I suppose test-bdrv-drain should test for the individual bugs, and a qemu-iotests case could try the higher level scenario of multiple commit jobs with dataplane.

Kevin Wolf (5):
blockjob: Wake up BDS when job becomes idle
tests: Acquire AioContext around job_finish_sync()
job: Drop AioContext lock around aio_poll()
block: Drop AioContext lock in bdrv_drain_poll_top_level() [WIP] Lock AioContext in bdrv_co_drain_bh_cb()

 include/block/blockjob.h | 13 +++++++++++++
 include/qemu/job.h       |  9 +++++++++
 block/io.c               | 31 ++++++++++++++++++++++++++++++-
 blockjob.c               | 18 ++++++++++++++++++
 job.c                    | 10 ++++++++++
 tests/test-bdrv-drain.c  |  6 ++++++
 tests/test-blockjob.c    |  6 ++++++
 7 files changed, 92 insertions(+), 1 deletion(-)

Signature · 1 line

2.13.6