fix sqpoll cancellation hangs
cover fix sqpoll cancellation hangs
1/2 is a prep, 2/2 fixes the issue. Many io_run_task_work_head() call sites look nasty, but I expect them all but one to go away after controlling SQPOLL task lifetime.
Pavel Begunkov (2):
io_uring: add generic callback_head helpers
io_uring: fix sqpoll cancellation via task_work
fs/io_uring.c | 68 +++++++++++++++++++++++++++++++-------------------- 1 file changed, 41 insertions(+), 27 deletions(-)
Signature · 1 line
2.24.0
PATCH 1/2 io_uring: add generic callback_head helpers
We already have helpers to run/add callback_head but taking ctx and working with ctx->exit_task_work. Extract generic versions of them implemented in terms of struct callback_head, it will be used later.
--- fs/io_uring.c | 62 ++++++++++++++++++++++++++++++--------------------- 1 file changed, 36 insertions(+), 26 deletions(-)
fs/io_uring.c +36 -26
@@ -1926,17 +1926,44 @@ static int io_req_task_work_add(struct io_kiocb *req)return ret; } -static void io_req_task_work_add_fallback (struct io_kiocb *req, -task_work_func_t cb) +static bool io_run_task_work_head (struct callback_head **work_head) +{ +struct callback_head *work, *next; +bool executed = false; + +do { + work =xchg (work_head,NULL ); +if (!work) +break ; + +do { + next = work->next; + work->func (work); + work = next; +cond_resched (); + }while (work); + executed = true; + }while (1 ); + +return executed; +} + +static void io_task_work_add_head (struct callback_head **work_head, +struct callback_head *task_work) { -struct io_ring_ctx *ctx = req->ctx;struct callback_head *head; -init_task_work (&req->task_work, cb);do { - head =READ_ONCE (ctx->exit_task_work); - req->task_work.next = head; - }while (cmpxchg (&ctx->exit_task_work, head, &req->task_work) != head); + head =READ_ONCE (*work_head); + task_work->next = head; + }while (cmpxchg (work_head, head, task_work) != head); +} + +static void io_req_task_work_add_fallback (struct io_kiocb *req, +task_work_func_t cb) +{ +init_task_work (&req->task_work, cb); +io_task_work_add_head (&req->ctx->exit_task_work, &req->task_work); }static void __io_req_task_cancel (struct io_kiocb *req,int error)
@@ -8468,26 +8495,9 @@ static int io_unregister_personality(struct io_ring_ctx *ctx, unsigned id)return -EINVAL ; } -static bool io_run_ctx_fallback (struct io_ring_ctx *ctx) +static inline bool io_run_ctx_fallback (struct io_ring_ctx *ctx) { -struct callback_head *work, *next; -bool executed = false; - -do { - work =xchg (&ctx->exit_task_work,NULL ); -if (!work) -break ; - -do { - next = work->next; - work->func (work); - work = next; -cond_resched (); - }while (work); - executed = true; - }while (1 ); - -return executed; +return io_run_task_work_head (&ctx->exit_task_work); }struct io_tctx_exit {
Signature · 1 line
2.24.0
PATCH 2/2 io_uring: fix sqpoll cancellation via task_work
In reply to Pavel BegunkovRunning sqpoll cancellations via task_work_run() is a bad idea because it depends on other task works to be run, but those may be locked in currently running task_work_run() because of how it's (splicing the list in batches).
Enqueue and run them through a separate callback head, namely struct io_sq_data::park_task_work. As a nice bonus we now precisely control where it's run, that's much safer than guessing where it can happen as it was before.
--- fs/io_uring.c | 6 +++++- 1 file changed, 5 insertions(+), 1 deletion(-)
fs/io_uring.c +5 -1
@@ -274,6 +274,7 @@ struct io_sq_data {unsigned long state;struct completion exited; +struct callback_head *park_task_work; };#define IO_IOPOLL_BATCH 8
@@ -6724,6 +6725,7 @@ static int io_sq_thread(void *data)cond_resched ();mutex_lock (&sqd->lock);io_run_task_work (); +io_run_task_work_head (&sqd->park_task_work); timeout = jiffies + sqd->sq_thread_idle;continue ; }
@@ -6778,6 +6780,7 @@ static int io_sq_thread(void *data) }finish_wait (&sqd->wait, &wait); +io_run_task_work_head (&sqd->park_task_work); timeout = jiffies + sqd->sq_thread_idle; }
@@ -6789,6 +6792,7 @@ static int io_sq_thread(void *data)mutex_unlock (&sqd->lock);io_run_task_work (); +io_run_task_work_head (&sqd->park_task_work);complete (&sqd->exited);do_exit (0 ); }
@@ -8886,7 +8890,7 @@ static void io_sqpoll_cancel_sync(struct io_ring_ctx *ctx)if (task) {init_completion (&work.completion);init_task_work (&work.task_work, io_sqpoll_cancel_cb); -WARN_ON_ONCE (task_work_add (task, &work.task_work,TWA_SIGNAL )); +io_task_work_add_head (&sqd->park_task_work, &work.task_work);wake_up_process (task); }io_sq_thread_unpark (sqd);
Signature · 1 line
2.24.0