io_req_complete_post() fix
cover io_req_complete_post() fix
May use more gazing by someone to avoid obvious/stupid mistakes
Pavel Begunkov (2):
io_uring: add io_disarm_next() helper
io_uring: fix complete_post races for linked req
fs/io_uring.c | 89 ++++++++++++++++++++++++++++----------------------- 1 file changed, 49 insertions(+), 40 deletions(-)
Signature · 1 line
2.24.0
PATCH 1/2 io_uring: add io_disarm_next() helper
A preparation patch placing all preparations before extracting a next request into a separate helper io_disarm_next().
Also, don't spuriously do ev_posted in a rare case where REQ_F_FAIL_LINK is set but there are no requests linked (i.e. after cancelling a linked timeout or setting IOSQE_IO_LINK on a last request of a submission batch).
--- fs/io_uring.c | 68 ++++++++++++++++++++++++++------------------------- 1 file changed, 35 insertions(+), 33 deletions(-)
fs/io_uring.c +35 -33
@@ -1706,15 +1706,11 @@ static inline void io_remove_next_linked(struct io_kiocb *req) nxt->link =NULL ; } -static void io_kill_linked_timeout (struct io_kiocb *req) +static bool io_kill_linked_timeout (struct io_kiocb *req) + __must_hold(&req->ctx->completion_lock) { -struct io_ring_ctx *ctx = req->ctx; -struct io_kiocb *link; +struct io_kiocb *link = req->link;bool cancelled = false; -unsigned long flags; - -spin_lock_irqsave (&ctx->completion_lock, flags); - link = req->link;/* * Can happen if a linked timeout fired and link had been like
@@ -1729,50 +1725,48 @@ static void io_kill_linked_timeout(struct io_kiocb *req) ret =hrtimer_try_to_cancel (&io->timer);if (ret !=-1 ) {io_cqring_fill_event (link, -ECANCELED ); -io_commit_cqring (ctx); +io_put_req_deferred (link,1 ); cancelled = true; } } req->flags &= ~REQ_F_LINK_TIMEOUT ; -spin_unlock_irqrestore (&ctx->completion_lock, flags); - -if (cancelled) { -io_cqring_ev_posted (ctx); -io_put_req (link); - } +return cancelled; } -static void io_fail_links (struct io_kiocb *req) + __must_hold(&req->ctx->completion_lock) { -struct io_kiocb *link, *nxt; -struct io_ring_ctx *ctx = req->ctx; -unsigned long flags; +struct io_kiocb *nxt, *link = req->link; -spin_lock_irqsave (&ctx->completion_lock, flags); - link = req->link; req->link =NULL ; -while (link) { nxt = link->link; link->link =NULL ;trace_io_uring_fail_link (req, link);io_cqring_fill_event (link, -ECANCELED ); -io_put_req_deferred (link,2 ); link = nxt; } -io_commit_cqring (ctx); -spin_unlock_irqrestore (&ctx->completion_lock, flags); +} -io_cqring_ev_posted (ctx); +static bool io_disarm_next (struct io_kiocb *req) + __must_hold(&req->ctx->completion_lock) +{ +bool posted = false; + +if (likely (req->flags &REQ_F_LINK_TIMEOUT )) + posted =io_kill_linked_timeout (req); +if (unlikely (req->flags &REQ_F_FAIL_LINK )) { + posted |= (req->link !=NULL ); +io_fail_links (req); + } +return posted; }static struct io_kiocb *__io_req_find_next (struct io_kiocb *req) { -if (req->flags &REQ_F_LINK_TIMEOUT ) -io_kill_linked_timeout (req); +struct io_kiocb *nxt;/* * If LINK is set, we have dependent requests in this chain. If we
@@ -1780,14 +1774,22 @@ static struct io_kiocb *__io_req_find_next(struct io_kiocb *req)* dependencies to the next request. In case of failure, fail the rest * of the chain. */ -if (likely (!(req->flags &REQ_F_FAIL_LINK ))) { -struct io_kiocb *nxt = req->link; +if (req->flags & (REQ_F_LINK_TIMEOUT |REQ_F_FAIL_LINK )) { +struct io_ring_ctx *ctx = req->ctx; +unsigned long flags; +bool posted; - req->link =NULL ; -return nxt; +spin_lock_irqsave (&ctx->completion_lock, flags); + posted =io_disarm_next (req); +if (posted) +io_commit_cqring (req->ctx); +spin_unlock_irqrestore (&ctx->completion_lock, flags); +if (posted) +io_cqring_ev_posted (ctx); } -io_fail_links (req); -return NULL ; + nxt = req->link; + req->link =NULL ; +return nxt; }static inline struct io_kiocb *io_req_find_next (struct io_kiocb *req)
Signature · 1 line
2.24.0
PATCH 2/2 io_uring: fix complete_post races for linked req
In reply to Pavel BegunkovCalling io_queue_next() after spin_unlock in io_req_complete_post() races with the other side extracting and reusing this request. Hand coded parts of io_req_find_next() considering that io_disarm_next() and io_req_task_queue() have (and safe) to be called with completion_lock held.
It already does io_commit_cqring() and io_cqring_ev_posted(), so just reuse it for post io_disarm_next().
--- fs/io_uring.c | 21 ++++++++++++++------- 1 file changed, 14 insertions(+), 7 deletions(-)
fs/io_uring.c +14 -7
@@ -986,6 +986,7 @@ static const struct io_op_def io_op_defs[] = { [IORING_OP_UNLINKAT] = {}, }; +static bool io_disarm_next (struct io_kiocb *req);static void io_uring_del_task_file (unsigned long index); static void io_uring_try_cancel_requests(struct io_ring_ctx *ctx, struct task_struct *task,
@@ -1526,15 +1527,14 @@ static void io_cqring_fill_event(struct io_kiocb *req, long res)__io_cqring_fill_event (req, res,0 ); } -static inline void io_req_complete_post (struct io_kiocb *req,long res, -unsigned int cflags) +static void io_req_complete_post (struct io_kiocb *req,long res, +unsigned int cflags) {struct io_ring_ctx *ctx = req->ctx;unsigned long flags;spin_lock_irqsave (&ctx->completion_lock, flags);__io_cqring_fill_event (req, res, cflags); -io_commit_cqring (ctx);/* * If we're the last reference to this request, add to our locked * free_list cache.
@@ -1542,19 +1542,26 @@ static inline void io_req_complete_post(struct io_kiocb *req, long res,if (refcount_dec_and_test (&req->refs)) {struct io_comp_state *cs = &ctx->submit_state.comp; +if (req->flags & (REQ_F_LINK |REQ_F_HARDLINK )) { +if (req->flags & (REQ_F_LINK_TIMEOUT |REQ_F_FAIL_LINK )) +io_disarm_next (req); +if (req->link) { +io_req_task_queue (req->link); + req->link =NULL ; + } + }io_dismantle_req (req);io_put_task (req->task,1 );list_add (&req->compl.list, &cs->locked_free_list); cs->locked_free_nr++; }else req =NULL ; +io_commit_cqring (ctx);spin_unlock_irqrestore (&ctx->completion_lock, flags); -io_cqring_ev_posted (ctx); -if (req) { -io_queue_next (req); + +if (req)percpu_ref_put (&ctx->refs); - } } static void io_req_complete_state(struct io_kiocb *req, long res,
Signature · 1 line
2.24.0
Jens Axboe
maintainerQuoted 8 lines · Pavel Begunkov
May use more gazing by someone to avoid obvious/stupid mistakes
Pavel Begunkov (2):
io_uring: addio_disarm_next()helper
io_uring: fixcomplete_postraces for linked reqfs/io_uring.c | 89 ++++++++++++++++++++++++++++----------------------- 1 file changed, 49 insertions(+), 40 deletions(-)
I looked through it and it looks good to me.
Signature · 1 line
Jens Axboe