Cancel requests at ring close time
Jens Axboe
maintainerPATCH v2 1/1 Cancel requests at ring close time
Hi,
For v1, see here:
https://lore.kernel.org/io-uring/[email protected]/
And for the current tree, go here:
https://git.kernel.org/pub/scm/linux/kernel/git/axboe/linux.git/log/?h=io_uring-exit-cancel.7
Changes since v1:
- Reorder the series so the fixes for the current tree come first, then the exit/release cancelation changes on top
- Add fix for io-wq completions that finish asynchronously (eg a direct read completing via ->ki_complete), ensuring CQE posting and file putting are sane.
- Ditto for IOPOLL completions reaped from io_do_iopoll()
- Put request files synchronously before posting the CQEs in the batch flush, fixing an issue with SQPOLL.
- Don't clear REQ_F_REFCOUNT from the completion path, a linked timeout can still take a reference on the request until it's disarmed
Signature · 1 line
Jens Axboe
Jens Axboe
maintainerPATCH 1/10 io_uring/io-wq: put the request file before posting a completion
Once io-wq is done with the request, put any potential file that request had pinned upfront. This ensures any file references are fully put by the time io-wq posts a completion. Registered files are held by the resource node and don't need this.
--- io_uring/io_uring.c | 25 ++++++++++++++++++++++++- 1 file changed, 24 insertions(+), 1 deletion(-)
io_uring/io_uring.c +24 -1
@@ -907,6 +907,24 @@ bool io_req_post_cqe32(struct io_kiocb *req, struct io_uring_cqe cqe[2])return posted; } +/* +* Drop any io-wq request with a file upfront, otherwise it gets deferred to +* much later post CQE posting. +*/ +static void io_req_put_file_iowq (struct io_kiocb *req,bool sync) +{ +struct file *file = req->file; + +if (!file || (req->flags & (REQ_F_FIXED_FILE |REQ_F_REISSUE ))) +return ; + +WRITE_ONCE (req->file,NULL ); +if (sync) +__fput_sync (file); +else +fput (file); +} +static void io_req_complete_post (struct io_kiocb *req,unsigned issue_flags) {struct io_ring_ctx *ctx = req->ctx;
@@ -919,6 +937,8 @@ static void io_req_complete_post(struct io_kiocb *req, unsigned issue_flags)if (WARN_ON_ONCE (!(issue_flags &IO_URING_F_IOWQ )))return ; +io_req_put_file_iowq (req, true); +/* * Handle special CQ sync cases via task_work. DEFER_TASKRUN requires * the submitter task context, IOPOLL protects with uring_lock.
@@ -1480,6 +1500,7 @@ void io_wq_submit_work(struct io_wq_work *work)/* either cancelled or io-wq is dying, so don't touch tctx->iowq */ if (atomic_read (&work->flags) &IO_WQ_WORK_CANCEL ) { fail: +io_req_put_file_iowq (req, false);io_req_task_queue_fail (req, err);return ; }
@@ -1555,8 +1576,10 @@ void io_wq_submit_work(struct io_wq_work *work) }while (1 );/* avoid locking problems by failing it from a clean context */ -if (ret) +if (ret) { +io_req_put_file_iowq (req, true);io_req_task_queue_fail (req, ret); + } } inline struct file *io_file_get_fixed(struct io_kiocb *req, int fd,
Signature · 1 line
2.55.0
Re: [PATCH 01/10] io_uring/io-wq: put the request file before posting a completion
Quoted 24 lines · Jens Axboe
Once io-wq is done with the request, put any potential file that request had pinned upfront. This ensures any file references are fully put by the time io-wq posts a completion. Registered files are held by the resource node and don't need this.
Signed-off-by Jens Axboe <[email protected]>--- io_uring/io_uring.c | 25 ++++++++++++++++++++++++- 1 file changed, 24 insertions(+), 1 deletion(-)io_uring/io_uring.c +6
-0@@ -907,6 +907,24 @@ bool io_req_post_cqe32(struct io_kiocb *req, struct io_uring_cqe cqe[2])return posted; } +/* +* Drop any io-wq request with a file upfront, otherwise it gets deferred to +* much later post CQE posting. +*/ +static void io_req_put_file_iowq (struct io_kiocb *req,bool sync) +{
io_uring/io_uring.c bool io_req_post_cqe32(struct io_kiocb *req, struct io_uring_cqe cqe[2]) · line 925
Quoted 10 lines
io_uring/io_uring.c +10
-0+struct file *file = req->file; + +if (!file || (req->flags & (REQ_F_FIXED_FILE |REQ_F_REISSUE ))) +return ; + +WRITE_ONCE (req->file,NULL ); +if (sync) +__fput_sync (file); +else +fput (file);
Why do you need an async fput on the error path? I'm missing why it necessary. Also, isn't that gonna cause the same sort of delay until the task work or delayed work is executed?
Other than that:
Quoted 37 lines
io_uring/io_uring.c +8
-1+} +static void io_req_complete_post (struct io_kiocb *req,unsigned issue_flags) {struct io_ring_ctx *ctx = req->ctx;@@ -919,6 +937,8 @@ static void io_req_complete_post(struct io_kiocb *req, unsigned issue_flags)if (WARN_ON_ONCE (!(issue_flags &IO_URING_F_IOWQ )))return ; +io_req_put_file_iowq (req, true); +/* * Handle special CQ sync cases via task_work. DEFER_TASKRUN requires * the submitter task context, IOPOLL protects with uring_lock. @@ -1480,6 +1500,7 @@ void io_wq_submit_work(struct io_wq_work *work)/* either cancelled or io-wq is dying, so don't touch tctx->iowq */ if (atomic_read (&work->flags) &IO_WQ_WORK_CANCEL ) { fail: +io_req_put_file_iowq (req, false);io_req_task_queue_fail (req, err);return ; }@@ -1555,8 +1576,10 @@ void io_wq_submit_work(struct io_wq_work *work) }while (1 );/* avoid locking problems by failing it from a clean context */ -if (ret) +if (ret) { +io_req_put_file_iowq (req, true);io_req_task_queue_fail (req, ret); + } } inline struct file *io_file_get_fixed(struct io_kiocb *req, int fd,Signature · 1 line
2.55.0
Signature · 1 line
Gabriel Krisman Bertazi
Jens Axboe
maintainerPATCH 2/10 io_uring: post io-wq completions from the last request reference
In reply to Jens Axboeio-wq holds a reference on a request for the duration of the issue, as the completion may run before the issue returns. If it does, the CQE is posted from that completion while the worker still holds its reference, and the file is only put once the worker drops it. That means the actual file put happens potentially long after CQE posting.
Post the completion with the last put for refcounted requests.
io_free_req() no longer marks the request as CQE_SKIP, that is now done by whoever posted the CQE while another reference was still held.
--- io_uring/io_uring.c | 28 +++++++++++++++++++++++----- 1 file changed, 23 insertions(+), 5 deletions(-)
io_uring/io_uring.c +23 -5
@@ -959,9 +959,11 @@ static void io_req_complete_post(struct io_kiocb *req, unsigned issue_flags) goto defer_complete;/* -* We don't free the request here because we know it's called from -* io-wq only, which holds a reference, so it cannot be the last put. +* Request not freed here because we know it's called from io-wq only, +* which holds a reference. Hence it can't be the last put. The CQE +* has been posted, last put frees it. */ + req->flags |=REQ_F_CQE_SKIP ;req_ref_put (req); }
@@ -1015,8 +1017,6 @@ __cold void io_free_req(struct io_kiocb *req) {/* refs were already put, restore them for io_req_task_complete() */ req->flags &= ~REQ_F_REFCOUNT ; -/* we only want to free it, don't post CQEs */ - req->flags |=REQ_F_CQE_SKIP ; req->io_task_work.func = io_req_task_complete;io_req_task_work_add (req); }
@@ -1119,6 +1119,8 @@ static void io_free_batch_list(struct io_ring_ctx *ctx, }if (req->flags &REQ_F_REFCOUNT ) { node = req->comp_list.next; +/* CQE posted, the last put only frees */ + req->flags |=REQ_F_CQE_SKIP ;if (!req_ref_put_and_test (req))continue ; }
@@ -1282,7 +1284,23 @@ static int io_iopoll_check(struct io_ring_ctx *ctx, unsigned int min_events)void io_req_task_complete (struct io_tw_req tw_req,io_tw_token_t tw) { -io_req_complete_defer (tw_req.req); +struct io_kiocb *req = tw_req.req; + +/* +* io-wq may still hold a reference if the issue completed async. +* Defer completion to the last put, so that file drop and CQE +* visibility are ordered. The last reference stays for the free +* path to put, a linked timeout may still look at the request. +*/ +if ((req->flags &REQ_F_REFCOUNT ) && !(req->flags &REQ_F_REISSUE ) && +atomic_read (&req->refs) !=1 ) { +if (!req_ref_put_and_test (req)) +return ; +/* the other put raced us, ours was the last after all */ +atomic_set (&req->refs,1 ); + } + +io_req_complete_defer (req); }/*
Signature · 1 line
2.55.0
Jens Axboe
maintainerPATCH 3/10 io_uring/rw: don't reap io-wq IOPOLL completions while io-wq has a reference
In reply to Jens AxboeIOPOLL requests issued from io-wq are reaped by io_do_iopoll() on the submitter, with the same window as the task_work completions: the CQE is posted while the worker may still hold its reference, and the file is only dropped once it does.
Leave such a request on the iopoll list until the worker has dropped its reference, the next pass then reaps it and the put from io_free_batch_list() is the last one. Also move the reference handling for the task_work case into a helper.
--- io_uring/io_uring.c | 17 ++--------------- io_uring/refs.h | 27 +++++++++++++++++++++++++++ io_uring/rw.c | 4 ++++ 3 files changed, 33 insertions(+), 15 deletions(-)
io_uring/io_uring.c +2 -15
@@ -1286,21 +1286,8 @@ void io_req_task_complete(struct io_tw_req tw_req, io_tw_token_t tw) {struct io_kiocb *req = tw_req.req; -/* -* io-wq may still hold a reference if the issue completed async. -* Defer completion to the last put, so that file drop and CQE -* visibility are ordered. The last reference stays for the free -* path to put, a linked timeout may still look at the request. -*/ -if ((req->flags &REQ_F_REFCOUNT ) && !(req->flags &REQ_F_REISSUE ) && -atomic_read (&req->refs) !=1 ) { -if (!req_ref_put_and_test (req)) -return ; -/* the other put raced us, ours was the last after all */ -atomic_set (&req->refs,1 ); - } - -io_req_complete_defer (req); +if (io_req_complete_ready (req)) +io_req_complete_defer (req); }/*
io_uring/refs.h +27 -0
@@ -56,6 +56,33 @@ static inline void __io_req_set_refcount(struct io_kiocb *req, int nr) } } +/* +* io-wq may still hold a reference if the issue completed async. If so, the +* last put completes the request, so that file drop and CQE visibility are +* ordered. The last reference stays for the free path to put, a linked +* timeout may still look at the request until then. +*/ +static inline bool io_req_complete_ready (struct io_kiocb *req) +{ +if (!(req->flags &REQ_F_REFCOUNT ) || (req->flags &REQ_F_REISSUE )) +return true; +if (atomic_read (&req->refs) ==1 ) +return true; +if (!req_ref_put_and_test (req)) +return false; +/* the other put raced us, ours was the last after all */ +atomic_set (&req->refs,1 ); +return true; +} + +/* io-wq still holds a reference, the request can't be completed yet */ +static inline bool io_req_shared (struct io_kiocb *req) +{ +if (!(req->flags &REQ_F_REFCOUNT ) || (req->flags &REQ_F_REISSUE )) +return false; +return atomic_read (&req->refs) >1 ; +} +static inline void io_req_set_refcount (struct io_kiocb *req) {__io_req_set_refcount (req,1 );
io_uring/rw.c +4 -0
@@ -16,6 +16,7 @@#include "filetable.h" #include "io_uring.h" +#include "refs.h" #include "opdef.h" #include "kbuf.h" #include "alloc_cache.h"
@@ -1370,6 +1371,9 @@ int io_do_iopoll(struct io_ring_ctx *ctx, bool force_nonspin)/* order with io_complete_rw_iopoll(), e.g. ->result updates */ if (!smp_load_acquire (&req->iopoll_completed))continue ; +/* io-wq still has a reference, reap it on the next pass */ +if (io_req_shared (req)) +continue ;list_del (&req->iopoll_node);wq_list_add_tail (&req->comp_list, &ctx->submit_state.compl_reqs); nr_events++;
Signature · 1 line
2.55.0
Jens Axboe
maintainerPATCH 4/10 io_uring: put request files before posting the completions
In reply to Jens AxboeThe batch flush posts the CQEs first and drops the files afterwards, with a plain fput() that defers the release to task_work. For the submitting task that runs on the way back to userspace, but with SQPOLL the sqpoll thread only gets to it after the io_uring task_work is done. Userspace sees the CQE in between, and closing the file and exec'ing it fails with ETXTBSY as the request still holds it.
Drop the files synchronously before the CQEs are posted, for requests that the flush is about to free. Ring files stay deferred, releasing one takes its uring_lock and may wait on its requests.
--- io_uring/io_uring.c | 29 ++++++++++++++++++++--------- 1 file changed, 20 insertions(+), 9 deletions(-)
io_uring/io_uring.c +20 -9
@@ -907,11 +907,8 @@ bool io_req_post_cqe32(struct io_kiocb *req, struct io_uring_cqe cqe[2])return posted; } -/* -* Drop any io-wq request with a file upfront, otherwise it gets deferred to -* much later post CQE posting. -*/ -static void io_req_put_file_iowq (struct io_kiocb *req,bool sync) +/* drop the request file before the CQE is posted, not deferred after it */ +static void io_req_put_file (struct io_kiocb *req,bool sync) {struct file *file = req->file;
@@ -919,7 +916,8 @@ static void io_req_put_file_iowq(struct io_kiocb *req, bool sync)return ;WRITE_ONCE (req->file,NULL ); -if (sync) +/* releasing a ring may wait on other rings, keep that deferred */ +if (sync && !io_is_uring_fops (file))__fput_sync (file);else fput (file);
@@ -937,7 +935,7 @@ static void io_req_complete_post(struct io_kiocb *req, unsigned issue_flags)if (WARN_ON_ONCE (!(issue_flags &IO_URING_F_IOWQ )))return ; -io_req_put_file_iowq (req, true); +io_req_put_file (req, true);/* * Handle special CQ sync cases via task_work. DEFER_TASKRUN requires
@@ -1152,6 +1150,19 @@ void __io_submit_flush_completions(struct io_ring_ctx *ctx)struct io_submit_state *state = &ctx->submit_state;struct io_wq_work_node *node; +/* +* Drop the files before the CQEs are posted, so they're released by +* the time the completions are visible. Not for requests that are +* requeued or still referenced, those aren't freed below. +*/ +__wq_list_for_each (node, &state->compl_reqs) { + struct io_kiocb *req = container_of(node, struct io_kiocb, + comp_list); + +if (!io_req_shared (req)) +io_req_put_file (req, true); + } +__io_cq_lock (ctx);__wq_list_for_each (node, &state->compl_reqs) { struct io_kiocb *req = container_of(node, struct io_kiocb,
@@ -1505,7 +1516,7 @@ void io_wq_submit_work(struct io_wq_work *work)/* either cancelled or io-wq is dying, so don't touch tctx->iowq */ if (atomic_read (&work->flags) &IO_WQ_WORK_CANCEL ) { fail: -io_req_put_file_iowq (req, false); +io_req_put_file (req, false);io_req_task_queue_fail (req, err);return ; }
@@ -1582,7 +1593,7 @@ void io_wq_submit_work(struct io_wq_work *work)/* avoid locking problems by failing it from a clean context */ if (ret) { -io_req_put_file_iowq (req, true); +io_req_put_file (req, true);io_req_task_queue_fail (req, ret); } }
Signature · 1 line
2.55.0
Jens Axboe
maintainerPATCH 5/10 io_uring/uring_cmd: only cancel requests of the given task
In reply to Jens Axboeio_uring_try_cancel_uring_cmd() ignores the task it is asked to cancel on behalf of as soon as cancel_all is true, and cancels every cancelable command on the ring instead. That differs from how other requests are matched, where a valid task always restricts matching to that task's requests and cancel_all simply lifts the REQ_F_INFLIGHT restriction.
So far the only caller passing both a task and cancel_all is exec, where tearing down another task's commands on a shared ring is somewhat impolite. The exit path is about to do the same for every exiting process though, so bring it in line. Match on the task if one is given, and cancel everything only for ring teardown, which passes NULL. This leaves cancel_all as useless, hence drop it.
--- io_uring/cancel.c | 2 +- io_uring/uring_cmd.c | 4 ++-- io_uring/uring_cmd.h | 2 +- 3 files changed, 4 insertions(+), 4 deletions(-)
io_uring/cancel.c +1 -1
@@ -560,7 +560,7 @@ __cold bool io_uring_try_cancel_requests(struct io_ring_ctx *ctx, ret |=io_poll_remove_all (ctx, tctx, cancel_all); ret |=io_waitid_remove_all (ctx, tctx, cancel_all); ret |=io_futex_remove_all (ctx, tctx, cancel_all); - ret |=io_uring_try_cancel_uring_cmd (ctx, tctx, cancel_all); + ret |=io_uring_try_cancel_uring_cmd (ctx, tctx); ret |=io_kill_timeouts (ctx, tctx, cancel_all);mutex_unlock (&ctx->uring_lock);if (tctx)
io_uring/uring_cmd.c +2 -2
@@ -49,7 +49,7 @@ void io_uring_cmd_cleanup(struct io_kiocb *req) }bool io_uring_try_cancel_uring_cmd (struct io_ring_ctx *ctx, -struct io_uring_task *tctx,bool cancel_all) +struct io_uring_task *tctx) {struct hlist_node *tmp;struct io_kiocb *req;
@@ -63,7 +63,7 @@ bool io_uring_try_cancel_uring_cmd(struct io_ring_ctx *ctx, struct io_uring_cmd);struct file *file = req->file; -if (!cancel_all && req->tctx != tctx) +if (tctx && req->tctx != tctx)continue ;if (cmd->flags &IORING_URING_CMD_CANCELABLE ) {
io_uring/uring_cmd.h +1 -1
@@ -14,7 +14,7 @@ void io_uring_cmd_sqe_copy(struct io_kiocb *req);void io_uring_cmd_cleanup (struct io_kiocb *req);bool io_uring_try_cancel_uring_cmd (struct io_ring_ctx *ctx, -struct io_uring_task *tctx,bool cancel_all); +struct io_uring_task *tctx); bool io_uring_cmd_post_mshot_cqe32(struct io_uring_cmd *cmd, unsigned int issue_flags,
Signature · 1 line
2.55.0
Jens Axboe
maintainerPATCH 6/10 io_uring/notif: count pending zerocopy notifications per ring
In reply to Jens AxboeSEND_ZC notifications are io_kiocbs like any other as far as request and task accounting goes, but they are special in that they cannot be canceled and only complete once the network stack is done with the data. With an unresponsive peer, that can take a while.
The exit and release paths are about to start waiting for requests to finish so that the files they pin get released in a timely manner, and notifications neither pin files nor finish in a timely manner. Keep a per-ring count of them so those paths can leave them out.
--- include/linux/io_uring_types.h | 2 ++ io_uring/notif.c | 2 ++ 2 files changed, 4 insertions(+)
include/linux/io_uring_types.h +2 -0
@@ -522,6 +522,8 @@ struct io_ring_ctx {/* protected by ->completion_lock */ unsigned nr_req_allocated; +/* pending SEND_ZC notifications, protected by ->uring_lock */ +unsigned nr_notifs;#ifdef CONFIG_NET_RX_BUSY_POLL struct list_head napi_list;/* track busy poll napi_id */
io_uring/notif.c +2 -0
@@ -36,6 +36,7 @@ static void io_notif_tw_complete(struct io_tw_req tw_req, io_tw_token_t tw) } nd = nd->next; + ctx->nr_notifs--;io_req_task_complete ((struct io_tw_req ){notif}, tw); }while (nd); }
@@ -119,6 +120,7 @@ struct io_kiocb *io_alloc_notif(struct io_ring_ctx *ctx)if (unlikely (!io_alloc_req (ctx, ¬if)))return NULL ; + ctx->nr_notifs++; notif->ctx = ctx; notif->opcode =IORING_OP_NOP ; notif->flags =0 ;
Signature · 1 line
2.55.0
Jens Axboe
maintainerPATCH 7/10 io_uring/cancel: cancel and wait for all requests on process exit
In reply to Jens AxboeWhen a task exits, io_uring only cancels and waits for the few request types that must not outlive it, and leaves everything else to be torn down whenever the ring itself goes away. This happens from a kernel workqueue, and may be some time after the final fput of the ring has been completed. This sometimes causes application issues, where a task that was doing IO on a file in /mnt, for example, will leave /mnt busy for a brief period of time after close(ring_fd) is done.
If the whole thread group is exiting, no thread is left to reap completions or care about those requests. For this case, tear everything down. This is basically what exec does today.
Two types of requests are ignored, as they never pin any files and will always complete on their own. One is armed timeouts, which may still be required to trigger if a ring is being shared, and the other is SEND_ZC notifications, which can take almost an unbounded time to complete. io_uring_try_cancel_requests() now takes flags to manage that behavior.
While in there, fix up the wait loop for the exit case. A task that is exiting because of a fatal signal still has that signal pending, which turns the interruptible sleep into a busy loop. Use short uninterruptible sleeps in that case instead, task_work still gets run in between.
--- io_uring/cancel.c | 87 ++++++++++++++++++++++++++++++++++++++------- io_uring/cancel.h | 13 ++++++- io_uring/io_uring.c | 2 +- io_uring/timeout.c | 18 ++++++++++ io_uring/timeout.h | 2 ++ 5 files changed, 108 insertions(+), 14 deletions(-)
io_uring/cancel.c +75 -12
@@ -514,8 +514,9 @@ static __cold bool io_uring_try_cancel_iowq(struct io_ring_ctx *ctx)__cold bool io_uring_try_cancel_requests (struct io_ring_ctx *ctx,struct io_uring_task *tctx, -bool cancel_all,bool is_sqpoll_thread) +unsigned int flags) { +bool cancel_all = flags &IO_CANCEL_ALL ;struct io_task_cancel cancel = { .tctx = tctx, .all = cancel_all, };enum io_wq_cancel cret;bool ret = false;
@@ -544,7 +545,7 @@ __cold bool io_uring_try_cancel_requests(struct io_ring_ctx *ctx,/* SQPOLL thread does its own polling */ if ((!(ctx->flags &IORING_SETUP_SQPOLL ) && cancel_all) || - is_sqpoll_thread) { + (flags &IO_CANCEL_SQPOLL )) {while (!list_empty (&ctx->iopoll_list)) {io_iopoll_try_reap_events (ctx); ret = true;
@@ -561,6 +562,8 @@ __cold bool io_uring_try_cancel_requests(struct io_ring_ctx *ctx, ret |=io_waitid_remove_all (ctx, tctx, cancel_all); ret |=io_futex_remove_all (ctx, tctx, cancel_all); ret |=io_uring_try_cancel_uring_cmd (ctx, tctx); +if (flags &IO_CANCEL_KEEP_TIMEOUTS ) + cancel_all = false; ret |=io_kill_timeouts (ctx, tctx, cancel_all);mutex_unlock (&ctx->uring_lock);if (tctx)
@@ -575,6 +578,44 @@ static s64 tctx_inflight(struct io_uring_task *tctx, bool tracked)return percpu_counter_sum (&tctx->inflight); } +/* +* If true, whole thread group is exiting, at which point no task is left that +* can reap completions and care about requests in-flight. +*/ +static bool io_task_group_exiting (void ) +{ +return current->signal->flags &SIGNAL_GROUP_EXIT ; +} + +/* +* Return a count of requests an exiting task should wait for. +*/ +static s64 tctx_inflight_exit (struct io_uring_task *tctx) +{ +struct io_tctx_node *node; +unsigned long index; +s64 inflight; + + inflight =tctx_inflight (tctx, false); +xa_for_each (&tctx->xa, index, node) { +/* unlocked read is fine, the caller re-evaluates until done */ + inflight -=data_race (node->ctx->nr_notifs); +/* takes ->uring_lock, we hold nothing on the group exit path */ + inflight -=io_timeouts_armed (node->ctx, tctx); + } +return inflight; +} + +static bool io_tctx_cancel_done (struct io_uring_task *tctx,bool cancel_all, +bool group_exit) +{ +if (cancel_all) +return !tctx_inflight (tctx, false); +if (tctx_inflight (tctx, true)) +return false; +return !group_exit ||tctx_inflight_exit (tctx) <=0 ; +} +/* * Find any io_uring ctx that this task has registered or done IO on, and cancel * requests. @sqd should be not-null IFF it's an SQPOLL thread cancellation.
@@ -584,7 +625,9 @@ __cold void io_uring_cancel_generic(bool cancel_all, struct io_sq_data *sqd)struct io_uring_task *tctx = current->io_uring;struct io_ring_ctx *ctx;struct io_tctx_node *node; +unsigned int flags =0 ;unsigned long index; +bool group_exit;s64 inflight;DEFINE_WAIT (wait);
@@ -595,18 +638,30 @@ __cold void io_uring_cancel_generic(bool cancel_all, struct io_sq_data *sqd)if (tctx->io_wq)io_wq_exit_start (tctx->io_wq); +/* +* If a whole thread group is exiting, nobody will look at completions. +* If a single thread is exiting, cancel only those that belong to that +* thread. +*/ + group_exit = !cancel_all &&io_task_group_exiting (); +if (cancel_all) + flags =IO_CANCEL_ALL ; +else if (group_exit) + flags =IO_CANCEL_ALL |IO_CANCEL_KEEP_TIMEOUTS ; +if (sqd) + flags |=IO_CANCEL_SQPOLL ; +atomic_inc (&tctx->in_cancel); do { bool loop = false; + unsigned int state; io_uring_drop_tctx_refs(current); - if (!tctx_inflight(tctx, !cancel_all)) + if (io_tctx_cancel_done(tctx, cancel_all, group_exit)) break; /* read completions before cancelations */ inflight = tctx_inflight(tctx, false); - if (!inflight) - break; if (!sqd) { xa_for_each(&tctx->xa, index, node) {
@@ -615,15 +670,13 @@ __cold void io_uring_cancel_generic(bool cancel_all, struct io_sq_data *sqd)continue ; loop |=io_uring_try_cancel_requests (node->ctx, current->io_uring, - cancel_all, - false); + flags); } }else {list_for_each_entry (ctx, &sqd->ctx_list, sqd_list) loop |=io_uring_try_cancel_requests (ctx, current->io_uring, - cancel_all, - true); + flags); }if (loop) {
@@ -631,7 +684,12 @@ __cold void io_uring_cancel_generic(bool cancel_all, struct io_sq_data *sqd)continue ; } -prepare_to_wait (&tctx->wait, &wait,TASK_INTERRUPTIBLE ); + state =TASK_INTERRUPTIBLE ; +if (task_sigpending (current)) + state =TASK_UNINTERRUPTIBLE ; +if (!cancel_all) + state |=TASK_FREEZABLE ; +prepare_to_wait (&tctx->wait, &wait, state);io_run_task_work ();io_uring_drop_tctx_refs (current);xa_for_each (&tctx->xa, index, node) {
@@ -646,8 +704,13 @@ __cold void io_uring_cancel_generic(bool cancel_all, struct io_sq_data *sqd)* avoids a race where a completion comes in before we did * prepare_to_wait(). */ -if (inflight ==tctx_inflight (tctx, !cancel_all)) -schedule (); +if (inflight ==tctx_inflight (tctx, false)) { +unsigned long timeout =1 ; + +if (state &TASK_INTERRUPTIBLE ) + timeout =MAX_SCHEDULE_TIMEOUT ; +schedule_timeout (timeout); + } end_wait:finish_wait (&tctx->wait, &wait); }while (1 );
io_uring/cancel.h +12 -1
@@ -30,9 +30,20 @@ bool io_cancel_remove_all(struct io_ring_ctx *ctx, struct io_uring_task *tctx,int io_cancel_remove (struct io_ring_ctx *ctx,struct io_cancel_data *cd,unsigned int issue_flags,struct hlist_head *list,bool (*cancel)(struct io_kiocb *)); + +/* io_uring_try_cancel_requests() flags */ +enum { +/* match all requests, not just REQ_F_INFLIGHT */ +IO_CANCEL_ALL =1 , +/* ignore timeouts */ +IO_CANCEL_KEEP_TIMEOUTS =2 , +/* called by the SQPOLL thread */ +IO_CANCEL_SQPOLL =4 , +}; +__cold bool io_uring_try_cancel_requests (struct io_ring_ctx *ctx,struct io_uring_task *tctx, -bool cancel_all,bool is_sqpoll_thread); +unsigned int flags);__cold void io_uring_cancel_generic (bool cancel_all,struct io_sq_data *sqd);__cold bool io_cancel_ctx_cb (struct io_wq_work *work,void *data);
io_uring/io_uring.c +1 -1
@@ -2376,7 +2376,7 @@ static __cold void io_ring_exit_work(struct work_struct *work)if (ctx->flags &IORING_SETUP_DEFER_TASKRUN )io_cancel_local_task_work (ctx);cond_resched (); - }while (io_uring_try_cancel_requests (ctx,NULL , true, false)); + }while (io_uring_try_cancel_requests (ctx,NULL ,IO_CANCEL_ALL ));if (ctx->sq_data) {struct io_sq_data *sqd = ctx->sq_data;
io_uring/timeout.c +18 -0
@@ -729,6 +729,24 @@ static bool io_match_task(struct io_kiocb *head, struct io_uring_task *tctx,return false; } +__cold unsigned int io_timeouts_armed (struct io_ring_ctx *ctx, +struct io_uring_task *tctx) +{ +struct io_timeout *timeout; +unsigned int nr =0 ; + +guard (mutex)(&ctx->uring_lock); +raw_spin_lock_irq (&ctx->timeout_lock); +list_for_each_entry (timeout, &ctx->timeout_list, list) { +struct io_kiocb *req =cmd_to_io_kiocb (timeout); + +if (req->tctx == tctx) + nr +=io_linked_nr (req); + } +raw_spin_unlock_irq (&ctx->timeout_lock); +return nr; +} +/* Returns true if we found and killed one or more timeouts */ __cold bool io_kill_timeouts (struct io_ring_ctx *ctx,struct io_uring_task *tctx,bool cancel_all)
io_uring/timeout.h +2 -0
@@ -11,6 +11,8 @@ struct io_timeout_data {__cold void io_flush_timeouts (struct io_ring_ctx *ctx);struct io_cancel_data ;int io_timeout_cancel (struct io_ring_ctx *ctx,struct io_cancel_data *cd); +__cold unsigned int io_timeouts_armed (struct io_ring_ctx *ctx, +struct io_uring_task *tctx);__cold bool io_kill_timeouts (struct io_ring_ctx *ctx,struct io_uring_task *tctx,bool cancel_all);void io_queue_linked_timeout (struct io_kiocb *req);
Signature · 1 line
2.55.0
Jens Axboe
maintainerPATCH 8/10 io_uring: run cancelations synchronously on ring release
In reply to Jens Axboeio_uring_release() just marks the ring as dying and punts everything else to exit_work, including the cancelation of requests that are easily cancelable right away Until that work has run, and the task_work it generates has as well, those requests keep their files pinned. An application that closes its ring and then expects the files it had been using to be closed as well may be confused by this currently not being the case.
Re-factor the cancelation pass out of io_ring_exit_work() and run it directly from release, before punting the rest.
--- io_uring/io_uring.c | 82 ++++++++++++++++++++++++++------------------- 1 file changed, 48 insertions(+), 34 deletions(-)
io_uring/io_uring.c +48 -34
@@ -2345,18 +2345,18 @@ static __cold void io_tctx_exit_cb(struct callback_head *cb)complete (&work->completion); } -static __cold void io_ring_exit_work (struct work_struct *work) +/* +* Cancel what can be canceled on a dying ring and reap what has completed. +* Only waits on polled I/O, never anything else. +*/ +static __cold void io_ring_ctx_cancel (struct io_ring_ctx *ctx) { - struct io_ring_ctx *ctx = container_of(work, struct io_ring_ctx, exit_work); -unsigned long timeout = jiffies +IO_URING_EXIT_WAIT_MAX ; -unsigned long interval =HZ /20 ; -struct io_tctx_exit exit; -struct io_tctx_node *node; -int ret; +struct io_sq_data *sqd = ctx->sq_data; -mutex_lock (&ctx->uring_lock); -io_terminate_zcrx (ctx); -mutex_unlock (&ctx->uring_lock); +if (test_bit (IO_CHECK_CQ_OVERFLOW_BIT , &ctx->check_cq)) { + scoped_guard(mutex, &ctx->uring_lock) +io_cqring_overflow_kill (ctx); + }/* * If we're doing polled IO and end up having requests being
@@ -2365,32 +2365,36 @@ static __cold void io_ring_exit_work(struct work_struct *work)* as nobody else will be looking for them. */ do { - if (test_bit(IO_CHECK_CQ_OVERFLOW_BIT, &ctx->check_cq)) { - mutex_lock(&ctx->uring_lock); - io_cqring_overflow_kill(ctx); - mutex_unlock(&ctx->uring_lock); - } +if (ctx->flags &IORING_SETUP_DEFER_TASKRUN ) +io_cancel_local_task_work (ctx); +cond_resched (); + }while (io_uring_try_cancel_requests (ctx,NULL ,IO_CANCEL_ALL )); - /* The SQPOLL thread never reaches this path */ - do { - if (ctx->flags & IORING_SETUP_DEFER_TASKRUN) - io_cancel_local_task_work(ctx); - cond_resched(); - } while (io_uring_try_cancel_requests(ctx, NULL, IO_CANCEL_ALL)); - - if (ctx->sq_data) { - struct io_sq_data *sqd = ctx->sq_data; - struct task_struct *tsk; - - io_sq_thread_park(sqd); - tsk = sqpoll_task_locked(sqd); - if (tsk && tsk->io_uring && tsk->io_uring->io_wq) - io_wq_cancel_cb(tsk->io_uring->io_wq, - io_cancel_ctx_cb, ctx, true); - io_sq_thread_unpark(sqd); - } +if (sqd) { +struct task_struct *tsk; + +io_sq_thread_park (sqd); + tsk =sqpoll_task_locked (sqd); +if (tsk && tsk->io_uring && tsk->io_uring->io_wq) +io_wq_cancel_cb (tsk->io_uring->io_wq, + io_cancel_ctx_cb, ctx, true); +io_sq_thread_unpark (sqd); + } + +io_req_caches_free (ctx); +} + +static __cold void io_ring_exit_work (struct work_struct *work) +{ + struct io_ring_ctx *ctx = container_of(work, struct io_ring_ctx, exit_work); +unsigned long timeout = jiffies +IO_URING_EXIT_WAIT_MAX ; +unsigned long interval =HZ /20 ; +struct io_tctx_exit exit; +struct io_tctx_node *node; +int ret; - io_req_caches_free(ctx); + do { + io_ring_ctx_cancel(ctx); if (WARN_ON_ONCE(time_after(jiffies, timeout))) { /* there is little hope left, don't run it too often */
@@ -2458,8 +2462,18 @@ static __cold void io_ring_ctx_wait_and_kill(struct io_ring_ctx *ctx)percpu_ref_kill (&ctx->refs);xa_for_each (&ctx->personalities, index, creds)io_unregister_personality (ctx, index); +io_terminate_zcrx (ctx);mutex_unlock (&ctx->uring_lock); +/* +* Do the first round of cancelations upfront rather than leaving it +* to to exit_work. Anything cancelable is then already on its way +* out, and for requests owned by the task closing the ring, this +* ensures any held files are put before close(2) returns. +*/ +if (!(current->flags &PF_IO_WORKER )) +io_ring_ctx_cancel (ctx); +INIT_WORK (&ctx->exit_work, io_ring_exit_work);/* * Use system_dfl_wq to avoid spawning tons of event kworkers
Signature · 1 line
2.55.0
Jens Axboe
maintainerPATCH 9/10 io_uring: drop registered files and buffers at release time
In reply to Jens AxboeRegistered file and buffer tables are only torn down from io_ring_ctx_free(), which only runs when all requests have completed and the ctx refs have hit zero. Before this, every registered file remains open.
The resource nodes are reference counted and any request using a registered file or buffer holds its own reference to the node, so this only releases what is actually unused. The rest follow as those requests complete or get canceled.
--- io_uring/io_uring.c | 5 +++-- 1 file changed, 3 insertions(+), 2 deletions(-)
io_uring/io_uring.c +3 -2
@@ -2196,8 +2196,6 @@ static __cold void io_ring_ctx_free(struct io_ring_ctx *ctx)io_sq_thread_finish (ctx);mutex_lock (&ctx->uring_lock); -io_sqe_buffers_unregister (ctx); -io_sqe_files_unregister (ctx);io_unregister_zcrx (ctx);io_cqring_overflow_kill (ctx);io_eventfd_unregister (ctx);
@@ -2463,6 +2461,9 @@ static __cold void io_ring_ctx_wait_and_kill(struct io_ring_ctx *ctx)xa_for_each (&ctx->personalities, index, creds)io_unregister_personality (ctx, index);io_terminate_zcrx (ctx); +/* Drop these now, rather than async, to unpin the files */ +io_sqe_buffers_unregister (ctx); +io_sqe_files_unregister (ctx);mutex_unlock (&ctx->uring_lock);/*
Signature · 1 line
2.55.0
Jens Axboe
maintainerPATCH 10/10 io_uring: wait for in-flight requests on ring release
In reply to Jens AxboeWith cancelations now run at release time, what's left in-flight on the ring afterwards is mostly I/O that has already been issued to a device and just needs to finish. Until that happens, the files from those requests pin the files they were using. Wait for those.
--- io_uring/io_uring.c | 74 ++++++++++++++++++++++++++++++++++++++++++++- 1 file changed, 73 insertions(+), 1 deletion(-)
io_uring/io_uring.c +73 -1
@@ -2382,6 +2382,76 @@ static __cold void io_ring_ctx_cancel(struct io_ring_ctx *ctx)io_req_caches_free (ctx); } +/* Number of requests that should be waited for */ +static __cold unsigned int io_ring_ctx_inflight (struct io_ring_ctx *ctx) +{ +guard (mutex)(&ctx->uring_lock); +__io_req_caches_free (ctx); +return ctx->nr_req_allocated - ctx->nr_notifs; +} + +/* +* Run task_work completions for current. Only do so if the io_uring callback +* itself can get pruned first, otherwise we risk recursing. +*/ +static __cold bool io_ring_run_own_completions (struct io_uring_task *tctx) +{ +unsigned int count =0 ; + +if (!tctx ||mpscq_empty (&tctx->task_list)) +return true; +if (!task_work_cancel (current, &tctx->task_work)) +return false; +tctx_task_work_run (tctx,UINT_MAX , &count); +return true; +} + +/* +* Requests may remain after cancelations have been run, as not all requests +* are cancelable. Storage I/O is an example. Wait for those so that once +* close(2) returns, files pinned by these requests have been released. +*/ +static __cold void io_ring_ctx_wait_inflight (struct io_ring_ctx *ctx) +{ +struct io_uring_task *tctx = current->io_uring; +bool ran_own = true; + +if (current->flags & (PF_KTHREAD |PF_EXITING )) +return ; +if (tctx &&atomic_read (&tctx->in_cancel)) +return ; + +while (io_ring_ctx_inflight (ctx) && !fatal_signal_pending (current)) { +unsigned int state; + +if (test_thread_flag (TIF_NOTIFY_SIGNAL )) { +clear_notify_signal (); +if (task_work_pending (current)) +set_notify_resume (current); + } + state =TASK_INTERRUPTIBLE ; +if (signal_pending (current)) + state =TASK_KILLABLE ; +set_current_state (state |TASK_FREEZABLE ); +/* don't sleep on work that's already there and that we can run */ +if (ran_own && ((tctx && !mpscq_empty (&tctx->task_list)) || +io_local_work_pending (ctx))) +__set_current_state (TASK_RUNNING ); +else +schedule_timeout (1 ); + +/* completions may be queued behind us */ +if (!io_ring_run_own_completions (tctx)) { +if (!ran_own) +break ; + ran_own = false; + }else { + ran_own = true; + } +io_ring_ctx_cancel (ctx); + } +} +static __cold void io_ring_exit_work (struct work_struct *work) { struct io_ring_ctx *ctx = container_of(work, struct io_ring_ctx, exit_work);
@@ -2472,8 +2542,10 @@ static __cold void io_ring_ctx_wait_and_kill(struct io_ring_ctx *ctx)* out, and for requests owned by the task closing the ring, this * ensures any held files are put before close(2) returns. */ -if (!(current->flags &PF_IO_WORKER )) +if (!(current->flags &PF_IO_WORKER )) {io_ring_ctx_cancel (ctx); +io_ring_ctx_wait_inflight (ctx); + }INIT_WORK (&ctx->exit_work, io_ring_exit_work);/*
Signature · 1 line
2.55.0