io_uring: unify task_work cancelation checks
Jens Axboe
maintainerPATCH 1/1 io_uring: unify task_work cancelation checks
Rather than do per-tw checking, which needs to dip into the task_struct for checking flags, do it upfront before running task_work. This places a 'cancel' member in io_tw_token_t, which is assigned before running task_work for that given ctx.
This is both more efficient in doing it upfront rather than for every task_work, and it means that io_should_terminate_tw() can be made private in io_uring.c rather than need to be called by various callbacks of task_work.
Signed-off-by Jens Axboe <[email protected]>
---
include/linux/io_uring_types.h +1 -0
@@ -474,6 +474,7 @@ struct io_ring_ctx {* ONLY core io_uring.c should instantiate this struct. */ struct io_tw_state { +bool cancel; };/* Alias to use in code that doesn't instantiate struct io_tw_state */ typedef struct io_tw_state io_tw_token_t ;
io_uring/io_uring.c +20 -7
@@ -265,6 +265,20 @@ static __cold void io_ring_ctx_ref_free(struct percpu_ref *ref)complete (&ctx->ref_comp); } +/* +* Terminate the request if either of these conditions are true: +* +* 1) It's being executed by the original task, but that task is marked +* with PF_EXITING as it's exiting. +* 2) PF_KTHREAD is set, in which case the invoker of the task_work is +* our fallback task_work. +* 3) The ring has been closed and is going away. +*/ +static inline bool io_should_terminate_tw (struct io_ring_ctx *ctx) +{ +return (current->flags & (PF_EXITING |PF_KTHREAD )) ||percpu_ref_is_dying (&ctx->refs); +} +static __cold void io_fallback_req_func (struct work_struct *work) { struct io_ring_ctx *ctx = container_of(work, struct io_ring_ctx,
@@ -275,8 +289,10 @@ static __cold void io_fallback_req_func(struct work_struct *work)percpu_ref_get (&ctx->refs);mutex_lock (&ctx->uring_lock); -llist_for_each_entry_safe (req, tmp, node, io_task_work.node) +llist_for_each_entry_safe (req, tmp, node, io_task_work.node) { + ts.cancel =io_should_terminate_tw (req->ctx); req->io_task_work.func (req, ts); + }io_submit_flush_completions (ctx);mutex_unlock (&ctx->uring_lock);percpu_ref_put (&ctx->refs);
@@ -1147,6 +1163,7 @@ struct llist_node *io_handle_tw_list(struct llist_node *node, ctx = req->ctx;mutex_lock (&ctx->uring_lock);percpu_ref_get (&ctx->refs); + ts.cancel =io_should_terminate_tw (ctx); } INDIRECT_CALL_2(req->io_task_work.func, io_poll_task_func, io_req_rw_complete,
@@ -1205,11 +1222,6 @@ struct llist_node *tctx_task_work_run(struct io_uring_task *tctx, {struct llist_node *node; -if (unlikely (current->flags &PF_EXITING )) { -io_fallback_tw (tctx, true); -return NULL ; - } - node =llist_del_all (&tctx->task_list);if (node) { node =llist_reverse_order (node);
@@ -1399,6 +1411,7 @@ static int __io_run_local_work(struct io_ring_ctx *ctx, io_tw_token_t tw,if (ctx->flags &IORING_SETUP_TASKRUN_FLAG )atomic_andnot (IORING_SQ_TASKRUN , &ctx->rings->sq_flags); again: + tw.cancel =io_should_terminate_tw (ctx); min_events -= ret; ret =__io_run_local_work_loop (&ctx->retry_llist.first, tw, max_events);if (ctx->retry_llist.first)
@@ -1458,7 +1471,7 @@ void io_req_task_submit(struct io_kiocb *req, io_tw_token_t tw)struct io_ring_ctx *ctx = req->ctx;io_tw_lock (ctx, tw); -if (unlikely (io_should_terminate_tw (ctx))) +if (unlikely (tw.cancel))io_req_defer_failed (req, -EFAULT );else if (req->flags &REQ_F_FORCE_ASYNC )io_queue_iowq (req);
io_uring/io_uring.h +0 -13
@@ -558,19 +558,6 @@ static inline bool io_allowed_run_tw(struct io_ring_ctx *ctx) ctx->submitter_task == current); } -/* -* Terminate the request if either of these conditions are true: -* -* 1) It's being executed by the original task, but that task is marked -* with PF_EXITING as it's exiting. -* 2) PF_KTHREAD is set, in which case the invoker of the task_work is -* our fallback task_work. -*/ -static inline bool io_should_terminate_tw (struct io_ring_ctx *ctx) -{ -return (current->flags & (PF_KTHREAD |PF_EXITING )) ||percpu_ref_is_dying (&ctx->refs); -} -static inline void io_req_queue_tw_complete (struct io_kiocb *req,s32 res) {io_req_set_res (req, res,0 );
io_uring/poll.c +1 -1
@@ -224,7 +224,7 @@ static int io_poll_check_events(struct io_kiocb *req, io_tw_token_t tw) {int v; -if (unlikely (io_should_terminate_tw (req->ctx))) +if (unlikely (tw.cancel))return -ECANCELED ; do {
io_uring/timeout.c +1 -1
@@ -324,7 +324,7 @@ static void io_req_task_link_timeout(struct io_kiocb *req, io_tw_token_t tw)int ret;if (prev) { -if (!io_should_terminate_tw (req->ctx)) { +if (!tw.cancel) {struct io_cancel_data cd = { .ctx = req->ctx, .data = prev->cqe.user_data,
io_uring/uring_cmd.c +1 -1
@@ -118,7 +118,7 @@ static void io_uring_cmd_work(struct io_kiocb *req, io_tw_token_t tw) struct io_uring_cmd *ioucmd = io_kiocb_to_cmd(req, struct io_uring_cmd);unsigned int flags =IO_URING_F_COMPLETE_DEFER ; -if (io_should_terminate_tw (req->ctx)) +if (unlikely (tw.cancel)) flags |=IO_URING_F_TASK_DEAD ;/* task_work executor checks the deffered list completion */
Signature · 1 line
Jens Axboe