io_uring: introduce type alias for io_tw_state
PATCH v2 1/2 io_uring: introduce type alias for io_tw_state
In preparation for changing how io_tw_state is passed, introduce a type alias io_tw_token_t for struct io_tw_state *. This allows for changing the representation in one place, without having to update the many functions that just forward their struct io_tw_state * argument.
Also add a comment to struct io_tw_state to explain its purpose.
--- include/linux/io_uring_types.h | 9 ++++++++- io_uring/futex.c | 16 ++++++++-------- io_uring/io_uring.c | 28 ++++++++++++++-------------- io_uring/io_uring.h | 8 ++++---- io_uring/msg_ring.c | 2 +- io_uring/notif.c | 4 ++-- io_uring/poll.c | 18 +++++++++--------- io_uring/poll.h | 4 +++- io_uring/rw.c | 4 ++-- io_uring/rw.h | 3 ++- io_uring/timeout.c | 16 ++++++++-------- io_uring/uring_cmd.c | 2 +- io_uring/waitid.c | 8 ++++---- 13 files changed, 66 insertions(+), 56 deletions(-)
include/linux/io_uring_types.h +8 -1
@@ -434,12 +434,19 @@ struct io_ring_ctx {struct io_mapped_region ring_region;/* used for optimised request parameter and wait argument passing */ struct io_mapped_region param_region; }; +/* +* Token indicating function is called in task work context: +* ctx->uring_lock is held and any completions generated will be flushed. +* ONLY core io_uring.c should instantiate this struct. +*/ struct io_tw_state { }; +/* Alias to use in code that doesn't instantiate struct io_tw_state */ +typedef struct io_tw_state *io_tw_token_t ;enum {REQ_F_FIXED_FILE_BIT =IOSQE_FIXED_FILE_BIT ,REQ_F_IO_DRAIN_BIT =IOSQE_IO_DRAIN_BIT ,REQ_F_LINK_BIT =IOSQE_IO_LINK_BIT ,
@@ -561,11 +568,11 @@ enum {REQ_F_BUF_NODE =IO_REQ_FLAG (REQ_F_BUF_NODE_BIT ),/* request has read/write metadata assigned */ REQ_F_HAS_METADATA = IO_REQ_FLAG(REQ_F_HAS_METADATA_BIT), }; -typedef void (*io_req_tw_func_t )(struct io_kiocb *req,struct io_tw_state *ts); +typedef void (*io_req_tw_func_t )(struct io_kiocb *req,io_tw_token_t tw);struct io_task_work {struct llist_node node;io_req_tw_func_t func; };
io_uring/futex.c +8 -8
@@ -42,34 +42,34 @@ bool io_futex_cache_init(struct io_ring_ctx *ctx)void io_futex_cache_free (struct io_ring_ctx *ctx) {io_alloc_cache_free (&ctx->futex_cache, kfree); } -static void __io_futex_complete (struct io_kiocb *req,struct io_tw_state *ts) +static void __io_futex_complete (struct io_kiocb *req,io_tw_token_t tw) { req->async_data =NULL ;hlist_del_init (&req->hash_node); -io_req_task_complete (req, ts); +io_req_task_complete (req, tw); } -static void io_futex_complete (struct io_kiocb *req,struct io_tw_state *ts) +static void io_futex_complete (struct io_kiocb *req,io_tw_token_t tw) {struct io_futex_data *ifd = req->async_data;struct io_ring_ctx *ctx = req->ctx; -io_tw_lock (ctx, ts); +io_tw_lock (ctx, tw);if (!io_alloc_cache_put (&ctx->futex_cache, ifd))kfree (ifd); -__io_futex_complete (req, ts); +__io_futex_complete (req, tw); } -static void io_futexv_complete (struct io_kiocb *req,struct io_tw_state *ts) +static void io_futexv_complete (struct io_kiocb *req,io_tw_token_t tw) { struct io_futex *iof = io_kiocb_to_cmd(req, struct io_futex);struct futex_vector *futexv = req->async_data; -io_tw_lock (req->ctx, ts); +io_tw_lock (req->ctx, tw);if (!iof->futexv_unqueued) {int res; res =futex_unqueue_multiple (futexv, iof->futex_nr);
@@ -77,11 +77,11 @@ static void io_futexv_complete(struct io_kiocb *req, struct io_tw_state *ts)io_req_set_res (req, res,0 ); }kfree (req->async_data); req->flags &= ~REQ_F_ASYNC_DATA ; -__io_futex_complete (req, ts); +__io_futex_complete (req, tw); }static bool io_futexv_claim (struct io_futex *iof) { if (test_bit(0, &iof->futexv_owned) ||
io_uring/io_uring.c +14 -14
@@ -541,11 +541,11 @@ static void io_queue_iowq(struct io_kiocb *req)io_wq_enqueue (tctx->io_wq, &req->work);if (link)io_queue_linked_timeout (link); } -static void io_req_queue_iowq_tw (struct io_kiocb *req,struct io_tw_state *ts) +static void io_req_queue_iowq_tw (struct io_kiocb *req,io_tw_token_t tw) {io_queue_iowq (req); }void io_req_queue_iowq (struct io_kiocb *req)
@@ -1020,11 +1020,11 @@ static inline struct io_kiocb *io_req_find_next(struct io_kiocb *req) nxt = req->link; req->link =NULL ;return nxt; } -static void ctx_flush_and_put (struct io_ring_ctx *ctx,struct io_tw_state *ts) +static void ctx_flush_and_put (struct io_ring_ctx *ctx,io_tw_token_t tw) {if (!ctx)return ;if (ctx->flags &IORING_SETUP_TASKRUN_FLAG )atomic_andnot (IORING_SQ_TASKRUN , &ctx->rings->sq_flags);
@@ -1275,31 +1275,31 @@ static bool io_run_local_work_continue(struct io_ring_ctx *ctx, int events,atomic_or (IORING_SQ_TASKRUN , &ctx->rings->sq_flags);return false; }static int __io_run_local_work_loop (struct llist_node **node, -struct io_tw_state *ts, +io_tw_token_t tw,int events) {int ret =0 ;while (*node) {struct llist_node *next = (*node)->next; struct io_kiocb *req = container_of(*node, struct io_kiocb, io_task_work.node);INDIRECT_CALL_2 (req->io_task_work.func, io_poll_task_func, io_req_rw_complete, - req, ts); + req, tw); *node = next;if (++ret >= events)break ; }return ret; } -static int __io_run_local_work (struct io_ring_ctx *ctx,struct io_tw_state *ts, +static int __io_run_local_work (struct io_ring_ctx *ctx,io_tw_token_t tw,int min_events,int max_events) {struct llist_node *node;unsigned int loops =0 ;int ret =0 ;
@@ -1308,20 +1308,20 @@ static int __io_run_local_work(struct io_ring_ctx *ctx, struct io_tw_state *ts,return -EEXIST ;if (ctx->flags &IORING_SETUP_TASKRUN_FLAG )atomic_andnot (IORING_SQ_TASKRUN , &ctx->rings->sq_flags); again: min_events -= ret; - ret =__io_run_local_work_loop (&ctx->retry_llist.first, ts, max_events); + ret =__io_run_local_work_loop (&ctx->retry_llist.first, tw, max_events);if (ctx->retry_llist.first) goto retry_done;/* * llists are in reverse order, flip it back the right way before * running the pending items. */ node =llist_reverse_order (llist_del_all (&ctx->work_llist)); - ret +=__io_run_local_work_loop (&node, ts, max_events - ret); + ret +=__io_run_local_work_loop (&node, tw, max_events - ret); ctx->retry_llist.first = node; loops++;if (io_run_local_work_continue (ctx, ret, min_events)) goto again;
@@ -1355,19 +1355,19 @@ static int io_run_local_work(struct io_ring_ctx *ctx, int min_events, ret =__io_run_local_work (ctx, &ts, min_events, max_events);mutex_unlock (&ctx->uring_lock);return ret; } -static void io_req_task_cancel (struct io_kiocb *req,struct io_tw_state *ts) +static void io_req_task_cancel (struct io_kiocb *req,io_tw_token_t tw) { -io_tw_lock (req->ctx, ts); +io_tw_lock (req->ctx, tw);io_req_defer_failed (req, req->cqe.res); } -void io_req_task_submit (struct io_kiocb *req,struct io_tw_state *ts) +void io_req_task_submit (struct io_kiocb *req,io_tw_token_t tw) { -io_tw_lock (req->ctx, ts); +io_tw_lock (req->ctx, tw);if (unlikely (io_should_terminate_tw ()))io_req_defer_failed (req, -EFAULT );else if (req->flags &REQ_F_FORCE_ASYNC )io_queue_iowq (req);else
@@ -1581,11 +1581,11 @@ static int io_iopoll_check(struct io_ring_ctx *ctx, long min) }while (nr_events < min);return 0 ; } -void io_req_task_complete (struct io_kiocb *req,struct io_tw_state *ts) +void io_req_task_complete (struct io_kiocb *req,io_tw_token_t tw) {io_req_complete_defer (req); }/*
@@ -1761,13 +1761,13 @@ static int io_issue_sqe(struct io_kiocb *req, unsigned int issue_flags)io_iopoll_req_issued (req, issue_flags); }return ret; } -int io_poll_issue (struct io_kiocb *req,struct io_tw_state *ts) +int io_poll_issue (struct io_kiocb *req,io_tw_token_t tw) { -io_tw_lock (req->ctx, ts); +io_tw_lock (req->ctx, tw);return io_issue_sqe (req,IO_URING_F_NONBLOCK |IO_URING_F_MULTISHOT |IO_URING_F_COMPLETE_DEFER ); }struct io_wq_work *io_wq_free_work (struct io_wq_work *work)
io_uring/io_uring.h +4 -4
@@ -88,13 +88,13 @@ struct file *io_file_get_fixed(struct io_kiocb *req, int fd,void __io_req_task_work_add (struct io_kiocb *req,unsigned flags);void io_req_task_work_add_remote (struct io_kiocb *req,struct io_ring_ctx *ctx,unsigned flags);bool io_alloc_async_data (struct io_kiocb *req);void io_req_task_queue (struct io_kiocb *req); -void io_req_task_complete (struct io_kiocb *req,struct io_tw_state *ts); +void io_req_task_complete (struct io_kiocb *req,io_tw_token_t tw);void io_req_task_queue_fail (struct io_kiocb *req,int ret); -void io_req_task_submit (struct io_kiocb *req,struct io_tw_state *ts); +void io_req_task_submit (struct io_kiocb *req,io_tw_token_t tw);struct llist_node *io_handle_tw_list (struct llist_node *node,unsigned int *count,unsigned int max_entries);struct llist_node *tctx_task_work_run (struct io_uring_task *tctx,unsigned int max_entries,unsigned int *count);void tctx_task_work (struct callback_head *cb);__cold void io_uring_cancel_generic (bool cancel_all,struct io_sq_data *sqd); int io_uring_alloc_task_context(struct task_struct *task,
@@ -102,11 +102,11 @@ int io_uring_alloc_task_context(struct task_struct *task,int io_ring_add_registered_file (struct io_uring_task *tctx,struct file *file,int start,int end);void io_req_queue_iowq (struct io_kiocb *req); -int io_poll_issue (struct io_kiocb *req,struct io_tw_state *ts); +int io_poll_issue (struct io_kiocb *req,io_tw_token_t tw);int io_submit_sqes (struct io_ring_ctx *ctx,unsigned int nr);int io_do_iopoll (struct io_ring_ctx *ctx,bool force_nonspin);void __io_submit_flush_completions (struct io_ring_ctx *ctx);struct io_wq_work *io_wq_free_work (struct io_wq_work *work);
@@ -374,11 +374,11 @@ static inline bool io_local_work_pending(struct io_ring_ctx *ctx)static inline bool io_task_work_pending (struct io_ring_ctx *ctx) {return task_work_pending (current) ||io_local_work_pending (ctx); } -static inline void io_tw_lock (struct io_ring_ctx *ctx,struct io_tw_state *ts) +static inline void io_tw_lock (struct io_ring_ctx *ctx,io_tw_token_t tw) {lockdep_assert_held (&ctx->uring_lock); }/*
io_uring/msg_ring.c +1 -1
@@ -69,11 +69,11 @@ void io_msg_ring_cleanup(struct io_kiocb *req)static inline bool io_msg_need_remote (struct io_ring_ctx *target_ctx) {return target_ctx->task_complete; } -static void io_msg_tw_complete (struct io_kiocb *req,struct io_tw_state *ts) +static void io_msg_tw_complete (struct io_kiocb *req,io_tw_token_t tw) {struct io_ring_ctx *ctx = req->ctx;io_add_aux_cqe (ctx, req->cqe.user_data, req->cqe.res, req->cqe.flags);if (spin_trylock (&ctx->msg_lock)) {
io_uring/notif.c +2 -2
@@ -9,11 +9,11 @@#include "notif.h" #include "rsrc.h" static const struct ubuf_info_ops io_ubuf_ops; -static void io_notif_tw_complete (struct io_kiocb *notif,struct io_tw_state *ts) +static void io_notif_tw_complete (struct io_kiocb *notif,io_tw_token_t tw) {struct io_notif_data *nd =io_notif_to_data (notif); do { notif = cmd_to_io_kiocb(nd);
@@ -27,11 +27,11 @@ static void io_notif_tw_complete(struct io_kiocb *notif, struct io_tw_state *ts)__io_unaccount_mem (notif->ctx->user, nd->account_pages); nd->account_pages =0 ; } nd = nd->next; -io_req_task_complete (notif, ts); +io_req_task_complete (notif, tw); }while (nd); }void io_tx_ubuf_complete (struct sk_buff *skb,struct ubuf_info *uarg,bool success)
io_uring/poll.c +9 -9
@@ -218,11 +218,11 @@ static inline void io_poll_execute(struct io_kiocb *req, int res)* require, which is either spurious wakeup or multishot CQE is served. * IOU_POLL_DONE when it's done with the request, then the mask is stored in * req->cqe.res. IOU_POLL_REMOVE_POLL_USE_RES indicates to remove multishot * poll and that the result is stored in req->cqe. */ -static int io_poll_check_events (struct io_kiocb *req,struct io_tw_state *ts) +static int io_poll_check_events (struct io_kiocb *req,io_tw_token_t tw) {int v;if (unlikely (io_should_terminate_tw ()))return -ECANCELED ;
@@ -286,11 +286,11 @@ static int io_poll_check_events(struct io_kiocb *req, struct io_tw_state *ts)if (!io_req_post_cqe (req, mask,IORING_CQE_F_MORE )) {io_req_set_res (req, mask,0 );return IOU_POLL_REMOVE_POLL_USE_RES ; } }else { -int ret =io_poll_issue (req, ts); +int ret =io_poll_issue (req, tw);if (ret ==IOU_STOP_MULTISHOT )return IOU_POLL_REMOVE_POLL_USE_RES ;else if (ret ==IOU_REQUEUE )return IOU_POLL_REQUEUE ;if (ret <0 )
@@ -309,15 +309,15 @@ static int io_poll_check_events(struct io_kiocb *req, struct io_tw_state *ts)io_napi_add (req);return IOU_POLL_NO_ACTION ; } -void io_poll_task_func (struct io_kiocb *req,struct io_tw_state *ts) +void io_poll_task_func (struct io_kiocb *req,io_tw_token_t tw) {int ret; - ret =io_poll_check_events (req, ts); + ret =io_poll_check_events (req, tw);if (ret ==IOU_POLL_NO_ACTION ) {io_kbuf_recycle (req,0 );return ; }else if (ret ==IOU_POLL_REQUEUE ) {io_kbuf_recycle (req,0 );
@@ -333,26 +333,26 @@ void io_poll_task_func(struct io_kiocb *req, struct io_tw_state *ts)struct io_poll *poll; poll = io_kiocb_to_cmd(req, struct io_poll); req->cqe.res =mangle_poll (req->cqe.res & poll->events); }else if (ret ==IOU_POLL_REISSUE ) { -io_req_task_submit (req, ts); +io_req_task_submit (req, tw);return ; }else if (ret !=IOU_POLL_REMOVE_POLL_USE_RES ) { req->cqe.res = ret;req_set_fail (req); }io_req_set_res (req, req->cqe.res,0 ); -io_req_task_complete (req, ts); +io_req_task_complete (req, tw); }else { -io_tw_lock (req->ctx, ts); +io_tw_lock (req->ctx, tw);if (ret ==IOU_POLL_REMOVE_POLL_USE_RES ) -io_req_task_complete (req, ts); +io_req_task_complete (req, tw);else if (ret ==IOU_POLL_DONE || ret ==IOU_POLL_REISSUE ) -io_req_task_submit (req, ts); +io_req_task_submit (req, tw);else io_req_defer_failed (req, ret); } }
io_uring/poll.h +3 -1
@@ -1,7 +1,9 @@// SPDX-License-Identifier: GPL-2.0 +#include <linux/io_uring_types.h> +#define IO_POLL_ALLOC_CACHE_MAX 32enum {IO_APOLL_OK ,IO_APOLL_ABORTED ,
@@ -41,6 +43,6 @@ int io_poll_cancel(struct io_ring_ctx *ctx, struct io_cancel_data *cd, unsigned issue_flags);int io_arm_poll_handler (struct io_kiocb *req,unsigned issue_flags);bool io_poll_remove_all (struct io_ring_ctx *ctx,struct io_uring_task *tctx,bool cancel_all); -void io_poll_task_func (struct io_kiocb *req,struct io_tw_state *ts); +void io_poll_task_func (struct io_kiocb *req,io_tw_token_t tw);
io_uring/rw.c +2 -2
@@ -509,11 +509,11 @@ static inline int io_fixup_rw_res(struct io_kiocb *req, long res) res += io->bytes_done; }return res; } -void io_req_rw_complete (struct io_kiocb *req,struct io_tw_state *ts) +void io_req_rw_complete (struct io_kiocb *req,io_tw_token_t tw) { struct io_rw *rw = io_kiocb_to_cmd(req, struct io_rw);struct kiocb *kiocb = &rw->kiocb;if ((kiocb->ki_flags &IOCB_DIO_CALLER_COMP ) && kiocb->dio_complete) {
@@ -526,11 +526,11 @@ void io_req_rw_complete(struct io_kiocb *req, struct io_tw_state *ts)if (req->flags & (REQ_F_BUFFER_SELECTED |REQ_F_BUFFER_RING )) req->cqe.flags |=io_put_kbuf (req, req->cqe.res,0 );io_req_rw_cleanup (req,0 ); -io_req_task_complete (req, ts); +io_req_task_complete (req, tw); }static void io_complete_rw (struct kiocb *kiocb,long res) { struct io_rw *rw = container_of(kiocb, struct io_rw, kiocb);
io_uring/rw.h +2 -1
@@ -1,7 +1,8 @@// SPDX-License-Identifier: GPL-2.0 +#include <linux/io_uring_types.h> #include <linux/pagemap.h> struct io_meta_state {u32 seed;struct iov_iter_state iter_meta;
@@ -37,9 +38,9 @@ int io_prep_read(struct io_kiocb *req, const struct io_uring_sqe *sqe);int io_prep_write (struct io_kiocb *req,const struct io_uring_sqe *sqe);int io_read (struct io_kiocb *req,unsigned int issue_flags);int io_write (struct io_kiocb *req,unsigned int issue_flags);void io_readv_writev_cleanup (struct io_kiocb *req);void io_rw_fail (struct io_kiocb *req); -void io_req_rw_complete (struct io_kiocb *req,struct io_tw_state *ts); +void io_req_rw_complete (struct io_kiocb *req,io_tw_token_t tw);int io_read_mshot_prep (struct io_kiocb *req,const struct io_uring_sqe *sqe);int io_read_mshot (struct io_kiocb *req,unsigned int issue_flags);void io_rw_cache_free (const void *entry);
io_uring/timeout.c +8 -8
@@ -63,11 +63,11 @@ static inline bool io_timeout_finish(struct io_timeout *timeout,return true; }static enum hrtimer_restart io_timeout_fn (struct hrtimer *timer); -static void io_timeout_complete (struct io_kiocb *req,struct io_tw_state *ts) +static void io_timeout_complete (struct io_kiocb *req,io_tw_token_t tw) { struct io_timeout *timeout = io_kiocb_to_cmd(req, struct io_timeout);struct io_timeout_data *data = req->async_data;struct io_ring_ctx *ctx = req->ctx;
@@ -80,11 +80,11 @@ static void io_timeout_complete(struct io_kiocb *req, struct io_tw_state *ts)raw_spin_unlock_irq (&ctx->timeout_lock);return ; } } -io_req_task_complete (req, ts); +io_req_task_complete (req, tw); }static __cold bool io_flush_killed_timeouts (struct list_head *list,int err) {if (list_empty (list))
@@ -152,22 +152,22 @@ __cold void io_flush_timeouts(struct io_ring_ctx *ctx) ctx->cq_last_tm_flush = seq;raw_spin_unlock_irq (&ctx->timeout_lock);io_flush_killed_timeouts (&list,0 ); } -static void io_req_tw_fail_links (struct io_kiocb *link,struct io_tw_state *ts) +static void io_req_tw_fail_links (struct io_kiocb *link,io_tw_token_t tw) { -io_tw_lock (link->ctx, ts); +io_tw_lock (link->ctx, tw);while (link) {struct io_kiocb *nxt = link->link;long res = -ECANCELED ;if (link->flags &REQ_F_FAIL ) res = link->cqe.res; link->link =NULL ;io_req_set_res (link, res,0 ); -io_req_task_complete (link, ts); +io_req_task_complete (link, tw); link = nxt; } }static void io_fail_links (struct io_kiocb *req)
@@ -310,11 +310,11 @@ int io_timeout_cancel(struct io_ring_ctx *ctx, struct io_cancel_data *cd)return PTR_ERR (req);io_req_task_queue_fail (req, -ECANCELED );return 0 ; } -static void io_req_task_link_timeout (struct io_kiocb *req,struct io_tw_state *ts) +static void io_req_task_link_timeout (struct io_kiocb *req,io_tw_token_t tw) { struct io_timeout *timeout = io_kiocb_to_cmd(req, struct io_timeout);struct io_kiocb *prev = timeout->prev;int ret;
@@ -328,15 +328,15 @@ static void io_req_task_link_timeout(struct io_kiocb *req, struct io_tw_state *t ret =io_try_cancel (req->tctx, &cd,0 ); }else { ret = -ECANCELED ; }io_req_set_res (req, ret ?: -ETIME ,0 ); -io_req_task_complete (req, ts); +io_req_task_complete (req, tw);io_put_req (prev); }else {io_req_set_res (req, -ETIME ,0 ); -io_req_task_complete (req, ts); +io_req_task_complete (req, tw); } }static enum hrtimer_restart io_link_timeout_fn (struct hrtimer *timer) {
io_uring/uring_cmd.c +1 -1
@@ -103,11 +103,11 @@ void io_uring_cmd_mark_cancelable(struct io_uring_cmd *cmd,io_ring_submit_unlock (ctx, issue_flags); } }EXPORT_SYMBOL_GPL (io_uring_cmd_mark_cancelable); -static void io_uring_cmd_work (struct io_kiocb *req,struct io_tw_state *ts) +static void io_uring_cmd_work (struct io_kiocb *req,io_tw_token_t tw) { struct io_uring_cmd *ioucmd = io_kiocb_to_cmd(req, struct io_uring_cmd);unsigned int flags =IO_URING_F_COMPLETE_DEFER ;if (io_should_terminate_tw ())
io_uring/waitid.c +4 -4
@@ -14,11 +14,11 @@#include "io_uring.h" #include "cancel.h" #include "waitid.h" #include "../kernel/exit.h" -static void io_waitid_cb (struct io_kiocb *req,struct io_tw_state *ts); +static void io_waitid_cb (struct io_kiocb *req,io_tw_token_t tw);#define IO_WAITID_CANCEL_FLAG BIT(31)#define IO_WAITID_REF_MASK GENMASK(30, 0)struct io_waitid {
@@ -183,17 +183,17 @@ static inline bool io_waitid_drop_issue_ref(struct io_kiocb *req)io_req_task_work_add (req);remove_wait_queue (iw->head, &iwa->wo.child_wait);return true; } -static void io_waitid_cb (struct io_kiocb *req,struct io_tw_state *ts) +static void io_waitid_cb (struct io_kiocb *req,io_tw_token_t tw) {struct io_waitid_async *iwa = req->async_data;struct io_ring_ctx *ctx = req->ctx;int ret; -io_tw_lock (ctx, ts); +io_tw_lock (ctx, tw); ret =__do_wait (&iwa->wo);/* * If we get -ERESTARTSYS here, we need to re-arm and check again
@@ -219,11 +219,11 @@ static void io_waitid_cb(struct io_kiocb *req, struct io_tw_state *ts)remove_wait_queue (iw->head, &iwa->wo.child_wait); } }io_waitid_complete (req, ret); -io_req_task_complete (req, ts); +io_req_task_complete (req, tw); }static int io_waitid_wait (struct wait_queue_entry *wait,unsigned mode,int sync,void *key) {
Signature · 1 line
2.45.2
PATCH v2 2/2 io_uring: pass struct io_tw_state by value
8e5b3b89ecaf ("io_uring: remove struct io_tw_state::locked") removed the only field of io_tw_state but kept it as a task work callback argument to "forc[e] users not to invoke them carelessly out of a wrong context". Passing the struct io_tw_state * argument adds a few instructions to all callers that can't inline the functions and see the argument is unused.
So pass struct io_tw_state by value instead. Since it's a 0-sized value, it can be passed without any instructions needed to initialize it.
--- include/linux/io_uring_types.h | 2 +- io_uring/io_uring.c | 14 +++++++------- 2 files changed, 8 insertions(+), 8 deletions(-)
include/linux/io_uring_types.h +1 -1
@@ -442,11 +442,11 @@ struct io_ring_ctx {* ONLY core io_uring.c should instantiate this struct. */ struct io_tw_state { };/* Alias to use in code that doesn't instantiate struct io_tw_state */ -typedef struct io_tw_state *io_tw_token_t ; +typedef struct io_tw_state io_tw_token_t ;enum {REQ_F_FIXED_FILE_BIT =IOSQE_FIXED_FILE_BIT ,REQ_F_IO_DRAIN_BIT =IOSQE_IO_DRAIN_BIT ,REQ_F_LINK_BIT =IOSQE_IO_LINK_BIT ,
io_uring/io_uring.c +7 -7
@@ -253,11 +253,11 @@ static __cold void io_fallback_req_func(struct work_struct *work)struct io_tw_state ts = {};percpu_ref_get (&ctx->refs);mutex_lock (&ctx->uring_lock);llist_for_each_entry_safe (req, tmp, node, io_task_work.node) - req->io_task_work.func (req, &ts); + req->io_task_work.func (req, ts);io_submit_flush_completions (ctx);mutex_unlock (&ctx->uring_lock);percpu_ref_put (&ctx->refs); }
@@ -1050,28 +1050,28 @@ struct llist_node *io_handle_tw_list(struct llist_node *node,struct llist_node *next = node->next; struct io_kiocb *req = container_of(node, struct io_kiocb, io_task_work.node);if (req->ctx != ctx) { -ctx_flush_and_put (ctx, &ts); +ctx_flush_and_put (ctx, ts); ctx = req->ctx;mutex_lock (&ctx->uring_lock);percpu_ref_get (&ctx->refs); }INDIRECT_CALL_2 (req->io_task_work.func, io_poll_task_func, io_req_rw_complete, - req, &ts); + req, ts); node = next; (*count)++;if (unlikely (need_resched ())) { -ctx_flush_and_put (ctx, &ts); +ctx_flush_and_put (ctx, ts); ctx =NULL ;cond_resched (); } }while (node && *count < max_entries); -ctx_flush_and_put (ctx, &ts); +ctx_flush_and_put (ctx, ts);return node; }static __cold void __io_fallback_tw (struct llist_node *node,bool sync) {
@@ -1339,22 +1339,22 @@ static inline int io_run_local_work_locked(struct io_ring_ctx *ctx, {struct io_tw_state ts = {};if (!io_local_work_pending (ctx))return 0 ; -return __io_run_local_work (ctx, &ts, min_events, +return __io_run_local_work (ctx, ts, min_events,max (IO_LOCAL_TW_DEFAULT_MAX , min_events)); }static int io_run_local_work (struct io_ring_ctx *ctx,int min_events,int max_events) {struct io_tw_state ts = {};int ret;mutex_lock (&ctx->uring_lock); - ret =__io_run_local_work (ctx, &ts, min_events, max_events); + ret =__io_run_local_work (ctx, ts, min_events, max_events);mutex_unlock (&ctx->uring_lock);return ret; }static void io_req_task_cancel (struct io_kiocb *req,io_tw_token_t tw)
Signature · 1 line
2.45.2
Quoted 8 lines · Caleb Sander Mateos
In preparation for changing how
io_tw_stateis passed, introduce a type aliasio_tw_token_tfor structio_tw_state*. This allows for changing the representation in one place, without having to update the many functions that just forward their structio_tw_state* argument.Also add a comment to struct
io_tw_stateto explain its purpose.[...]
Applied, thanks!
[1/2] io_uring: introduce type alias for io_tw_state
commit: bcf8a0293a019bb0c4aebafdebe9a1e7a923249a
[2/2] io_uring: pass struct io_tw_state by value
commit: 94a4274bb6ebc5b4293559304d0f00928de0d8c0Best regards,
Signature · 1 line
Jens Axboe