introduce tw state
cover introduce tw state
Add a task_work state instead of passing a raw bool pointer. This will better encapsulate it, e.g. to not expose too much to cmd requests, and may also be needed for further extensions.
Pavel Begunkov (2):
io_uring: remove extra tw trylocks
io_uring: encapsulate task_work state
include/linux/io_uring_types.h | 7 +++- io_uring/io_uring.c | 74 +++++++++++++++++----------------- io_uring/io_uring.h | 14 +++---- io_uring/notif.c | 4 +- io_uring/poll.c | 32 +++++++-------- io_uring/rw.c | 6 +-- io_uring/timeout.c | 14 +++---- io_uring/uring_cmd.c | 2 +- 8 files changed, 79 insertions(+), 74 deletions(-)
Signature · 1 line
2.39.1
PATCH 1/2 io_uring: remove extra tw trylocks
Before cond_resched()'ing in handle_tw_list() we also drop the current ring context, and so the next loop iteration will need to pick/pin a new context and do trylock.
The chunk removed by this patch was intended to be an optimisation covering exactly this case, i.e. retaking the lock after reschedule, but in reality it's skipped for the first iteration after resched as described and will keep hammering the lock if it's contended.
--- io_uring/io_uring.c | 3 +-- 1 file changed, 1 insertion(+), 2 deletions(-)
io_uring/io_uring.c +1 -2
@@ -1186,8 +1186,7 @@ static unsigned int handle_tw_list(struct llist_node *node,/* if not contended, grab and improve batching */ *locked =mutex_trylock (&(*ctx)->uring_lock);percpu_ref_get (&(*ctx)->refs); - }else if (!*locked) - *locked =mutex_trylock (&(*ctx)->uring_lock); + } req->io_task_work.func (req, locked); node = next; count++;
Signature · 1 line
2.39.1
PATCH 2/2 io_uring: encapsulate task_work state
In reply to Pavel BegunkovFor task works we're passing around a bool pointer for whether the current ring is locked or not, let's wrap it in a structure, that will make it more opaque preventing abuse and will also help us to pass more info in the future if needed.
--- include/linux/io_uring_types.h | 7 +++- io_uring/io_uring.c | 71 +++++++++++++++++----------------- io_uring/io_uring.h | 14 +++---- io_uring/notif.c | 4 +- io_uring/poll.c | 32 +++++++-------- io_uring/rw.c | 6 +-- io_uring/timeout.c | 14 +++---- io_uring/uring_cmd.c | 2 +- 8 files changed, 78 insertions(+), 72 deletions(-)
include/linux/io_uring_types.h +6 -1
@@ -367,6 +367,11 @@ struct io_ring_ctx {unsigned evfd_last_cq_tail; }; +struct io_tw_state { +/* ->uring_lock is taken, callbacks can use io_tw_lock to lock it */ +bool locked; +}; +enum {REQ_F_FIXED_FILE_BIT =IOSQE_FIXED_FILE_BIT ,REQ_F_IO_DRAIN_BIT =IOSQE_IO_DRAIN_BIT ,
@@ -473,7 +478,7 @@ enum { REQ_F_HASH_LOCKED = BIT(REQ_F_HASH_LOCKED_BIT), }; -typedef void (*io_req_tw_func_t )(struct io_kiocb *req,bool *locked); +typedef void (*io_req_tw_func_t )(struct io_kiocb *req,struct io_tw_state *ts);struct io_task_work {struct llist_node node;
io_uring/io_uring.c +36 -35
205 lines not on this page · raw message
io_uring/io_uring.h +7 -7
@@ -52,16 +52,16 @@ void __io_req_task_work_add(struct io_kiocb *req, bool allow_local);bool io_is_uring_fops (struct file *file);bool io_alloc_async_data (struct io_kiocb *req);void io_req_task_queue (struct io_kiocb *req); -void io_queue_iowq (struct io_kiocb *req,bool *dont_use); -void io_req_task_complete (struct io_kiocb *req,bool *locked); +void io_queue_iowq (struct io_kiocb *req,struct io_tw_state *ts_dont_use); +void io_req_task_complete (struct io_kiocb *req,struct io_tw_state *ts);void io_req_task_queue_fail (struct io_kiocb *req,int ret); -void io_req_task_submit (struct io_kiocb *req,bool *locked); +void io_req_task_submit (struct io_kiocb *req,struct io_tw_state *ts);void tctx_task_work (struct callback_head *cb);__cold void io_uring_cancel_generic (bool cancel_all,struct io_sq_data *sqd);int io_uring_alloc_task_context (struct task_struct *task,struct io_ring_ctx *ctx); -int io_poll_issue (struct io_kiocb *req,bool *locked); +int io_poll_issue (struct io_kiocb *req,struct io_tw_state *ts);int io_submit_sqes (struct io_ring_ctx *ctx,unsigned int nr);int io_do_iopoll (struct io_ring_ctx *ctx,bool force_nonspin);void io_free_batch_list (struct io_ring_ctx *ctx,struct io_wq_work_node *node);
@@ -299,11 +299,11 @@ static inline bool io_task_work_pending(struct io_ring_ctx *ctx)return task_work_pending (current) || !wq_list_empty (&ctx->work_llist); } -static inline void io_tw_lock (struct io_ring_ctx *ctx,bool *locked) +static inline void io_tw_lock (struct io_ring_ctx *ctx,struct io_tw_state *ts) { -if (!*locked) { +if (!ts->locked) {mutex_lock (&ctx->uring_lock); - *locked = true; + ts->locked = true; } }
io_uring/notif.c +2 -2
@@ -9,7 +9,7 @@#include "notif.h" #include "rsrc.h" -static void io_notif_complete_tw_ext (struct io_kiocb *notif,bool *locked) +static void io_notif_complete_tw_ext (struct io_kiocb *notif,struct io_tw_state *ts) {struct io_notif_data *nd =io_notif_to_data (notif);struct io_ring_ctx *ctx = notif->ctx;
@@ -21,7 +21,7 @@ static void io_notif_complete_tw_ext(struct io_kiocb *notif, bool *locked)__io_unaccount_mem (ctx->user, nd->account_pages); nd->account_pages =0 ; } -io_req_task_complete (notif, locked); +io_req_task_complete (notif, ts); } static void io_tx_ubuf_callback(struct sk_buff *skb, struct ubuf_info *uarg,
io_uring/poll.c +16 -16
@@ -148,7 +148,7 @@ static void io_poll_req_insert_locked(struct io_kiocb *req)hlist_add_head (&req->hash_node, &table->hbs[index].list); } -static void io_poll_tw_hash_eject (struct io_kiocb *req,bool *locked) +static void io_poll_tw_hash_eject (struct io_kiocb *req,struct io_tw_state *ts) {struct io_ring_ctx *ctx = req->ctx;
@@ -159,7 +159,7 @@ static void io_poll_tw_hash_eject(struct io_kiocb *req, bool *locked)* already grabbed the mutex for us, but there is a chance it * failed. */ -io_tw_lock (ctx, locked); +io_tw_lock (ctx, ts);hash_del (&req->hash_node); req->flags &= ~REQ_F_HASH_LOCKED ; }else {
@@ -238,7 +238,7 @@ enum {* req->cqe.res. IOU_POLL_REMOVE_POLL_USE_RES indicates to remove multishot * poll and that the result is stored in req->cqe. */ -static int io_poll_check_events (struct io_kiocb *req,bool *locked) +static int io_poll_check_events (struct io_kiocb *req,struct io_tw_state *ts) {int v;
@@ -300,13 +300,13 @@ static int io_poll_check_events(struct io_kiocb *req, bool *locked)__poll_t mask =mangle_poll (req->cqe.res & req->apoll_events); -if (!io_aux_cqe (req->ctx, *locked, req->cqe.user_data, +if (!io_aux_cqe (req->ctx, ts->locked, req->cqe.user_data, mask,IORING_CQE_F_MORE , false)) {io_req_set_res (req, mask,0 );return IOU_POLL_REMOVE_POLL_USE_RES ; } }else { -int ret =io_poll_issue (req, locked); +int ret =io_poll_issue (req, ts);if (ret ==IOU_STOP_MULTISHOT )return IOU_POLL_REMOVE_POLL_USE_RES ;if (ret <0 )
@@ -326,15 +326,15 @@ static int io_poll_check_events(struct io_kiocb *req, bool *locked)return IOU_POLL_NO_ACTION ; } -static void io_poll_task_func (struct io_kiocb *req,bool *locked) +static void io_poll_task_func (struct io_kiocb *req,struct io_tw_state *ts) {int ret; - ret =io_poll_check_events (req, locked); + ret =io_poll_check_events (req, ts);if (ret ==IOU_POLL_NO_ACTION )return ;io_poll_remove_entries (req); -io_poll_tw_hash_eject (req, locked); +io_poll_tw_hash_eject (req, ts);if (req->opcode ==IORING_OP_POLL_ADD ) {if (ret ==IOU_POLL_DONE ) {
@@ -343,7 +343,7 @@ static void io_poll_task_func(struct io_kiocb *req, bool *locked) poll = io_kiocb_to_cmd(req, struct io_poll); req->cqe.res =mangle_poll (req->cqe.res & poll->events); }else if (ret ==IOU_POLL_REISSUE ) { -io_req_task_submit (req, locked); +io_req_task_submit (req, ts);return ; }else if (ret !=IOU_POLL_REMOVE_POLL_USE_RES ) { req->cqe.res = ret;
@@ -351,14 +351,14 @@ static void io_poll_task_func(struct io_kiocb *req, bool *locked) }io_req_set_res (req, req->cqe.res,0 ); -io_req_task_complete (req, locked); +io_req_task_complete (req, ts); }else { -io_tw_lock (req->ctx, locked); +io_tw_lock (req->ctx, ts);if (ret ==IOU_POLL_REMOVE_POLL_USE_RES ) -io_req_task_complete (req, locked); +io_req_task_complete (req, ts);else if (ret ==IOU_POLL_DONE || ret ==IOU_POLL_REISSUE ) -io_req_task_submit (req, locked); +io_req_task_submit (req, ts);else io_req_defer_failed (req, ret); }
@@ -976,7 +976,7 @@ int io_poll_remove(struct io_kiocb *req, unsigned int issue_flags)struct io_hash_bucket *bucket;struct io_kiocb *preq;int ret2, ret =0 ; -bool locked; +struct io_tw_state ts = {}; preq =io_poll_find (ctx, true, &cd, &ctx->cancel_table, &bucket); ret2 =io_poll_disarm (preq);
@@ -1026,8 +1026,8 @@ int io_poll_remove(struct io_kiocb *req, unsigned int issue_flags)req_set_fail (preq);io_req_set_res (preq, -ECANCELED ,0 ); - locked = !(issue_flags &IO_URING_F_UNLOCKED ); -io_req_task_complete (preq, &locked); + ts.locked = !(issue_flags &IO_URING_F_UNLOCKED ); +io_req_task_complete (preq, &ts); out:if (ret <0 ) {req_set_fail (req);
io_uring/rw.c +3 -3
@@ -283,16 +283,16 @@ static inline int io_fixup_rw_res(struct io_kiocb *req, long res)return res; } -static void io_req_rw_complete (struct io_kiocb *req,bool *locked) +static void io_req_rw_complete (struct io_kiocb *req,struct io_tw_state *ts) {io_req_io_end (req);if (req->flags & (REQ_F_BUFFER_SELECTED |REQ_F_BUFFER_RING )) { -unsigned issue_flags = *locked ?0 :IO_URING_F_UNLOCKED ; +unsigned issue_flags = ts->locked ?0 :IO_URING_F_UNLOCKED ; req->cqe.flags |=io_put_kbuf (req, issue_flags); } -io_req_task_complete (req, locked); +io_req_task_complete (req, ts); }static void io_complete_rw (struct kiocb *kiocb,long res)
io_uring/timeout.c +7 -7
@@ -101,9 +101,9 @@ __cold void io_flush_timeouts(struct io_ring_ctx *ctx)spin_unlock_irq (&ctx->timeout_lock); } -static void io_req_tw_fail_links (struct io_kiocb *link,bool *locked) +static void io_req_tw_fail_links (struct io_kiocb *link,struct io_tw_state *ts) { -io_tw_lock (link->ctx, locked); +io_tw_lock (link->ctx, ts);while (link) {struct io_kiocb *nxt = link->link;long res = -ECANCELED ;
@@ -112,7 +112,7 @@ static void io_req_tw_fail_links(struct io_kiocb *link, bool *locked) res = link->cqe.res; link->link =NULL ;io_req_set_res (link, res,0 ); -io_req_task_complete (link, locked); +io_req_task_complete (link, ts); link = nxt; } }
@@ -265,9 +265,9 @@ int io_timeout_cancel(struct io_ring_ctx *ctx, struct io_cancel_data *cd)return 0 ; } -static void io_req_task_link_timeout (struct io_kiocb *req,bool *locked) +static void io_req_task_link_timeout (struct io_kiocb *req,struct io_tw_state *ts) { -unsigned issue_flags = *locked ?0 :IO_URING_F_UNLOCKED ; +unsigned issue_flags = ts->locked ?0 :IO_URING_F_UNLOCKED ; struct io_timeout *timeout = io_kiocb_to_cmd(req, struct io_timeout);struct io_kiocb *prev = timeout->prev;int ret = -ENOENT ;
@@ -282,11 +282,11 @@ static void io_req_task_link_timeout(struct io_kiocb *req, bool *locked) ret =io_try_cancel (req->task->io_uring, &cd, issue_flags); }io_req_set_res (req, ret ?: -ETIME ,0 ); -io_req_task_complete (req, locked); +io_req_task_complete (req, ts);io_put_req (prev); }else {io_req_set_res (req, -ETIME ,0 ); -io_req_task_complete (req, locked); +io_req_task_complete (req, ts); } }
io_uring/uring_cmd.c +1 -1
@@ -12,7 +12,7 @@#include "rsrc.h" #include "uring_cmd.h" -static void io_uring_cmd_work (struct io_kiocb *req,bool *locked) +static void io_uring_cmd_work (struct io_kiocb *req,struct io_tw_state *ts) { struct io_uring_cmd *ioucmd = io_kiocb_to_cmd(req, struct io_uring_cmd);
Signature · 1 line
2.39.1
Quoted 9 lines · Pavel Begunkov
Add a
task_workstate instead of passing a raw bool pointer. This will better encapsulate it, e.g. to not expose too much to cmd requests, and may also be needed for further extensions.Pavel Begunkov (2):
io_uring: remove extra tw trylocks
io_uring: encapsulatetask_workstate[...]
Applied, thanks!
[1/2] io_uring: remove extra tw trylocks
commit: f14d58928002ea2fac729531bdf5bc7b1b2fffd1
[2/2] io_uring: encapsulate task_work state
commit: f14d58928002ea2fac729531bdf5bc7b1b2fffd1Best regards,
Signature · 1 line
Jens Axboe