complete cleanups
cover complete cleanups
Random cleanups for iopoll and generic completions, just for shedding some lines and overhead.
Pavel Begunkov (4):
io_uring: clean up io_req_free_batch_finish()
io_uring: simplify iopoll reissuing
io_uring: move res check out of io_rw_reissue()
io_uring: inline io_complete_rw_common()
fs/io_uring.c | 67 +++++++++++++++------------------------------------ 1 file changed, 20 insertions(+), 47 deletions(-)
Signature · 1 line
2.24.0
PATCH 1/4 io_uring: clean up io_req_free_batch_finish()
io_req_free_batch_finish() is final and does not permit struct req_batch to be reused without re-init. To be more consistent don't clear ->task there.
--- fs/io_uring.c | 4 +--- 1 file changed, 1 insertion(+), 3 deletions(-)
fs/io_uring.c +1 -3
@@ -2394,10 +2394,8 @@ static inline void io_init_req_batch(struct req_batch *rb)static void io_req_free_batch_finish (struct io_ring_ctx *ctx,struct req_batch *rb) { -if (rb->task) { +if (rb->task)io_put_task (rb->task, rb->task_refs); - rb->task =NULL ; - }if (rb->ctx_refs)percpu_ref_put_many (&ctx->refs, rb->ctx_refs); }
Signature · 1 line
2.24.0
PATCH 2/4 io_uring: simplify iopoll reissuing
In reply to Pavel BegunkovDon't stash -EAGAIN'ed iopoll requests into a list to reissue it later, do it eagerly. It removes overhead on keeping and checking that list, and allows in case of failure for these requests to be completed through normal iopoll completion path.
--- fs/io_uring.c | 26 +++++--------------------- 1 file changed, 5 insertions(+), 21 deletions(-)
fs/io_uring.c +5 -21
@@ -1026,8 +1026,7 @@ static struct fixed_rsrc_ref_node *alloc_fixed_rsrc_ref_node(static void init_fixed_file_ref_node (struct io_ring_ctx *ctx,struct fixed_rsrc_ref_node *ref_node); -static void __io_complete_rw (struct io_kiocb *req,long res,long res2, -unsigned int issue_flags); +static bool io_rw_reissue (struct io_kiocb *req,long res);static void io_cqring_fill_event (struct io_kiocb *req,long res);static void io_put_req (struct io_kiocb *req);static void io_put_req_deferred (struct io_kiocb *req,int nr);
@@ -2555,17 +2554,6 @@ static inline bool io_run_task_work(void)return false; } -static void io_iopoll_queue (struct list_head *again) -{ -struct io_kiocb *req; - -do { - req = list_first_entry(again, struct io_kiocb, inflight_entry); -list_del (&req->inflight_entry); -__io_complete_rw (req, -EAGAIN ,0 ,0 ); - }while (!list_empty (again)); -} -/* * Find and free completed poll iocbs */
@@ -2574,7 +2562,6 @@ static void io_iopoll_complete(struct io_ring_ctx *ctx, unsigned int *nr_events, {struct req_batch rb;struct io_kiocb *req; -LIST_HEAD (again);/* order with ->result store in io_complete_rw_iopoll() */ smp_rmb ();
@@ -2584,13 +2571,13 @@ static void io_iopoll_complete(struct io_ring_ctx *ctx, unsigned int *nr_events,int cflags =0 ; req = list_first_entry(done, struct io_kiocb, inflight_entry); +list_del (&req->inflight_entry); +if (READ_ONCE (req->result) == -EAGAIN ) { - req->result =0 ; req->iopoll_completed =0 ; -list_move_tail (&req->inflight_entry, &again); -continue ; +if (io_rw_reissue (req, -EAGAIN )) +continue ; } -list_del (&req->inflight_entry);if (req->flags &REQ_F_BUFFER_SELECTED ) cflags =io_put_rw_kbuf (req);
@@ -2605,9 +2592,6 @@ static void io_iopoll_complete(struct io_ring_ctx *ctx, unsigned int *nr_events,io_commit_cqring (ctx);io_cqring_ev_posted_iopoll (ctx);io_req_free_batch_finish (ctx, &rb); - -if (!list_empty (&again)) -io_iopoll_queue (&again); } static int io_do_iopoll(struct io_ring_ctx *ctx, unsigned int *nr_events,
Signature · 1 line
2.24.0
PATCH 4/4 io_uring: inline io_complete_rw_common()
In reply to Pavel Begunkov__io_complete_rw() casts request to kiocb for it to be immediately contaier_of()'ed by io_complete_rw_common(). And the last function's name doesn't do a great job of illuminating about its purposes, so just inline it in its only user.
--- fs/io_uring.c | 26 +++++++++----------------- 1 file changed, 9 insertions(+), 17 deletions(-)
fs/io_uring.c +9 -17
@@ -2758,22 +2758,6 @@ static void kiocb_end_write(struct io_kiocb *req)file_end_write (req->file); } -static void io_complete_rw_common (struct kiocb *kiocb,long res, -unsigned int issue_flags) -{ - struct io_kiocb *req = container_of(kiocb, struct io_kiocb, rw.kiocb); -int cflags =0 ; - -if (kiocb->ki_flags &IOCB_WRITE ) -kiocb_end_write (req); - -if (res != req->result) -req_set_fail_links (req); -if (req->flags &REQ_F_BUFFER_SELECTED ) - cflags =io_put_rw_kbuf (req); -__io_req_complete (req, issue_flags, res, cflags); -} -#ifdef CONFIG_BLOCK static bool io_resubmit_prep (struct io_kiocb *req) {
@@ -2837,10 +2821,18 @@ static bool io_rw_reissue(struct io_kiocb *req)static void __io_complete_rw (struct io_kiocb *req,long res,long res2,unsigned int issue_flags) { +int cflags =0 ; +if ((res == -EAGAIN || res == -EOPNOTSUPP ) &&io_rw_reissue (req))return ; +if (res != req->result) +req_set_fail_links (req); -io_complete_rw_common (&req->rw.kiocb, res, issue_flags); +if (req->rw.kiocb.ki_flags &IOCB_WRITE ) +kiocb_end_write (req); +if (req->flags &REQ_F_BUFFER_SELECTED ) + cflags =io_put_rw_kbuf (req); +__io_req_complete (req, issue_flags, res, cflags); }static void io_complete_rw (struct kiocb *kiocb,long res,long res2)
Signature · 1 line
2.24.0
PATCH 3/4 io_uring: move res check out of io_rw_reissue()
In reply to Pavel BegunkovWe pass return code into io_rw_reissue() for it to check for -EAGAIN. It's not the cleaniest approach and may prevent inlining of the non-EAGAIN fast path, so do it at call sites.
--- fs/io_uring.c | 17 ++++++++--------- 1 file changed, 8 insertions(+), 9 deletions(-)
fs/io_uring.c +8 -9
@@ -1026,7 +1026,7 @@ static struct fixed_rsrc_ref_node *alloc_fixed_rsrc_ref_node(static void init_fixed_file_ref_node (struct io_ring_ctx *ctx,struct fixed_rsrc_ref_node *ref_node); -static bool io_rw_reissue (struct io_kiocb *req,long res); +static bool io_rw_reissue (struct io_kiocb *req);static void io_cqring_fill_event (struct io_kiocb *req,long res);static void io_put_req (struct io_kiocb *req);static void io_put_req_deferred (struct io_kiocb *req,int nr);
@@ -2575,7 +2575,7 @@ static void io_iopoll_complete(struct io_ring_ctx *ctx, unsigned int *nr_events,if (READ_ONCE (req->result) == -EAGAIN ) { req->iopoll_completed =0 ; -if (io_rw_reissue (req, -EAGAIN )) +if (io_rw_reissue (req))continue ; }
@@ -2809,15 +2809,12 @@ static bool io_resubmit_prep(struct io_kiocb *req) }#endif -static bool io_rw_reissue (struct io_kiocb *req,long res) +static bool io_rw_reissue (struct io_kiocb *req) {#ifdef CONFIG_BLOCK -umode_t mode; +umode_t mode =file_inode (req->file)->i_mode;int ret; -if (res != -EAGAIN && res != -EOPNOTSUPP ) -return false; - mode =file_inode (req->file)->i_mode;if (!S_ISBLK (mode) && !S_ISREG (mode))return false;if ((req->flags &REQ_F_NOWAIT ) ||io_wq_current_is_worker ())
@@ -2840,8 +2837,10 @@ static bool io_rw_reissue(struct io_kiocb *req, long res)static void __io_complete_rw (struct io_kiocb *req,long res,long res2,unsigned int issue_flags) { -if (!io_rw_reissue (req, res)) -io_complete_rw_common (&req->rw.kiocb, res, issue_flags); +if ((res == -EAGAIN || res == -EOPNOTSUPP ) &&io_rw_reissue (req)) +return ; + +io_complete_rw_common (&req->rw.kiocb, res, issue_flags); }static void io_complete_rw (struct kiocb *kiocb,long res,long res2)
Signature · 1 line
2.24.0
Jens Axboe
maintainerQuoted 11 lines · Pavel Begunkov
Random cleanups for iopoll and generic completions, just for shedding some lines and overhead.
Pavel Begunkov (4):
io_uring: clean upio_req_free_batch_finish()
io_uring: simplify iopoll reissuing
io_uring: move res check out ofio_rw_reissue()
io_uring: inlineio_complete_rw_common()fs/io_uring.c | 67 +++++++++++++++------------------------------------ 1 file changed, 20 insertions(+), 47 deletions(-)
Good cleanups, thanks.
Signature · 1 line
Jens Axboe