skip request refcounting
v2 cover skip request refcounting
With some tricks, we can avoid refcounting in most of the cases and so save on atomics. 1-2 are simple preparations and 3-4 are the meat. 5/5 is a hint to the compiler, which stopped to similarly optimise it as is.
Jens tried out a prototype before, apparently it gave ~3% win for the default read test. Not much has changed since then, so I'd expect same result, and also hope that it should be of even greater benefit to multithreaded workloads.
The previous version had a flaw, so it was decided to move all completions out of IRQ and base on that assumption. On top of io_uring-irq branch.
v2: Rebase to IRQ branch and updated descriptions. Removed prep
patches. The main part is split in 2: dealing with submission refs, and completion. Added 5/5.
Pavel Begunkov (5):
io_uring: move req_ref_get() and friends
io_uring: remove req_ref_sub_and_test()
io_uring: remove submission references
io_uring: skip request refcounting
io_uring: optimise hot path of ltimeout prep
fs/io_uring.c | 173 +++++++++++++++++++++++++++----------------------- 1 file changed, 94 insertions(+), 79 deletions(-)
Signature · 1 line
2.32.0
PATCH v2 1/5 io_uring: move req_ref_get() and friends
Move all request refcount helpers to avoid forward declarations in the future.
--- fs/io_uring.c | 70 +++++++++++++++++++++++++-------------------------- 1 file changed, 35 insertions(+), 35 deletions(-)
fs/io_uring.c +35 -35
@@ -1078,6 +1078,41 @@ EXPORT_SYMBOL(io_uring_get_socket);#define io_for_each_link (pos, head) \ for (pos = (head); pos; pos = pos->link) +/* +* Shamelessly stolen from the mm implementation of page reference checking, +* see commit f958d7b528b1 for details. +*/ +#define req_ref_zero_or_close_to_overflow (req) \ + ((unsigned int) atomic_read(&(req->refs)) + 127u <= 127u) + +static inline bool req_ref_inc_not_zero (struct io_kiocb *req) +{ +return atomic_inc_not_zero (&req->refs); +} + +static inline bool req_ref_sub_and_test (struct io_kiocb *req,int refs) +{ +WARN_ON_ONCE (req_ref_zero_or_close_to_overflow (req)); +return atomic_sub_and_test (refs, &req->refs); +} + +static inline bool req_ref_put_and_test (struct io_kiocb *req) +{ +WARN_ON_ONCE (req_ref_zero_or_close_to_overflow (req)); +return atomic_dec_and_test (&req->refs); +} + +static inline void req_ref_put (struct io_kiocb *req) +{ +WARN_ON_ONCE (req_ref_put_and_test (req)); +} + +static inline void req_ref_get (struct io_kiocb *req) +{ +WARN_ON_ONCE (req_ref_zero_or_close_to_overflow (req)); +atomic_inc (&req->refs); +} +static inline void io_req_set_rsrc_node (struct io_kiocb *req) {struct io_ring_ctx *ctx = req->ctx;
@@ -1533,41 +1568,6 @@ static bool io_cqring_overflow_flush(struct io_ring_ctx *ctx)return ret; } -/* -* Shamelessly stolen from the mm implementation of page reference checking, -* see commit f958d7b528b1 for details. -*/ -#define req_ref_zero_or_close_to_overflow (req) \ - ((unsigned int) atomic_read(&(req->refs)) + 127u <= 127u) - -static inline bool req_ref_inc_not_zero (struct io_kiocb *req) -{ -return atomic_inc_not_zero (&req->refs); -} - -static inline bool req_ref_sub_and_test (struct io_kiocb *req,int refs) -{ -WARN_ON_ONCE (req_ref_zero_or_close_to_overflow (req)); -return atomic_sub_and_test (refs, &req->refs); -} - -static inline bool req_ref_put_and_test (struct io_kiocb *req) -{ -WARN_ON_ONCE (req_ref_zero_or_close_to_overflow (req)); -return atomic_dec_and_test (&req->refs); -} - -static inline void req_ref_put (struct io_kiocb *req) -{ -WARN_ON_ONCE (req_ref_put_and_test (req)); -} - -static inline void req_ref_get (struct io_kiocb *req) -{ -WARN_ON_ONCE (req_ref_zero_or_close_to_overflow (req)); -atomic_inc (&req->refs); -} -/* must to be called somewhat shortly after putting a request */ static inline void io_put_task (struct task_struct *task,int nr) {
Signature · 1 line
2.32.0
PATCH v2 2/5 io_uring: remove req_ref_sub_and_test()
In reply to Pavel BegunkovSoon, we won't need to put several references at once, remove req_ref_sub_and_test() and @nr argument from io_put_req_deferred(), and put the rest of the references by hand.
--- fs/io_uring.c | 31 ++++++++++++++----------------- 1 file changed, 14 insertions(+), 17 deletions(-)
fs/io_uring.c +14 -17
@@ -1041,7 +1041,7 @@ static void io_uring_cancel_generic(bool cancel_all, struct io_sq_data *sqd);static bool io_cqring_fill_event (struct io_ring_ctx *ctx,u64 user_data,long res,unsigned int cflags);static void io_put_req (struct io_kiocb *req); -static void io_put_req_deferred (struct io_kiocb *req,int nr); +static void io_put_req_deferred (struct io_kiocb *req);static void io_dismantle_req (struct io_kiocb *req);static struct io_kiocb *io_prep_linked_timeout (struct io_kiocb *req);static void io_queue_linked_timeout (struct io_kiocb *req);
@@ -1090,12 +1090,6 @@ static inline bool req_ref_inc_not_zero(struct io_kiocb *req)return atomic_inc_not_zero (&req->refs); } -static inline bool req_ref_sub_and_test (struct io_kiocb *req,int refs) -{ -WARN_ON_ONCE (req_ref_zero_or_close_to_overflow (req)); -return atomic_sub_and_test (refs, &req->refs); -} -static inline bool req_ref_put_and_test (struct io_kiocb *req) {WARN_ON_ONCE (req_ref_zero_or_close_to_overflow (req));
@@ -1374,7 +1368,7 @@ static void io_kill_timeout(struct io_kiocb *req, int status) atomic_read(&req->ctx->cq_timeouts) + 1);list_del_init (&req->timeout.list);io_cqring_fill_event (req->ctx, req->user_data, status,0 ); -io_put_req_deferred (req,1 ); +io_put_req_deferred (req); } }
@@ -1856,7 +1850,7 @@ static bool io_kill_linked_timeout(struct io_kiocb *req)if (hrtimer_try_to_cancel (&io->timer) !=-1 ) {io_cqring_fill_event (link->ctx, link->user_data, -ECANCELED ,0 ); -io_put_req_deferred (link,1 ); +io_put_req_deferred (link);return true; } }
@@ -1875,7 +1869,9 @@ static void io_fail_links(struct io_kiocb *req)trace_io_uring_fail_link (req, link);io_cqring_fill_event (link->ctx, link->user_data, -ECANCELED ,0 ); -io_put_req_deferred (link,2 ); + +io_put_req (link); +io_put_req_deferred (link); link = nxt; } }
@@ -2156,7 +2152,8 @@ static void io_submit_flush_completions(struct io_ring_ctx *ctx)struct io_kiocb *req = state->compl_reqs[i];/* submission and completion refs */ -if (req_ref_sub_and_test (req,2 )) +io_put_req (req); +if (req_ref_put_and_test (req))io_req_free_batch (&rb, req, &ctx->submit_state); }
@@ -2185,9 +2182,9 @@ static inline void io_put_req(struct io_kiocb *req)io_free_req (req); } -static inline void io_put_req_deferred (struct io_kiocb *req,int refs) +static inline void io_put_req_deferred (struct io_kiocb *req) { -if (req_ref_sub_and_test (req, refs)) { +if (req_ref_put_and_test (req)) { req->io_task_work.func = io_free_req;io_req_task_work_add (req); }
@@ -5229,7 +5226,6 @@ static bool __io_poll_remove_one(struct io_kiocb *req,static bool io_poll_remove_one (struct io_kiocb *req) __must_hold(&req->ctx->completion_lock) { -int refs;bool do_complete;io_poll_remove_double (req);
@@ -5241,8 +5237,9 @@ static bool io_poll_remove_one(struct io_kiocb *req)req_set_fail (req);/* non-poll requests have submit ref still */ - refs =1 + (req->opcode !=IORING_OP_POLL_ADD ); -io_put_req_deferred (req, refs); +if (req->opcode !=IORING_OP_POLL_ADD ) +io_put_req (req); +io_put_req_deferred (req); }return do_complete; }
@@ -5543,7 +5540,7 @@ static int io_timeout_cancel(struct io_ring_ctx *ctx, __u64 user_data)req_set_fail (req);io_cqring_fill_event (ctx, req->user_data, -ECANCELED ,0 ); -io_put_req_deferred (req,1 ); +io_put_req_deferred (req);return 0 ; }
Signature · 1 line
2.32.0
PATCH v2 3/5 io_uring: remove submission references
In reply to Pavel BegunkovRequests are by default given with two references, submission and completion. Completion references are straightforward, they represent request ownership and are put when a request is completed or so. Submission references are a bit more trickier. They're needed when io_issue_sqe() followed deep into the submission stack (e.g. in fs, block, drivers, etc.), request may have given away for concurrent execution or already completed, and the code unwinding back to io_issue_sqe() may be accessing some pieces of our requests, e.g. file or iov.
Now, we prevent such async/in-depth completions by pushing requests through task_work. Punting to io-wq is also done through task_works, apart from a couple of cases with a pretty well known context. So, there're two cases:
1) io_issue_sqe() from the task context and protected by ->uring_lock. Either requests return back to io_uring or handed to task_work, which won't be executed because we're currently controlling that task. So, we can be sure that requests are staying alive all the time and we don't need submission references to pin them.
2) io_issue_sqe() from io-wq, which doesn't hold the mutex. The role of submission reference is played by io-wq reference, which is put by io_wq_submit_work(). Hence, it should be fine.
Considering that, we can carefully kill the submission reference.
--- fs/io_uring.c | 37 ++++++++++++++----------------------- 1 file changed, 14 insertions(+), 23 deletions(-)
fs/io_uring.c +14 -23
@@ -1699,7 +1699,6 @@ static inline void io_req_complete(struct io_kiocb *req, long res)static void io_req_complete_failed (struct io_kiocb *req,long res) {req_set_fail (req); -io_put_req (req);io_req_complete_post (req, res,0 ); }
@@ -1754,7 +1753,14 @@ static bool io_flush_cached_reqs(struct io_ring_ctx *ctx)return nr !=0 ; } +/* +* A request might get retired back into the request caches even before opcode +* handlers and io_issue_sqe() are done with it, e.g. inline completion path. +* Because of that, io_alloc_req() should be called only under ->uring_lock +* and with extra caution to not get a request that is still worked on. +*/ static struct io_kiocb *io_alloc_req (struct io_ring_ctx *ctx) + __must_hold(&req->ctx->uring_lock) {struct io_submit_state *state = &ctx->submit_state;gfp_t gfp =GFP_KERNEL | __GFP_NOWARN;
@@ -1869,8 +1875,6 @@ static void io_fail_links(struct io_kiocb *req)trace_io_uring_fail_link (req, link);io_cqring_fill_event (link->ctx, link->user_data, -ECANCELED ,0 ); - -io_put_req (link);io_put_req_deferred (link); link = nxt; }
@@ -2151,8 +2155,6 @@ static void io_submit_flush_completions(struct io_ring_ctx *ctx)for (i =0 ; i < nr; i++) {struct io_kiocb *req = state->compl_reqs[i]; -/* submission and completion refs */ -io_put_req (req);if (req_ref_put_and_test (req))io_req_free_batch (&rb, req, &ctx->submit_state); }
@@ -2257,7 +2259,6 @@ static void io_iopoll_complete(struct io_ring_ctx *ctx, unsigned int *nr_events,if (READ_ONCE (req->result) == -EAGAIN && resubmit && !(req->flags &REQ_F_DONT_REISSUE )) { req->iopoll_completed =0 ; -req_ref_get (req);io_req_task_queue_reissue (req);continue ; }
@@ -2755,7 +2756,6 @@ static void kiocb_done(struct kiocb *kiocb, ssize_t ret,if (check_reissue && (req->flags &REQ_F_REISSUE )) { req->flags &= ~REQ_F_REISSUE ;if (io_resubmit_prep (req)) { -req_ref_get (req);io_req_task_queue_reissue (req); }else {int cflags =0 ;
@@ -3181,9 +3181,6 @@ static int io_async_buf_func(struct wait_queue_entry *wait, unsigned mode, req->rw.kiocb.ki_flags &= ~IOCB_WAITQ ;list_del_init (&wait->entry); - -/* submit ref gets dropped, acquire a new one */ -req_ref_get (req);io_req_task_queue (req);return 1 ; }
@@ -5235,10 +5232,6 @@ static bool io_poll_remove_one(struct io_kiocb *req)io_cqring_fill_event (req->ctx, req->user_data, -ECANCELED ,0 );io_commit_cqring (req->ctx);req_set_fail (req); - -/* non-poll requests have submit ref still */ -if (req->opcode !=IORING_OP_POLL_ADD ) -io_put_req (req);io_put_req_deferred (req); }return do_complete;
@@ -6273,6 +6266,9 @@ static void io_wq_submit_work(struct io_wq_work *work)struct io_kiocb *timeout;int ret =0 ; +/* will be dropped by ->io_free_work() after returning to io-wq */ +req_ref_get (req); + timeout =io_prep_linked_timeout (req);if (timeout)io_queue_linked_timeout (timeout);
@@ -6295,11 +6291,8 @@ static void io_wq_submit_work(struct io_wq_work *work) }/* avoid locking problems by failing it from a clean context */ -if (ret) { -/* io-wq is going to take one down */ -req_ref_get (req); +if (ret)io_req_task_queue_fail (req, ret); - } } static inline struct io_fixed_file *io_fixed_file_slot(struct io_file_table *table,
@@ -6441,6 +6434,8 @@ static struct io_kiocb *io_prep_linked_timeout(struct io_kiocb *req) nxt->opcode != IORING_OP_LINK_TIMEOUT)return NULL ; +/* linked timeouts should have two refs once prep'ed */ +req_ref_get (nxt); nxt->timeout.head = req; nxt->flags |=REQ_F_LTIMEOUT_ACTIVE ; req->flags |=REQ_F_LINK_TIMEOUT ;
@@ -6461,7 +6456,6 @@ static void __io_queue_sqe(struct io_kiocb *req)* doesn't support non-blocking read/write attempts */ if (likely (!ret)) { -/* drop submission reference */ if (req->flags &REQ_F_COMPLETE_INLINE ) {struct io_ring_ctx *ctx = req->ctx;struct io_submit_state *state = &ctx->submit_state;
@@ -6469,8 +6463,6 @@ static void __io_queue_sqe(struct io_kiocb *req) state->compl_reqs[state->compl_nr++] = req;if (state->compl_nr ==ARRAY_SIZE (state->compl_reqs))io_submit_flush_completions (ctx); - }else { -io_put_req (req); } }else if (ret == -EAGAIN && !(req->flags &REQ_F_NOWAIT )) {switch (io_arm_poll_handler (req)) {
@@ -6550,8 +6542,7 @@ static int io_init_req(struct io_ring_ctx *ctx, struct io_kiocb *req, req->user_data =READ_ONCE (sqe->user_data); req->file =NULL ; req->fixed_rsrc_refs =NULL ; -/* one is dropped after submission, the other at completion */ -atomic_set (&req->refs,2 ); +atomic_set (&req->refs,1 ); req->task = current;/* enforce forwards compatibility on users */
Signature · 1 line
2.32.0
PATCH v2 4/5 io_uring: skip request refcounting
In reply to Pavel BegunkovAs submission references are gone, there is only one initial reference left. Instead of actually doing atomic refcounting, add a flag indicating whether we're going to take more refs or doing any other sync magic. The flag should be set before the request may get used in parallel.
Together with the previous patch it saves 2 refcount atomics per request for IOPOLL and IRQ completions, and 1 atomic per req for inline completions, with some exceptions. In particular, currently, there are three cases, when the refcounting have to be enabled:
- Polling, including apoll. Because double poll entries takes a ref. Might get relaxed in the near future.
- Link timeouts, enabled for both, the timeout and the request it's bound to, because they work in-parallel and we need to synchronise to cancel one of them on completion.
- When a request gets in io-wq, because it doesn't hold uring_lock and we need guarantees of submission references.
--- fs/io_uring.c | 24 +++++++++++++++++++++++- 1 file changed, 23 insertions(+), 1 deletion(-)
fs/io_uring.c +23 -1
@@ -710,6 +710,7 @@ enum {REQ_F_REISSUE_BIT ,REQ_F_DONT_REISSUE_BIT ,REQ_F_CREDS_BIT , +REQ_F_REFCOUNT_BIT ,/* keep async read/write and isreg together and in order */ REQ_F_NOWAIT_READ_BIT , REQ_F_NOWAIT_WRITE_BIT,
@@ -765,6 +766,8 @@ enum {REQ_F_ISREG =BIT (REQ_F_ISREG_BIT ),/* has creds assigned */ REQ_F_CREDS =BIT (REQ_F_CREDS_BIT ), +/* skip refcounting if not set */ + REQ_F_REFCOUNT = BIT(REQ_F_REFCOUNT_BIT), }; struct async_poll {
@@ -1087,26 +1090,40 @@ EXPORT_SYMBOL(io_uring_get_socket);static inline bool req_ref_inc_not_zero (struct io_kiocb *req) { +WARN_ON_ONCE (!(req->flags &REQ_F_REFCOUNT ));return atomic_inc_not_zero (&req->refs); }static inline bool req_ref_put_and_test (struct io_kiocb *req) { +if (likely (!(req->flags &REQ_F_REFCOUNT ))) +return true; +WARN_ON_ONCE (req_ref_zero_or_close_to_overflow (req));return atomic_dec_and_test (&req->refs); }static inline void req_ref_put (struct io_kiocb *req) { +WARN_ON_ONCE (!(req->flags &REQ_F_REFCOUNT ));WARN_ON_ONCE (req_ref_put_and_test (req)); }static inline void req_ref_get (struct io_kiocb *req) { +WARN_ON_ONCE (!(req->flags &REQ_F_REFCOUNT ));WARN_ON_ONCE (req_ref_zero_or_close_to_overflow (req));atomic_inc (&req->refs); } +static inline void io_req_refcount (struct io_kiocb *req) +{ +if (!(req->flags &REQ_F_REFCOUNT )) { + req->flags |=REQ_F_REFCOUNT ; +atomic_set (&req->refs,1 ); + } +} +static inline void io_req_set_rsrc_node (struct io_kiocb *req) {struct io_ring_ctx *ctx = req->ctx;
@@ -5185,6 +5202,7 @@ static int io_arm_poll_handler(struct io_kiocb *req) req->apoll = apoll; req->flags |=REQ_F_POLLED ; ipt.pt._qproc = io_async_queue_proc; +io_req_refcount (req); ret =__io_arm_poll_handler (req, &apoll->poll, &ipt, mask, io_async_wake);
@@ -5375,6 +5393,7 @@ static int io_poll_add_prep(struct io_kiocb *req, const struct io_uring_sqe *sqeif (flags & ~IORING_POLL_ADD_MULTI )return -EINVAL ; +io_req_refcount (req); poll->events =io_poll_parse_events (sqe, flags);return 0 ; }
@@ -6266,6 +6285,7 @@ static void io_wq_submit_work(struct io_wq_work *work)struct io_kiocb *timeout;int ret =0 ; +io_req_refcount (req);/* will be dropped by ->io_free_work() after returning to io-wq */ req_ref_get (req);
@@ -6435,7 +6455,10 @@ static struct io_kiocb *io_prep_linked_timeout(struct io_kiocb *req)return NULL ;/* linked timeouts should have two refs once prep'ed */ +io_req_refcount (req); +io_req_refcount (nxt);req_ref_get (nxt); + nxt->timeout.head = req; nxt->flags |=REQ_F_LTIMEOUT_ACTIVE ; req->flags |=REQ_F_LINK_TIMEOUT ;
@@ -6542,7 +6565,6 @@ static int io_init_req(struct io_ring_ctx *ctx, struct io_kiocb *req, req->user_data =READ_ONCE (sqe->user_data); req->file =NULL ; req->fixed_rsrc_refs =NULL ; -atomic_set (&req->refs,1 ); req->task = current;/* enforce forwards compatibility on users */
Signature · 1 line
2.32.0
PATCH v2 5/5 io_uring: optimise hot path of ltimeout prep
In reply to Pavel Begunkovio_prep_linked_timeout() grew too heavy and compiler now refuse to inline the function. Help it by splitting in two and annotating with inline.
--- fs/io_uring.c | 45 +++++++++++++++++++++++++-------------------- 1 file changed, 25 insertions(+), 20 deletions(-)
fs/io_uring.c +25 -20
@@ -1046,7 +1046,6 @@ static bool io_cqring_fill_event(struct io_ring_ctx *ctx, u64 user_data,static void io_put_req (struct io_kiocb *req);static void io_put_req_deferred (struct io_kiocb *req);static void io_dismantle_req (struct io_kiocb *req); -static struct io_kiocb *io_prep_linked_timeout (struct io_kiocb *req);static void io_queue_linked_timeout (struct io_kiocb *req); static int __io_register_rsrc_update(struct io_ring_ctx *ctx, unsigned type, struct io_uring_rsrc_update2 *up,
@@ -1297,6 +1296,31 @@ static void io_req_track_inflight(struct io_kiocb *req) } } +static struct io_kiocb *__io_prep_linked_timeout (struct io_kiocb *req) +{ +struct io_kiocb *nxt = req->link; + +if (req->flags &REQ_F_LINK_TIMEOUT ) +return NULL ; + +/* linked timeouts should have two refs once prep'ed */ +io_req_refcount (req); +io_req_refcount (nxt); +req_ref_get (nxt); + + nxt->timeout.head = req; + nxt->flags |=REQ_F_LTIMEOUT_ACTIVE ; + req->flags |=REQ_F_LINK_TIMEOUT ; +return nxt; +} + +static inline struct io_kiocb *io_prep_linked_timeout (struct io_kiocb *req) +{ +if (likely (!req->link || req->link->opcode !=IORING_OP_LINK_TIMEOUT )) +return NULL ; +return __io_prep_linked_timeout (req); +} +static void io_prep_async_work (struct io_kiocb *req) {const struct io_op_def *def = &io_op_defs[req->opcode];
@@ -6446,25 +6470,6 @@ static void io_queue_linked_timeout(struct io_kiocb *req)io_put_req (req); } -static struct io_kiocb *io_prep_linked_timeout (struct io_kiocb *req) -{ -struct io_kiocb *nxt = req->link; - -if (!nxt || (req->flags &REQ_F_LINK_TIMEOUT ) || - nxt->opcode !=IORING_OP_LINK_TIMEOUT ) -return NULL ; - -/* linked timeouts should have two refs once prep'ed */ -io_req_refcount (req); -io_req_refcount (nxt); -req_ref_get (nxt); - - nxt->timeout.head = req; - nxt->flags |=REQ_F_LTIMEOUT_ACTIVE ; - req->flags |=REQ_F_LINK_TIMEOUT ; -return nxt; -} -static void __io_queue_sqe (struct io_kiocb *req) __must_hold(&req->ctx->uring_lock) {
Signature · 1 line
2.32.0
Jens Axboe
maintainerQuoted 5 lines · Pavel Begunkov
With some tricks, we can avoid refcounting in most of the cases and so save on atomics. 1-2 are simple preparations and 3-4 are the meat. 5/5 is a hint to the compiler, which stopped to similarly optimise it as is.
Quoted 8 lines
Jens tried out a prototype before, apparently it gave ~3% win for the default read test. Not much has changed since then, so I'd expect same result, and also hope that it should be of even greater benefit to multithreaded workloads.
The previous version had a flaw, so it was decided to move all completions out of IRQ and base on that assumption. On top of io_uring-irq branch.
This is really nice, both in terms of how the series is laid out, but also the reasoning behind it. I can't shoot any immediate holes in it, let's get it queued for 5.15.
Signature · 1 line
Jens Axboe