req alloc and task_work speedups
Jens Axboe
maintainercover req alloc and task_work speedups
Hi,
This small series ensures persistent req allocations across invocations of io_uring_enter(). It's sitting on top of the 4 patches from Pavel that move the submit/completion state cache into the ctx itself.
Signature · 1 line
Jens Axboe
Jens Axboe
maintainerPATCH 1/3 io_ring: use persistent request cache
Now that we have the submit_state in the ring itself, we can have io_kiocb allocations that are persistent across invocations. This reduces the time spent doing slab allocations and frees.
--- fs/io_uring.c | 58 +++++++++++++++++++++++++++++++-------------------- 1 file changed, 35 insertions(+), 23 deletions(-)
fs/io_uring.c +35 -23
@@ -263,8 +263,9 @@ struct io_sq_data {#define IO_COMPL_BATCH 32struct io_comp_state { -unsigned int nr;struct io_kiocb *reqs[IO_COMPL_BATCH ]; +unsigned int nr; +struct list_head free_list; };struct io_submit_state {
@@ -1290,7 +1291,6 @@ static inline bool io_is_timeout_noseq(struct io_kiocb *req)static struct io_ring_ctx *io_ring_ctx_alloc (struct io_uring_params *p) { -struct io_submit_state *submit_state;struct io_ring_ctx *ctx;int hash_bits;
@@ -1343,10 +1343,7 @@ static struct io_ring_ctx *io_ring_ctx_alloc(struct io_uring_params *p)INIT_DELAYED_WORK (&ctx->rsrc_put_work, io_rsrc_put_work);init_llist_head (&ctx->rsrc_put_llist); - submit_state = &ctx->submit_state; - submit_state->comp.nr =0 ; - submit_state->file_refs =0 ; - submit_state->free_reqs =0 ; +INIT_LIST_HEAD (&ctx->submit_state.comp.free_list);return ctx; err:if (ctx->fallback_req)
@@ -1969,6 +1966,14 @@ static struct io_kiocb *io_alloc_req(struct io_ring_ctx *ctx) {struct io_submit_state *state = &ctx->submit_state; +if (!list_empty (&state->comp.free_list)) { +struct io_kiocb *req; + + req = list_first_entry(&state->comp.free_list, struct io_kiocb, + compl.list); +list_del (&req->compl.list); +return req; + }if (!state->free_reqs) {gfp_t gfp =GFP_KERNEL | __GFP_NOWARN;size_t sz;
@@ -2255,33 +2260,25 @@ static void io_free_req(struct io_kiocb *req) }struct req_batch { -void *reqs[IO_IOPOLL_BATCH ]; -int to_free; -struct task_struct *task;int task_refs; +int ctx_refs; };static inline void io_init_req_batch (struct req_batch *rb) { - rb->to_free =0 ; + rb->ctx_refs =0 ; rb->task_refs =0 ; rb->task =NULL ; } -static void __io_req_free_batch_flush (struct io_ring_ctx *ctx, -struct req_batch *rb) -{ -kmem_cache_free_bulk (req_cachep, rb->to_free, rb->reqs); -percpu_ref_put_many (&ctx->refs, rb->to_free); - rb->to_free =0 ; -} -static void io_req_free_batch_finish (struct io_ring_ctx *ctx,struct req_batch *rb) { -if (rb->to_free) -__io_req_free_batch_flush (ctx, rb); +if (rb->ctx_refs) { +percpu_ref_put_many (&ctx->refs, rb->ctx_refs); + rb->ctx_refs =0 ; + }if (rb->task) {io_put_task (rb->task, rb->task_refs); rb->task =NULL ;
@@ -2290,6 +2287,8 @@ static void io_req_free_batch_finish(struct io_ring_ctx *ctx,static void io_req_free_batch (struct req_batch *rb,struct io_kiocb *req) { +struct io_comp_state *cs = &req->ctx->submit_state.comp; +if (unlikely (io_is_fallback_req (req))) {io_free_req (req);return ;
@@ -2305,9 +2304,8 @@ static void io_req_free_batch(struct req_batch *rb, struct io_kiocb *req) rb->task_refs++;io_dismantle_req (req); - rb->reqs[rb->to_free++] = req; -if (unlikely (rb->to_free ==ARRAY_SIZE (rb->reqs))) -__io_req_free_batch_flush (req->ctx, rb); + rb->ctx_refs++; +list_add (&req->compl.list, &cs->free_list); } static void io_submit_flush_completions(struct io_comp_state *cs,
@@ -8668,6 +8666,19 @@ static void io_destroy_buffers(struct io_ring_ctx *ctx)idr_destroy (&ctx->io_buffer_idr); } +static void io_req_cache_free (struct io_ring_ctx *ctx) +{ +struct io_comp_state *cs = &ctx->submit_state.comp; + +while (!list_empty (&cs->free_list)) { +struct io_kiocb *req; + + req = list_first_entry(&cs->free_list, struct io_kiocb, compl.list); +list_del (&req->compl.list); +kmem_cache_free (req_cachep, req); + } +} +static void io_ring_ctx_free (struct io_ring_ctx *ctx) {io_finish_async (ctx);
@@ -8705,6 +8716,7 @@ static void io_ring_ctx_free(struct io_ring_ctx *ctx)put_cred (ctx->creds);kfree (ctx->cancel_hash);kmem_cache_free (req_cachep, ctx->fallback_req); +io_req_cache_free (ctx);kfree (ctx); }
Signature · 1 line
2.30.0
Jens Axboe
maintainerPATCH 2/3 io_uring: provide FIFO ordering for task_work
In reply to Jens Axboetask_work is a LIFO list, due to how it's implemented as a lockless list. For long chains of task_work, this can be problematic as the first entry added is the last one processed. Similarly, we'd waste a lot of CPU cycles reversing this list.
Wrap the task_work so we have a single task_work entry per task per ctx, and use that to run it in the right order.
--- fs/io-wq.h | 9 ---- fs/io_uring.c | 101 ++++++++++++++++++++++++++++++++++++--- include/linux/io_uring.h | 14 ++++++ 3 files changed, 108 insertions(+), 16 deletions(-)
fs/io-wq.h +0 -9
@@ -27,15 +27,6 @@ enum io_wq_cancel { IO_WQ_CANCEL_NOTFOUND, /* work not found */ }; -struct io_wq_work_node { -struct io_wq_work_node *next; -}; - -struct io_wq_work_list { -struct io_wq_work_node *first; -struct io_wq_work_node *last; -}; -static inline void wq_list_add_after (struct io_wq_work_node *node,struct io_wq_work_node *pos,struct io_wq_work_list *list)
fs/io_uring.c +94 -7
161 lines not on this page · raw message
include/linux/io_uring.h +14 -0
@@ -22,6 +22,15 @@ struct io_identity {refcount_t count; }; +struct io_wq_work_node { +struct io_wq_work_node *next; +}; + +struct io_wq_work_list { +struct io_wq_work_node *first; +struct io_wq_work_node *last; +}; +struct io_uring_task {/* submission side */ struct xarray xa;
@@ -32,6 +41,11 @@ struct io_uring_task {struct io_identity *identity;atomic_t in_idle;bool sqpoll; + +spinlock_t task_lock; +struct io_wq_work_list task_list; +unsigned long task_state; +struct callback_head task_work; };#if defined(CONFIG_IO_URING )
Signature · 1 line
2.30.0
Jens Axboe
maintainerPATCH 3/3 io_uring: enable req cache for task_work items
In reply to Jens Axboetask_work is run without utilizing the req alloc cache, so any deferred items don't get to take advantage of either the alloc or free side of it. With task_work now being wrapped by io_uring, we can use the ctx completion state to both use the req cache and the completion flush batching.
With this, the only request type that cannot take advantage of the req cache is IRQ driven IO for regular files / block devices. Anything else, including IOPOLL polled IO to those same tyes, will take advantage of it.
--- fs/io_uring.c | 22 +++++++++++++++++++++- 1 file changed, 21 insertions(+), 1 deletion(-)
fs/io_uring.c +21 -1
@@ -1047,6 +1047,8 @@ static int io_setup_async_rw(struct io_kiocb *req, const struct iovec *iovec,const struct iovec *fast_iov, struct iov_iter *iter, bool force);static void io_req_task_queue (struct io_kiocb *req); +static void io_submit_flush_completions (struct io_comp_state *cs, +struct io_ring_ctx *ctx);static struct kmem_cache *req_cachep;
@@ -2160,6 +2162,7 @@ static inline struct io_kiocb *io_req_find_next(struct io_kiocb *req)static bool __tctx_task_work (struct io_uring_task *tctx) { +struct io_ring_ctx *ctx =NULL ;struct io_wq_work_list list;struct io_wq_work_node *node;
@@ -2174,11 +2177,28 @@ static bool __tctx_task_work(struct io_uring_task *tctx) node = list.first;while (node) {struct io_wq_work_node *next = node->next; +struct io_ring_ctx *this_ctx;struct io_kiocb *req; req = container_of(node, struct io_kiocb, io_task_work.node); + this_ctx = req->ctx; req->task_work.func (&req->task_work); node = next; + +if (!ctx) { + ctx = this_ctx; + }else if (ctx != this_ctx) { +mutex_lock (&ctx->uring_lock); +io_submit_flush_completions (&ctx->submit_state.comp, ctx); +mutex_unlock (&ctx->uring_lock); + ctx = this_ctx; + } + } + +if (ctx && ctx->submit_state.comp.nr) { +mutex_lock (&ctx->uring_lock); +io_submit_flush_completions (&ctx->submit_state.comp, ctx); +mutex_unlock (&ctx->uring_lock); }return list.first !=NULL ;
@@ -2301,7 +2321,7 @@ static void __io_req_task_submit(struct io_kiocb *req)if (!ctx->sqo_dead && !__io_sq_thread_acquire_mm (ctx) && !__io_sq_thread_acquire_files (ctx)) -__io_queue_sqe (req,NULL ); +__io_queue_sqe (req, &ctx->submit_state.comp);else __io_req_task_cancel (req, -EFAULT );mutex_unlock (&ctx->uring_lock);
Signature · 1 line
2.30.0