5.20 cleanups
cover 5.20 cleanups
Simple cleanups split off from a larger series.
Pavel Begunkov (10):
io_uring: make reg buf init consistent
io_uring: move defer_list to slow data
io_uring: better caching for ctx timeout fields
io_uring: refactor ctx slow data placement
io_uring: move small helpers to headers
io_uring: explain io_wq_work::cancel_seq placement
io_uring: inline ->registered_rings
io_uring: never defer-complete multi-apoll
io_uring: remove check_cq checking from hot paths
io_uring: don't set REQ_F_COMPLETE_INLINE in tw
io_uring/io-wq.h | 1 + io_uring/io_uring.c | 55 +++++++++---------------- io_uring/io_uring.h | 22 ++++++++++ io_uring/io_uring_types.h | 87 ++++++++++++++++++++------------------- io_uring/rsrc.c | 9 ++-- io_uring/tctx.c | 9 ---- io_uring/tctx.h | 3 +- 7 files changed, 93 insertions(+), 93 deletions(-)
Signature · 1 line
2.36.1
PATCH 1/10 io_uring: make reg buf init consistent
The default (i.e. empty) state of register buffer is dummy_ubuf, so set it to dummy on init instead of NULL.
--- io_uring/rsrc.c | 9 +++------ 1 file changed, 3 insertions(+), 6 deletions(-)
io_uring/rsrc.c +3 -6
@@ -567,7 +567,7 @@ static int __io_sqe_buffers_update(struct io_ring_ctx *ctx,io_buffer_unmap (ctx, &imu);break ; } - ctx->user_bufs[i] =NULL ; + ctx->user_bufs[i] = ctx->dummy_ubuf; needs_switch = true; }
@@ -1200,14 +1200,11 @@ static int io_sqe_buffer_register(struct io_ring_ctx *ctx, struct iovec *iov,size_t size;int ret, nr_pages, i; -if (!iov->iov_base) { - *pimu = ctx->dummy_ubuf; + *pimu = ctx->dummy_ubuf; +if (!iov->iov_base)return 0 ; - } - *pimu =NULL ; ret = -ENOMEM ; - pages =io_pin_pages ((unsigned long ) iov->iov_base, iov->iov_len, &nr_pages);if (IS_ERR (pages)) {
Signature · 1 line
2.36.1
PATCH 2/10 io_uring: move defer_list to slow data
In reply to Pavel Begunkovdraining is slow path, move defer_list to the end where slow data lives inside the context.
--- io_uring/io_uring_types.h | 5 ++++- 1 file changed, 4 insertions(+), 1 deletion(-)
io_uring/io_uring_types.h +4 -1
@@ -160,7 +160,6 @@ struct io_ring_ctx {struct io_uring_sqe *sq_sqes;unsigned cached_sq_head;unsigned sq_entries; -struct list_head defer_list;/* * Fixed resources fast path, should be accessed only under
@@ -272,8 +271,12 @@ struct io_ring_ctx {struct work_struct exit_work;struct list_head tctx_list;struct completion ref_comp; + +/* io-wq management, e.g. thread count */ u32 iowq_limits[2 ];bool iowq_limits_set; + +struct list_head defer_list; }; };
Signature · 1 line
2.36.1
PATCH 3/10 io_uring: better caching for ctx timeout fields
In reply to Pavel BegunkovFollowing timeout fields access patterns, move all of them into a separate cache line inside ctx, so they don't intervene with normal completion caching, especially since timeout removals and completion are separated and the later is done via tw.
It also sheds some bytes from io_ring_ctx, 1216B -> 1152B
--- io_uring/io_uring_types.h | 15 +++++++++------ 1 file changed, 9 insertions(+), 6 deletions(-)
io_uring/io_uring_types.h +9 -6
@@ -179,8 +179,6 @@ struct io_ring_ctx {struct xarray io_bl_xa;struct list_head io_buffers_cache; -struct list_head timeout_list; -struct list_head ltimeout_list;struct list_head cq_overflow_list;struct list_head apoll_cache;struct xarray personalities;
@@ -213,15 +211,11 @@ struct io_ring_ctx {struct io_ev_fd __rcu *io_ev_fd;struct wait_queue_head cq_wait;unsigned cq_extra; -atomic_t cq_timeouts; -unsigned cq_last_tm_flush; }____cacheline_aligned_in_smp ;struct {spinlock_t completion_lock; -spinlock_t timeout_lock; -/* * ->iopoll_list is protected by the ctx->uring_lock for * io_uring instances that don't use IORING_SETUP_SQPOLL.
@@ -253,6 +247,15 @@ struct io_ring_ctx {struct list_head io_buffers_pages; }; +/* timeouts */ +struct { +spinlock_t timeout_lock; +atomic_t cq_timeouts; +struct list_head timeout_list; +struct list_head ltimeout_list; +unsigned cq_last_tm_flush; + }____cacheline_aligned_in_smp ; +/* Keep this last, we don't need it for the fast path */ struct {#if defined(CONFIG_UNIX )
Signature · 1 line
2.36.1
PATCH 4/10 io_uring: refactor ctx slow data placement
In reply to Pavel BegunkovShove all slow path data at the end of ctx and get rid of extra indention.
--- io_uring/io_uring_types.h | 81 +++++++++++++++++++-------------------- 1 file changed, 39 insertions(+), 42 deletions(-)
io_uring/io_uring_types.h +39 -42
@@ -183,7 +183,6 @@ struct io_ring_ctx {struct list_head apoll_cache;struct xarray personalities;u32 pers_next; -unsigned sq_thread_idle; }____cacheline_aligned_in_smp ;/* IRQ completion list, under ->completion_lock */
@@ -230,23 +229,6 @@ struct io_ring_ctx {struct list_head io_buffers_comp; }____cacheline_aligned_in_smp ; -struct io_restriction restrictions; - -/* slow path rsrc auxilary data, used by update/register */ -struct { -struct io_rsrc_node *rsrc_backup_node; -struct io_mapped_ubuf *dummy_ubuf; -struct io_rsrc_data *file_data; -struct io_rsrc_data *buf_data; - -struct delayed_work rsrc_put_work; -struct llist_head rsrc_put_llist; -struct list_head rsrc_ref_list; -spinlock_t rsrc_ref_lock; - -struct list_head io_buffers_pages; - }; -/* timeouts */ struct {spinlock_t timeout_lock;
@@ -257,30 +239,45 @@ struct io_ring_ctx { }____cacheline_aligned_in_smp ;/* Keep this last, we don't need it for the fast path */ -struct { -#if defined(CONFIG_UNIX ) -struct socket *ring_sock; -#endif -/* hashed buffered write serialization */ -struct io_wq_hash *hash_map; - -/* Only used for accounting purposes */ -struct user_struct *user; -struct mm_struct *mm_account; - -/* ctx exit and cancelation */ -struct llist_head fallback_llist; -struct delayed_work fallback_work; -struct work_struct exit_work; -struct list_head tctx_list; -struct completion ref_comp; - -/* io-wq management, e.g. thread count */ -u32 iowq_limits[2 ]; -bool iowq_limits_set; - -struct list_head defer_list; - }; + +struct io_restriction restrictions; + +/* slow path rsrc auxilary data, used by update/register */ +struct io_rsrc_node *rsrc_backup_node; +struct io_mapped_ubuf *dummy_ubuf; +struct io_rsrc_data *file_data; +struct io_rsrc_data *buf_data; + +struct delayed_work rsrc_put_work; +struct llist_head rsrc_put_llist; +struct list_head rsrc_ref_list; +spinlock_t rsrc_ref_lock; + +struct list_head io_buffers_pages; + +#if defined(CONFIG_UNIX ) +struct socket *ring_sock; +#endif +/* hashed buffered write serialization */ +struct io_wq_hash *hash_map; + +/* Only used for accounting purposes */ +struct user_struct *user; +struct mm_struct *mm_account; + +/* ctx exit and cancelation */ +struct llist_head fallback_llist; +struct delayed_work fallback_work; +struct work_struct exit_work; +struct list_head tctx_list; +struct completion ref_comp; + +/* io-wq management, e.g. thread count */ +u32 iowq_limits[2 ]; +bool iowq_limits_set; + +struct list_head defer_list; +unsigned sq_thread_idle; };enum {
Signature · 1 line
2.36.1
PATCH 5/10 io_uring: move small helpers to headers
In reply to Pavel BegunkovThere is a bunch of inline helpers that will be useful not only to the core of io_uring, move them to headers.
--- io_uring/io_uring.c | 17 ----------------- io_uring/io_uring.h | 22 ++++++++++++++++++++++ 2 files changed, 22 insertions(+), 17 deletions(-)
io_uring/io_uring.c +0 -17
@@ -615,14 +615,6 @@ struct sock *io_uring_get_socket(struct file *file) }EXPORT_SYMBOL (io_uring_get_socket); -static inline void io_tw_lock (struct io_ring_ctx *ctx,bool *locked) -{ -if (!*locked) { -mutex_lock (&ctx->uring_lock); - *locked = true; - } -} -static inline void io_submit_flush_completions (struct io_ring_ctx *ctx) {if (!wq_list_empty (&ctx->submit_state.compl_reqs))
@@ -878,15 +870,6 @@ static void io_prep_async_link(struct io_kiocb *req) } } -static inline void io_req_add_compl_list (struct io_kiocb *req) -{ -struct io_submit_state *state = &req->ctx->submit_state; - -if (!(req->flags &REQ_F_CQE_SKIP )) - state->flush_cqes = true; -wq_list_add_tail (&req->comp_list, &state->compl_reqs); -} -void io_queue_iowq (struct io_kiocb *req,bool *dont_use) {struct io_kiocb *link =io_prep_linked_timeout (req);
io_uring/io_uring.h +22 -0
@@ -217,6 +217,28 @@ static inline bool io_run_task_work(void)return false; } +static inline void io_req_complete_state (struct io_kiocb *req) +{ + req->flags |=REQ_F_COMPLETE_INLINE ; +} + +static inline void io_tw_lock (struct io_ring_ctx *ctx,bool *locked) +{ +if (!*locked) { +mutex_lock (&ctx->uring_lock); + *locked = true; + } +} + +static inline void io_req_add_compl_list (struct io_kiocb *req) +{ +struct io_submit_state *state = &req->ctx->submit_state; + +if (!(req->flags &REQ_F_CQE_SKIP )) + state->flush_cqes = true; +wq_list_add_tail (&req->comp_list, &state->compl_reqs); +} +int io_run_task_work_sig (void );void io_req_complete_failed (struct io_kiocb *req,s32 res);void __io_req_complete (struct io_kiocb *req,unsigned issue_flags);
Signature · 1 line
2.36.1
PATCH 6/10 io_uring: explain io_wq_work::cancel_seq placement
In reply to Pavel BegunkovAdd a comment on why we keep ->cancel_seq in struct io_wq_work instead of struct io_kiocb despite it needed only by io_uring but not io-wq.
--- io_uring/io-wq.h | 1 + 1 file changed, 1 insertion(+)
io_uring/io-wq.h +1 -0
@@ -155,6 +155,7 @@ struct io_wq_work_node *wq_stack_extract(struct io_wq_work_node *stack)struct io_wq_work {struct io_wq_work_node list;unsigned flags; +/* place it here instead of io_kiocb as it fills padding and saves 4B */ int cancel_seq; };
Signature · 1 line
2.36.1
PATCH 7/10 io_uring: inline ->registered_rings
In reply to Pavel BegunkovThere can be only 16 registered rings, no need to allocate an array for them separately but store it in tctx.
--- io_uring/io_uring.c | 1 - io_uring/tctx.c | 9 --------- io_uring/tctx.h | 3 ++- 3 files changed, 2 insertions(+), 11 deletions(-)
io_uring/io_uring.c +0 -1
@@ -2744,7 +2744,6 @@ void __io_uring_free(struct task_struct *tsk)WARN_ON_ONCE (tctx->io_wq);WARN_ON_ONCE (tctx->cached_refs); -kfree (tctx->registered_rings);percpu_counter_destroy (&tctx->inflight);kfree (tctx); tsk->io_uring =NULL ;
io_uring/tctx.c +0 -9
@@ -55,16 +55,8 @@ __cold int io_uring_alloc_task_context(struct task_struct *task,if (unlikely (!tctx))return -ENOMEM ; - tctx->registered_rings =kcalloc (IO_RINGFD_REG_MAX , -sizeof (struct file *),GFP_KERNEL ); -if (unlikely (!tctx->registered_rings)) { -kfree (tctx); -return -ENOMEM ; - } - ret =percpu_counter_init (&tctx->inflight,0 ,GFP_KERNEL );if (unlikely (ret)) { -kfree (tctx->registered_rings);kfree (tctx);return ret; }
@@ -73,7 +65,6 @@ __cold int io_uring_alloc_task_context(struct task_struct *task,if (IS_ERR (tctx->io_wq)) { ret =PTR_ERR (tctx->io_wq);percpu_counter_destroy (&tctx->inflight); -kfree (tctx->registered_rings);kfree (tctx);return ret; }
io_uring/tctx.h +2 -1
@@ -20,8 +20,9 @@ struct io_uring_task {struct io_wq_work_list task_list;struct io_wq_work_list prio_task_list;struct callback_head task_work; -struct file **registered_rings;bool task_running; + +struct file *registered_rings[IO_RINGFD_REG_MAX ]; };struct io_tctx_node {
Signature · 1 line
2.36.1
PATCH 8/10 io_uring: never defer-complete multi-apoll
In reply to Pavel BegunkovLuckily, nnobody completes multi-apoll requests outside the polling functions, but don't set IO_URING_F_COMPLETE_DEFER in any case as there is nobody who is catching REQ_F_COMPLETE_INLINE, and so will leak requests if used.
--- io_uring/io_uring.c | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-)
io_uring/io_uring.c +1 -1
@@ -2054,7 +2054,7 @@ int io_poll_issue(struct io_kiocb *req, bool *locked)io_tw_lock (req->ctx, locked);if (unlikely (req->task->flags &PF_EXITING ))return -EFAULT ; -return io_issue_sqe (req,IO_URING_F_NONBLOCK |IO_URING_F_COMPLETE_DEFER ); +return io_issue_sqe (req,IO_URING_F_NONBLOCK ); }struct io_wq_work *io_wq_free_work (struct io_wq_work *work)
Signature · 1 line
2.36.1
PATCH 9/10 io_uring: remove check_cq checking from hot paths
In reply to Pavel BegunkovAll ctx->check_cq events are slow path, don't test every single flag one by one in the hot path, but add a common guarding if.
--- io_uring/io_uring.c | 34 +++++++++++++++++++--------------- 1 file changed, 19 insertions(+), 15 deletions(-)
io_uring/io_uring.c +19 -15
@@ -1714,24 +1714,25 @@ static int io_iopoll_check(struct io_ring_ctx *ctx, long min)int ret =0 ;unsigned long check_cq; + check_cq =READ_ONCE (ctx->check_cq); +if (unlikely (check_cq)) { +if (check_cq &BIT (IO_CHECK_CQ_OVERFLOW_BIT )) +__io_cqring_overflow_flush (ctx, false); +/* +* Similarly do not spin if we have not informed the user of any +* dropped CQE. +*/ +if (check_cq &BIT (IO_CHECK_CQ_DROPPED_BIT )) +return -EBADR ; + }/* * Don't enter poll loop if we already have events pending. * If we do, we can potentially be spinning for commands that * already triggered a CQE (eg in error). */ - check_cq =READ_ONCE (ctx->check_cq); -if (check_cq &BIT (IO_CHECK_CQ_OVERFLOW_BIT )) -__io_cqring_overflow_flush (ctx, false);if (io_cqring_events (ctx))return 0 ; -/* -* Similarly do not spin if we have not informed the user of any -* dropped CQE. -*/ -if (unlikely (check_cq &BIT (IO_CHECK_CQ_DROPPED_BIT ))) -return -EBADR ; - do { /* * If a submit got punted to a workqueue, we can have the
@@ -2657,12 +2658,15 @@ static inline int io_cqring_wait_schedule(struct io_ring_ctx *ctx, ret =io_run_task_work_sig ();if (ret ||io_should_wake (iowq))return ret; + check_cq =READ_ONCE (ctx->check_cq); -/* let the caller flush overflows, retry */ -if (check_cq &BIT (IO_CHECK_CQ_OVERFLOW_BIT )) -return 1 ; -if (unlikely (check_cq &BIT (IO_CHECK_CQ_DROPPED_BIT ))) -return -EBADR ; +if (unlikely (check_cq)) { +/* let the caller flush overflows, retry */ +if (check_cq &BIT (IO_CHECK_CQ_OVERFLOW_BIT )) +return 1 ; +if (check_cq &BIT (IO_CHECK_CQ_DROPPED_BIT )) +return -EBADR ; + }if (!schedule_hrtimeout (&timeout,HRTIMER_MODE_ABS ))return -ETIME ;return 1 ;
Signature · 1 line
2.36.1
PATCH 10/10 io_uring: don't set REQ_F_COMPLETE_INLINE in tw
In reply to Pavel Begunkovio_req_task_complete() enqueues requests for state completion itself, no need for REQ_F_COMPLETE_INLINE, which is only serve the purpose of not bloating the kernel.
--- io_uring/io_uring.c | 1 - 1 file changed, 1 deletion(-)
io_uring/io_uring.c +0 -1
@@ -1769,7 +1769,6 @@ inline void io_req_task_complete(struct io_kiocb *req, bool *locked) {if (*locked) { req->cqe.flags |=io_put_kbuf (req,0 ); - req->flags |=REQ_F_COMPLETE_INLINE ;io_req_add_compl_list (req); }else { req->cqe.flags |=io_put_kbuf (req,IO_URING_F_UNLOCKED );
Signature · 1 line
2.36.1
Quoted 14 lines · Pavel Begunkov
Simple cleanups split off from a larger series.
Pavel Begunkov (10):
io_uring: make reg buf init consistent
io_uring: movedefer_listto slow data
io_uring: better caching for ctx timeout fields
io_uring: refactor ctx slow data placement
io_uring: move small helpers to headers
io_uring: explainio_wq_work::cancel_seqplacement
io_uring: inline ->registered_rings
io_uring: never defer-complete multi-apoll
io_uring: removecheck_cqchecking from hot paths
io_uring: don't setREQ_F_COMPLETE_INLINEin tw
Quoted 1 line
[...]
Applied, thanks!
[01/10] io_uring: make reg buf init consistent
commit: 8c81b9a8afeb9bf9a77ed7b8ae18fdcdd5e8738c
[02/10] io_uring: move defer_list to slow data
commit: 2946124bd54c6bde7d8223764f9e29ee5e9c2872
[03/10] io_uring: better caching for ctx timeout fields
commit: fee5d8c21d58b32c0e7f4dbddfa79ea2badfe080
[04/10] io_uring: refactor ctx slow data placement
commit: 5545259f66477791ead5305d080e7315ab93e1d2
[05/10] io_uring: move small helpers to headers
commit: 8f056215cea9a0b8a86d980c71da5587291f11c8
[06/10] io_uring: explain io_wq_work::cancel_seq placement
commit: 588383e3417729d24c804d43d9f08f3b1756c5cf
[07/10] io_uring: inline ->registered_rings
commit: 9d0222c4d9d1de014fea4ef151e6743b8eb30e8a
[08/10] io_uring: never defer-complete multi-apoll
commit: 5bbc2038f4d8f3de273c74779882ecb9a959a46d
[09/10] io_uring: remove check_cq checking from hot paths
commit: bc132bba5459cb501737c5793d1a273354dbf8db
[10/10] io_uring: don't set REQ_F_COMPLETE_INLINE in tw
commit: 01955c135f1753e60587fc28679fc1fab8ebc4d4Best regards,
Signature · 1 line
Jens Axboe