notification optimisation
v2 cover notification optimisation
Reuse request infra for zc notifications for optimisation and also a nice line count reduction.
v2:
add missing patch exporting io_alloc_req()
Pavel Begunkov (2):
io_uring: export req alloc from core
io_uring: notification completion optimisation
include/linux/io_uring_types.h | 7 -- io_uring/io_uring.c | 25 +----- io_uring/io_uring.h | 21 +++++ io_uring/net.c | 4 +- io_uring/notif.c | 159 +++++++++++---------------------- io_uring/notif.h | 42 +++------ 6 files changed, 89 insertions(+), 169 deletions(-)
Signature · 1 line
2.37.0
PATCH v2 1/2 io_uring: export req alloc from core
We want to do request allocation out of the core io_uring code, make the allocation functions public for other io_uring parts.
--- io_uring/io_uring.c | 22 +--------------------- io_uring/io_uring.h | 21 +++++++++++++++++++++ 2 files changed, 22 insertions(+), 21 deletions(-)
io_uring/io_uring.c +1 -21
@@ -852,18 +852,13 @@ static void io_flush_cached_locked_reqs(struct io_ring_ctx *ctx,spin_unlock (&ctx->completion_lock); } -static inline bool io_req_cache_empty (struct io_ring_ctx *ctx) -{ -return !ctx->submit_state.free_list.next; -} -/* * A request might get retired back into the request caches even before opcode * handlers and io_issue_sqe() are done with it, e.g. inline completion path. * Because of that, io_alloc_req() should be called only under ->uring_lock * and with extra caution to not get a request that is still worked on. */ -static __cold bool __io_alloc_req_refill (struct io_ring_ctx *ctx) +__cold bool __io_alloc_req_refill (struct io_ring_ctx *ctx) __must_hold(&ctx->uring_lock) {gfp_t gfp =GFP_KERNEL | __GFP_NOWARN;
@@ -904,21 +899,6 @@ static __cold bool __io_alloc_req_refill(struct io_ring_ctx *ctx)return true; } -static inline bool io_alloc_req_refill (struct io_ring_ctx *ctx) -{ -if (unlikely (io_req_cache_empty (ctx))) -return __io_alloc_req_refill (ctx); -return true; -} - -static inline struct io_kiocb *io_alloc_req (struct io_ring_ctx *ctx) -{ -struct io_wq_work_node *node; - - node =wq_stack_extract (&ctx->submit_state.free_list); - return container_of(node, struct io_kiocb, comp_list); -} -static inline void io_dismantle_req (struct io_kiocb *req) {unsigned int flags = req->flags;
io_uring/io_uring.h +21 -0
@@ -75,6 +75,7 @@ void io_free_req(struct io_kiocb *req);void io_queue_next (struct io_kiocb *req);void __io_put_task (struct task_struct *task,int nr);void io_task_refs_refill (struct io_uring_task *tctx); +bool __io_alloc_req_refill (struct io_ring_ctx *ctx);bool io_match_task_safe (struct io_kiocb *head,struct task_struct *task,bool cancel_all);
@@ -280,4 +281,24 @@ static inline void io_get_task_refs(int nr)io_task_refs_refill (tctx); } +static inline bool io_req_cache_empty (struct io_ring_ctx *ctx) +{ +return !ctx->submit_state.free_list.next; +} + +static inline bool io_alloc_req_refill (struct io_ring_ctx *ctx) +{ +if (unlikely (io_req_cache_empty (ctx))) +return __io_alloc_req_refill (ctx); +return true; +} + +static inline struct io_kiocb *io_alloc_req (struct io_ring_ctx *ctx) +{ +struct io_wq_work_node *node; + + node =wq_stack_extract (&ctx->submit_state.free_list); + return container_of(node, struct io_kiocb, comp_list); +} +#endif
Signature · 1 line
2.37.0
PATCH v2 2/2 io_uring: notification completion optimisation
In reply to Pavel BegunkovWe want to use all optimisations that we have for io_uring requests like completion batching, memory caching and more but for zc notifications. Fortunately, notification perfectly fit the request model so we can overlay them onto struct io_kiocb and use all the infratructure.
Most of the fields of struct io_notif natively fits into io_kiocb, so we replace struct io_notif with struct io_kiocb carrying struct io_notif_data in the cmd cache line. Then we adapt io_alloc_notif() to use io_alloc_req()/io_alloc_req_refill(), and kill leftovers of hand coded caching. __io_notif_complete_tw() is converted to use io_uring's tw infra.
--- include/linux/io_uring_types.h | 7 -- io_uring/io_uring.c | 3 - io_uring/net.c | 4 +- io_uring/notif.c | 159 +++++++++++---------------------- io_uring/notif.h | 42 +++------ 5 files changed, 67 insertions(+), 148 deletions(-)
include/linux/io_uring_types.h +0 -7
@@ -249,9 +249,6 @@ struct io_ring_ctx {struct xarray io_bl_xa;struct list_head io_buffers_cache; -/* struct io_notif cache, protected by uring_lock */ -struct list_head notif_list; -struct io_hash_table cancel_table_locked;struct list_head cq_overflow_list;struct io_alloc_cache apoll_cache;
@@ -262,10 +259,6 @@ struct io_ring_ctx {struct io_wq_work_list locked_free_list;unsigned int locked_free_nr; -/* struct io_notif cache protected by completion_lock */ -struct list_head notif_list_locked; -unsigned int notif_locked_nr; -const struct cred *sq_creds;/* cred used for __io_sq_thread() */ struct io_sq_data *sq_data;/* if using sq thread polling */
io_uring/io_uring.c +0 -3
@@ -321,8 +321,6 @@ static __cold struct io_ring_ctx *io_ring_ctx_alloc(struct io_uring_params *p)INIT_WQ_LIST (&ctx->locked_free_list);INIT_DELAYED_WORK (&ctx->fallback_work, io_fallback_req_func);INIT_WQ_LIST (&ctx->submit_state.compl_reqs); -INIT_LIST_HEAD (&ctx->notif_list); -INIT_LIST_HEAD (&ctx->notif_list_locked);return ctx; err:kfree (ctx->dummy_ubuf);
@@ -2466,7 +2464,6 @@ static __cold void io_ring_ctx_free(struct io_ring_ctx *ctx)WARN_ON_ONCE (!list_empty (&ctx->ltimeout_list));WARN_ON_ONCE (ctx->notif_slots || ctx->nr_notif_slots); -io_notif_cache_purge (ctx);io_mem_free (ctx->rings);io_mem_free (ctx->sq_sqes);
io_uring/net.c +2 -2
@@ -946,7 +946,7 @@ int io_sendzc(struct io_kiocb *req, unsigned int issue_flags)struct io_ring_ctx *ctx = req->ctx;struct io_sendzc *zc =io_kiocb_to_cmd (req);struct io_notif_slot *notif_slot; -struct io_notif *notif; +struct io_kiocb *notif;struct msghdr msg;struct iovec iov;struct socket *sock;
@@ -1005,7 +1005,7 @@ int io_sendzc(struct io_kiocb *req, unsigned int issue_flags) min_ret =iov_iter_count (&msg.msg_iter); msg.msg_flags = msg_flags; - msg.msg_ubuf = ¬if->uarg; + msg.msg_ubuf = &io_notif_to_data (notif)->uarg; msg.sg_from_iter = io_sg_from_iter; ret =sock_sendmsg (sock, &msg);
io_uring/notif.c +52 -107
216 lines not on this page · raw message
io_uring/notif.h +13 -29
83 lines not on this page · raw message
Signature · 1 line
2.37.0
Quoted 11 lines · Pavel Begunkov
Reuse request infra for zc notifications for optimisation and also a nice line count reduction.
v2:
add missing patch exportingio_alloc_req()Pavel Begunkov (2):
io_uring: export req alloc from core
io_uring: notification completion optimisation[...]
Applied, thanks!
[1/2] io_uring: export req alloc from core
commit: bd1a3783dd749012134b142b52e5704f7c142897
[2/2] io_uring: notification completion optimisation
commit: 14b146b688ad9593f5eee93d51a34d09a47e50b5Best regards,
Signature · 1 line
Jens Axboe