small tw add improvements
cover small tw add improvements
Fiddle with inlining of task_work add so the compiler generates more concise and efficient code.
Pavel Begunkov (2):
io_uring: inline io_req_task_work_add()
io_uring: split tw fallback into a function
io_uring/io_uring.c | 32 ++++++++++++++++---------------- io_uring/io_uring.h | 7 ++++++- 2 files changed, 22 insertions(+), 17 deletions(-)
Signature · 1 line
2.38.1
PATCH 1/2 io_uring: inline io_req_task_work_add()
__io_req_task_work_add() is huge but marked inline, that makes compilers to generate lots of garbage. Inline the wrapper caller io_req_task_work_add() instead.
before and after:
text data bss dec hex filename
47347 16248 8 63603 f873 io_uring/io_uring.o
text data bss dec hex filename
45303 16248 8 61559 f077 io_uring/io_uring.o
--- io_uring/io_uring.c | 7 +------ io_uring/io_uring.h | 7 ++++++- 2 files changed, 7 insertions(+), 7 deletions(-)
io_uring/io_uring.c +1 -6
@@ -1117,7 +1117,7 @@ static void io_req_local_work_add(struct io_kiocb *req)__io_cqring_wake (ctx); } -static inline void __io_req_task_work_add (struct io_kiocb *req,bool allow_local) +void __io_req_task_work_add (struct io_kiocb *req,bool allow_local) {struct io_uring_task *tctx = req->task->io_uring;struct io_ring_ctx *ctx = req->ctx;
@@ -1149,11 +1149,6 @@ static inline void __io_req_task_work_add(struct io_kiocb *req, bool allow_local } } -void io_req_task_work_add (struct io_kiocb *req) -{ -__io_req_task_work_add (req, true); -} -static void __cold io_move_task_work_from_local (struct io_ring_ctx *ctx) {struct llist_node *node;
io_uring/io_uring.h +6 -1
@@ -48,9 +48,9 @@ static inline bool io_req_ffs_set(struct io_kiocb *req)return req->flags &REQ_F_FIXED_FILE ; } +void __io_req_task_work_add (struct io_kiocb *req,bool allow_local);bool io_is_uring_fops (struct file *file);bool io_alloc_async_data (struct io_kiocb *req); -void io_req_task_work_add (struct io_kiocb *req);void io_req_tw_post_queue (struct io_kiocb *req,s32 res,u32 cflags);void io_req_task_queue (struct io_kiocb *req);void io_queue_iowq (struct io_kiocb *req,bool *dont_use);
@@ -80,6 +80,11 @@ bool __io_alloc_req_refill(struct io_ring_ctx *ctx);bool io_match_task_safe (struct io_kiocb *head,struct task_struct *task,bool cancel_all); +static inline void io_req_task_work_add (struct io_kiocb *req) +{ +__io_req_task_work_add (req, true); +} +#define io_for_each_link (pos, head) \ for (pos = (head); pos; pos = pos->link)
Signature · 1 line
2.38.1
PATCH 2/2 io_uring: split tw fallback into a function
In reply to Pavel BegunkovWhen the target process is dying and so task_work_add() is not allowed we push all task_work item to the fallback workqueue. Move the part responsible for moving tw items out of __io_req_task_work_add() into a separate function. Makes it a bit cleaner and gives the compiler a bit of extra info.
--- io_uring/io_uring.c | 25 +++++++++++++++---------- 1 file changed, 15 insertions(+), 10 deletions(-)
io_uring/io_uring.c +15 -10
@@ -1095,6 +1095,20 @@ void tctx_task_work(struct callback_head *cb)trace_io_uring_task_work_run (tctx, count, loops); } +static __cold void io_fallback_tw (struct io_uring_task *tctx) +{ +struct llist_node *node =llist_del_all (&tctx->task_list); +struct io_kiocb *req; + +while (node) { + req = container_of(node, struct io_kiocb, io_task_work.node); + node = node->next; +if (llist_add (&req->io_task_work.node, + &req->ctx->fallback_llist)) +schedule_delayed_work (&req->ctx->fallback_work,1 ); + } +} +static void io_req_local_work_add (struct io_kiocb *req) {struct io_ring_ctx *ctx = req->ctx;
@@ -1121,7 +1135,6 @@ void __io_req_task_work_add(struct io_kiocb *req, bool allow_local) {struct io_uring_task *tctx = req->task->io_uring;struct io_ring_ctx *ctx = req->ctx; -struct llist_node *node;if (allow_local && ctx->flags &IORING_SETUP_DEFER_TASKRUN ) {io_req_local_work_add (req);
@@ -1138,15 +1151,7 @@ void __io_req_task_work_add(struct io_kiocb *req, bool allow_local)if (likely (!task_work_add (req->task, &tctx->task_work, ctx->notify_method)))return ; - node =llist_del_all (&tctx->task_list); - -while (node) { - req = container_of(node, struct io_kiocb, io_task_work.node); - node = node->next; -if (llist_add (&req->io_task_work.node, - &req->ctx->fallback_llist)) -schedule_delayed_work (&req->ctx->fallback_work,1 ); - } +io_fallback_tw (tctx); }static void __cold io_move_task_work_from_local (struct io_ring_ctx *ctx)
Signature · 1 line
2.38.1
Quoted 8 lines · Pavel Begunkov
Fiddle with inlining of
task_workadd so the compiler generates more concise and efficient code.Pavel Begunkov (2):
io_uring: inlineio_req_task_work_add()
io_uring: split tw fallback into a function[...]
Applied, thanks!
[1/2] io_uring: inline io_req_task_work_add()
commit: 912f3f541dd8fb9e987e4bef0f2466b5233ec480
[2/2] io_uring: split tw fallback into a function
commit: 79fde04791f9a6789ff1a9b90b06e754efb55bd7Best regards,
Signature · 1 line
Jens Axboe