io_uring/uring_cmd: skip io_uring_cmd_issue_blocking() task work
PATCH 1/1 io_uring/uring_cmd: skip io_uring_cmd_issue_blocking() task work
io_uring_cmd_issue_blocking() is only called from blk_cmd_complete(), which is already a task work callback. However, it queues another task work item, to call io_queue_iowq(). Just call io_queue_iowq() directly to skip the CPU cost and latency of the redundant task work proxying.
--- io_uring/io_uring.c | 13 +------------ io_uring/io_uring.h | 2 +- io_uring/uring_cmd.c | 2 +- 3 files changed, 3 insertions(+), 14 deletions(-)
io_uring/io_uring.c +1 -12
@@ -405,11 +405,11 @@ static void io_prep_async_link(struct io_kiocb *req)io_for_each_link (cur, req)io_prep_async_work (cur); } } -static void io_queue_iowq (struct io_kiocb *req) +void io_queue_iowq (struct io_kiocb *req) {struct io_uring_task *tctx = req->tctx;BUG_ON (!tctx);
@@ -433,21 +433,10 @@ static void io_queue_iowq(struct io_kiocb *req)trace_io_uring_queue_async_work (req,io_wq_is_hashed (&req->work));io_wq_enqueue (tctx->io_wq, &req->work); } -static void io_req_queue_iowq_tw (struct io_tw_req tw_req,io_tw_token_t tw) -{ -io_queue_iowq (tw_req.req); -} - -void io_req_queue_iowq (struct io_kiocb *req) -{ - req->io_task_work.func = io_req_queue_iowq_tw; -io_req_task_work_add (req); -} -unsigned io_linked_nr (struct io_kiocb *req) {struct io_kiocb *tmp;unsigned nr =0 ;
io_uring/io_uring.h +1 -1
@@ -193,11 +193,11 @@ void io_req_task_queue_fail(struct io_kiocb *req, int ret);void io_req_task_submit (struct io_tw_req tw_req,io_tw_token_t tw);__cold void io_uring_drop_tctx_refs (struct task_struct *task);int io_ring_add_registered_file (struct io_uring_task *tctx,struct file *file,int start,int end); -void io_req_queue_iowq (struct io_kiocb *req); +void io_queue_iowq (struct io_kiocb *req);int io_poll_issue (struct io_kiocb *req,io_tw_token_t tw);int io_submit_sqes (struct io_ring_ctx *ctx,unsigned int nr);int io_do_iopoll (struct io_ring_ctx *ctx,bool force_nonspin);__cold void io_iopoll_try_reap_events (struct io_ring_ctx *ctx);
io_uring/uring_cmd.c +1 -1
@@ -324,11 +324,11 @@ EXPORT_SYMBOL_GPL(io_uring_cmd_import_fixed_vec);void io_uring_cmd_issue_blocking (struct io_uring_cmd *ioucmd) {struct io_kiocb *req =cmd_to_io_kiocb (ioucmd); -io_req_queue_iowq (req); +io_queue_iowq (req); }int io_cmd_poll_multishot (struct io_uring_cmd *cmd,unsigned int issue_flags,__poll_t mask) {
Signature · 1 line
2.54.0
Hi Jens,
I think you had mentioned planning to queue this patch up for 7.3:
https://lore.kernel.org/io-uring/[email protected]/
I didn't see it in for-7.3/io_uring, so just checking if that's still your plan.
Thanks,
Caleb
Quoted 89 lines · Caleb Sander Mateos
io_uring_cmd_issue_blocking()is only called fromblk_cmd_complete(), which is already a task work callback. However, it queues another task work item, to callio_queue_iowq(). Just callio_queue_iowq()directly to skip the CPU cost and latency of the redundant task work proxying.Signed-off-by Caleb Sander Mateos <[email protected]>--- io_uring/io_uring.c | 13 +------------ io_uring/io_uring.h | 2 +- io_uring/uring_cmd.c | 2 +- 3 files changed, 3 insertions(+), 14 deletions(-)io_uring/io_uring.c +1
-12@@ -405,11 +405,11 @@ static void io_prep_async_link(struct io_kiocb *req)io_for_each_link (cur, req)io_prep_async_work (cur); } } -static void io_queue_iowq (struct io_kiocb *req) +void io_queue_iowq (struct io_kiocb *req) {struct io_uring_task *tctx = req->tctx;BUG_ON (!tctx);@@ -433,21 +433,10 @@ static void io_queue_iowq(struct io_kiocb *req)trace_io_uring_queue_async_work (req,io_wq_is_hashed (&req->work));io_wq_enqueue (tctx->io_wq, &req->work); } -static void io_req_queue_iowq_tw (struct io_tw_req tw_req,io_tw_token_t tw) -{ -io_queue_iowq (tw_req.req); -} - -void io_req_queue_iowq (struct io_kiocb *req) -{ - req->io_task_work.func = io_req_queue_iowq_tw; -io_req_task_work_add (req); -} -unsigned io_linked_nr (struct io_kiocb *req) {struct io_kiocb *tmp;unsigned nr =0 ;io_uring/io_uring.h +1
-1@@ -193,11 +193,11 @@ void io_req_task_queue_fail(struct io_kiocb *req, int ret);void io_req_task_submit (struct io_tw_req tw_req,io_tw_token_t tw);__cold void io_uring_drop_tctx_refs (struct task_struct *task);int io_ring_add_registered_file (struct io_uring_task *tctx,struct file *file,int start,int end); -void io_req_queue_iowq (struct io_kiocb *req); +void io_queue_iowq (struct io_kiocb *req);int io_poll_issue (struct io_kiocb *req,io_tw_token_t tw);int io_submit_sqes (struct io_ring_ctx *ctx,unsigned int nr);int io_do_iopoll (struct io_ring_ctx *ctx,bool force_nonspin);__cold void io_iopoll_try_reap_events (struct io_ring_ctx *ctx);io_uring/uring_cmd.c +1
-2@@ -324,11 +324,11 @@ EXPORT_SYMBOL_GPL(io_uring_cmd_import_fixed_vec);void io_uring_cmd_issue_blocking (struct io_uring_cmd *ioucmd) {struct io_kiocb *req =cmd_to_io_kiocb (ioucmd); -io_req_queue_iowq (req); +io_queue_iowq (req); }int io_cmd_poll_multishot (struct io_uring_cmd *cmd,unsigned int issue_flags,__poll_t mask) { --2.54.0
Quoted 4 lines · Caleb Sander Mateos
io_uring_cmd_issue_blocking()is only called fromblk_cmd_complete(), which is already a task work callback. However, it queues another task work item, to callio_queue_iowq(). Just callio_queue_iowq()directly to skip the CPU cost and latency of the redundant task work proxying.
Applied, thanks!
[1/1] io_uring/uring_cmd: skip io_uring_cmd_issue_blocking() task work
commit: 13b0742fd05864384724710c1bb12643b5448015
Best regards,
Signature · 1 line
Jens Axboe