Cleanup and improve MSG_RING performance
Jens Axboe
maintainercover Cleanup and improve MSG_RING performance
Hi,
MSG_RING rolls its own task_work handling, which there's really no need for. Rather, it should use the generic io_uring infrastructure for this.
Add a helper for remote execution, and switch over MSG_RING to use it. This both cleans up the code, and improves performance of this opcode considerably.
io_uring/io_uring.c | 27 +++++++++++++++++++-------- io_uring/io_uring.h | 2 ++ io_uring/msg_ring.c | 37 ++++++++++++++----------------------- 3 files changed, 35 insertions(+), 31 deletions(-)
Signature · 1 line
Jens Axboe
Jens Axboe
maintainerPATCH 1/3 io_uring: add remote task_work execution helper
All our task_work handling is targeted at the state in the io_kiocb itself, which is what it is being used for. However, MSG_RING rolls its own task_work handling, ignoring how that is usually done.
In preparation for switching MSG_RING to be able to use the normal task_work handling, add io_req_task_work_add_remote() which allows the caller to pass in the target io_ring_ctx and task.
--- io_uring/io_uring.c | 27 +++++++++++++++++++-------- io_uring/io_uring.h | 2 ++ 2 files changed, 21 insertions(+), 8 deletions(-)
io_uring/io_uring.c +19 -8
@@ -1241,9 +1241,10 @@ void tctx_task_work(struct callback_head *cb)WARN_ON_ONCE (ret); } -static inline void io_req_local_work_add (struct io_kiocb *req,unsigned tw_flags) +static inline void io_req_local_work_add (struct io_kiocb *req, +struct io_ring_ctx *ctx, +unsigned tw_flags) { -struct io_ring_ctx *ctx = req->ctx;unsigned nr_wait, nr_tw, nr_tw_prev;unsigned long flags;
@@ -1291,9 +1292,10 @@ static inline void io_req_local_work_add(struct io_kiocb *req, unsigned tw_flagswake_up_state (ctx->submitter_task,TASK_INTERRUPTIBLE ); } -static void io_req_normal_work_add (struct io_kiocb *req) +static void io_req_normal_work_add (struct io_kiocb *req, +struct task_struct *task) { -struct io_uring_task *tctx = req->task->io_uring; +struct io_uring_task *tctx = task->io_uring;struct io_ring_ctx *ctx = req->ctx;unsigned long flags;bool was_empty;
@@ -1319,7 +1321,7 @@ static void io_req_normal_work_add(struct io_kiocb *req)return ; } -if (likely (!task_work_add (req->task, &tctx->task_work, ctx->notify_method))) +if (likely (!task_work_add (task, &tctx->task_work, ctx->notify_method)))return ;io_fallback_tw (tctx, false);
@@ -1328,9 +1330,18 @@ static void io_req_normal_work_add(struct io_kiocb *req)void __io_req_task_work_add (struct io_kiocb *req,unsigned flags) {if (req->ctx->flags &IORING_SETUP_DEFER_TASKRUN ) -io_req_local_work_add (req, flags); +io_req_local_work_add (req, req->ctx, flags); +else +io_req_normal_work_add (req, req->task); +} + +void io_req_task_work_add_remote (struct io_kiocb *req,struct task_struct *task, +struct io_ring_ctx *ctx,unsigned flags) +{ +if (req->ctx->flags &IORING_SETUP_DEFER_TASKRUN ) +io_req_local_work_add (req, ctx, flags);else -io_req_normal_work_add (req); +io_req_normal_work_add (req, task); }static void __cold io_move_task_work_from_local (struct io_ring_ctx *ctx)
@@ -1349,7 +1360,7 @@ static void __cold io_move_task_work_from_local(struct io_ring_ctx *ctx) io_task_work.node); node = node->next; -io_req_normal_work_add (req); +io_req_normal_work_add (req, req->task); } }
io_uring/io_uring.h +2 -0
@@ -74,6 +74,8 @@ struct file *io_file_get_fixed(struct io_kiocb *req, int fd, unsigned issue_flags);void __io_req_task_work_add (struct io_kiocb *req,unsigned flags); +void io_req_task_work_add_remote (struct io_kiocb *req,struct task_struct *task, +struct io_ring_ctx *ctx,unsigned flags);bool io_alloc_async_data (struct io_kiocb *req);void io_req_task_queue (struct io_kiocb *req);void io_req_task_complete (struct io_kiocb *req,struct io_tw_state *ts);
Signature · 1 line
2.43.0
Re: [PATCH 1/3] io_uring: add remote task_work execution helper
Quoted 55 lines · Jens Axboe
All our
task_workhandling is targeted at the state in theio_kiocbitself, which is what it is being used for. However,MSG_RINGrolls its owntask_workhandling, ignoring how that is usually done.In preparation for switching
MSG_RINGto be able to use the normaltask_workhandling, addio_req_task_work_add_remote()which allows the caller to pass in the targetio_ring_ctxand task.Signed-off-by Jens Axboe <[email protected]>--- io_uring/io_uring.c | 27 +++++++++++++++++++-------- io_uring/io_uring.h | 2 ++ 2 files changed, 21 insertions(+), 8 deletions(-)io_uring/io_uring.c +7
-5@@ -1241,9 +1241,10 @@ void tctx_task_work(struct callback_head *cb)WARN_ON_ONCE (ret); } -static inline void io_req_local_work_add (struct io_kiocb *req,unsigned tw_flags) +static inline void io_req_local_work_add (struct io_kiocb *req, +struct io_ring_ctx *ctx, +unsigned tw_flags) { -struct io_ring_ctx *ctx = req->ctx;unsigned nr_wait, nr_tw, nr_tw_prev;unsigned long flags;@@ -1291,9 +1292,10 @@ static inline void io_req_local_work_add(struct io_kiocb *req, unsigned tw_flagswake_up_state (ctx->submitter_task,TASK_INTERRUPTIBLE ); } -static void io_req_normal_work_add (struct io_kiocb *req) +static void io_req_normal_work_add (struct io_kiocb *req, +struct task_struct *task) { -struct io_uring_task *tctx = req->task->io_uring; +struct io_uring_task *tctx = task->io_uring;struct io_ring_ctx *ctx = req->ctx;unsigned long flags;bool was_empty;@@ -1319,7 +1321,7 @@ static void io_req_normal_work_add(struct io_kiocb *req)return ; } -if (likely (!task_work_add (req->task, &tctx->task_work, ctx->notify_method))) +if (likely (!task_work_add (task, &tctx->task_work, ctx->notify_method)))return ;io_fallback_tw (tctx, false);@@ -1328,9 +1330,18 @@ static void io_req_normal_work_add(struct io_kiocb *req)void __io_req_task_work_add (struct io_kiocb *req,unsigned flags)
io_uring/io_uring.c static void io_req_normal_work_add(struct io_kiocb *req) · line 1339
Quoted 10 lines
io_uring/io_uring.c +7
-1{if (req->ctx->flags &IORING_SETUP_DEFER_TASKRUN ) -io_req_local_work_add (req, flags); +io_req_local_work_add (req, req->ctx, flags); +else +io_req_normal_work_add (req, req->task); +} + +void io_req_task_work_add_remote (struct io_kiocb *req,struct task_struct *task, +struct io_ring_ctx *ctx,unsigned flags)
Urgh, even the declration screams that there is something wrong considering it _either_ targets @ctx or @task.
Just pass @ctx, so it either use ctx->submitter_task or req->task, hmm?
A side note, it's a dangerous game, I told it before. At least it would've been nice to abuse lockdep in a form of:
io_req_task_complete(req, tw, ctx) {
lockdep_assert(req->ctx == ctx);
...
}but we don't have @ctx there, maybe we'll add it to tw later.
Quoted 31 lines
io_uring/io_uring.c +5
-2+{ +if (req->ctx->flags &IORING_SETUP_DEFER_TASKRUN ) +io_req_local_work_add (req, ctx, flags);else -io_req_normal_work_add (req); +io_req_normal_work_add (req, task); }static void __cold io_move_task_work_from_local (struct io_ring_ctx *ctx)@@ -1349,7 +1360,7 @@ static void __cold io_move_task_work_from_local(struct io_ring_ctx *ctx) io_task_work.node); node = node->next; -io_req_normal_work_add (req); +io_req_normal_work_add (req, req->task); } }io_uring/io_uring.h +2
-0@@ -74,6 +74,8 @@ struct file *io_file_get_fixed(struct io_kiocb *req, int fd, unsigned issue_flags);void __io_req_task_work_add (struct io_kiocb *req,unsigned flags); +void io_req_task_work_add_remote (struct io_kiocb *req,struct task_struct *task, +struct io_ring_ctx *ctx,unsigned flags);bool io_alloc_async_data (struct io_kiocb *req);void io_req_task_queue (struct io_kiocb *req);void io_req_task_complete (struct io_kiocb *req,struct io_tw_state *ts);
Signature · 1 line
Pavel Begunkov
Jens Axboe
maintainerRe: [PATCH 1/3] io_uring: add remote task_work execution helper
Quoted 65 lines · Pavel Begunkov
Quoted 50 lines · Jens Axboe
All our
task_workhandling is targeted at the state in theio_kiocbitself, which is what it is being used for. However,MSG_RINGrolls its owntask_workhandling, ignoring how that is usually done.In preparation for switching
MSG_RINGto be able to use the normaltask_workhandling, addio_req_task_work_add_remote()which allows the caller to pass in the targetio_ring_ctxand task.Signed-off-by Jens Axboe <[email protected]>--- io_uring/io_uring.c | 27 +++++++++++++++++++-------- io_uring/io_uring.h | 2 ++ 2 files changed, 21 insertions(+), 8 deletions(-)io_uring/io_uring.c +7
-2@@ -1241,9 +1241,10 @@ void tctx_task_work(struct callback_head *cb)WARN_ON_ONCE (ret); } -static inline void io_req_local_work_add(struct io_kiocb *req, unsigned tw_flags) +static inline void io_req_local_work_add(struct io_kiocb *req, + struct io_ring_ctx *ctx, + unsigned tw_flags) { -struct io_ring_ctx *ctx = req->ctx;unsigned nr_wait, nr_tw, nr_tw_prev;unsigned long flags; @@ -1291,9 +1292,10 @@ static inline void io_req_local_work_add(struct io_kiocb *req, unsigned tw_flags wake_up_state(ctx->submitter_task, TASK_INTERRUPTIBLE); } -static void io_req_normal_work_add(struct io_kiocb *req) +static void io_req_normal_work_add(struct io_kiocb *req, + struct task_struct *task) { -struct io_uring_task *tctx = req->task->io_uring; +struct io_uring_task *tctx = task->io_uring;struct io_ring_ctx *ctx = req->ctx;unsigned long flags;bool was_empty;@@ -1319,7 +1321,7 @@ static void io_req_normal_work_add(struct io_kiocb *req)return ; } - if (likely(!task_work_add(req->task, &tctx->task_work, ctx->notify_method))) +if (likely (!task_work_add (task, &tctx->task_work, ctx->notify_method)))return ;io_fallback_tw (tctx, false);@@ -1328,9 +1330,18 @@ static void io_req_normal_work_add(struct io_kiocb *req)void __io_req_task_work_add (struct io_kiocb *req,unsigned flags)io_uring/io_uring.c static void io_req_normal_work_add(struct io_kiocb *req) · line 1339
Quoted 10 lines
io_uring/io_uring.c +7
-1{if (req->ctx->flags &IORING_SETUP_DEFER_TASKRUN ) -io_req_local_work_add (req, flags); +io_req_local_work_add (req, req->ctx, flags); +else +io_req_normal_work_add (req, req->task); +} + +void io_req_task_work_add_remote (struct io_kiocb *req,struct task_struct *task, +struct io_ring_ctx *ctx,unsigned flags)Urgh, even the declration screams that there is something wrong considering it _either_ targets @ctx or @task.
Quoted 2 lines
Just pass @ctx, so it either use
ctx->submitter_taskorreq->task, hmm?
I actually since changed the above to use a common helper, so was scratching my head a bit over your comment as it can't really work in that setup without needing to check for whether ->submitter_task is set or not. But I do agree this would be nicer, so I'll just return to using the separate helpers for this and it should fall out nicely. The only odd caller is the MSG_RING side, so makes sense to have it a bit more separate rather than try and fold it in with the regular side of using task_work.
Quoted 9 lines
A side note, it's a dangerous game, I told it before. At least it would've been nice to abuse lockdep in a form of:
io_req_task_complete(req, tw, ctx) { lockdep_assert(req->ctx == ctx); ... }but we don't have @ctx there, maybe we'll add it to tw later.
Agree, but a separate thing imho.
Signature · 1 line
Jens Axboe
Re: [PATCH 1/3] io_uring: add remote task_work execution helper
Quoted 81 lines · Jens Axboe
Quoted 65 lines · Pavel Begunkov
Quoted 50 lines · Jens Axboe
All our
task_workhandling is targeted at the state in theio_kiocbitself, which is what it is being used for. However,MSG_RINGrolls its owntask_workhandling, ignoring how that is usually done.In preparation for switching
MSG_RINGto be able to use the normaltask_workhandling, addio_req_task_work_add_remote()which allows the caller to pass in the targetio_ring_ctxand task.Signed-off-by Jens Axboe <[email protected]>--- io_uring/io_uring.c | 27 +++++++++++++++++++-------- io_uring/io_uring.h | 2 ++ 2 files changed, 21 insertions(+), 8 deletions(-)io_uring/io_uring.c +7
-2@@ -1241,9 +1241,10 @@ void tctx_task_work(struct callback_head *cb)WARN_ON_ONCE (ret); } -static inline void io_req_local_work_add(struct io_kiocb *req, unsigned tw_flags) +static inline void io_req_local_work_add(struct io_kiocb *req, + struct io_ring_ctx *ctx, + unsigned tw_flags) { -struct io_ring_ctx *ctx = req->ctx;unsigned nr_wait, nr_tw, nr_tw_prev;unsigned long flags; @@ -1291,9 +1292,10 @@ static inline void io_req_local_work_add(struct io_kiocb *req, unsigned tw_flags wake_up_state(ctx->submitter_task, TASK_INTERRUPTIBLE); } -static void io_req_normal_work_add(struct io_kiocb *req) +static void io_req_normal_work_add(struct io_kiocb *req, + struct task_struct *task) { -struct io_uring_task *tctx = req->task->io_uring; +struct io_uring_task *tctx = task->io_uring;struct io_ring_ctx *ctx = req->ctx;unsigned long flags;bool was_empty;@@ -1319,7 +1321,7 @@ static void io_req_normal_work_add(struct io_kiocb *req)return ; } - if (likely(!task_work_add(req->task, &tctx->task_work, ctx->notify_method))) +if (likely (!task_work_add (task, &tctx->task_work, ctx->notify_method)))return ;io_fallback_tw (tctx, false);@@ -1328,9 +1330,18 @@ static void io_req_normal_work_add(struct io_kiocb *req)void __io_req_task_work_add (struct io_kiocb *req,unsigned flags)io_uring/io_uring.c static void io_req_normal_work_add(struct io_kiocb *req) · line 1339
Quoted 10 lines
io_uring/io_uring.c +7
-1{if (req->ctx->flags &IORING_SETUP_DEFER_TASKRUN ) -io_req_local_work_add (req, flags); +io_req_local_work_add (req, req->ctx, flags); +else +io_req_normal_work_add (req, req->task); +} + +void io_req_task_work_add_remote (struct io_kiocb *req,struct task_struct *task, +struct io_ring_ctx *ctx,unsigned flags)Urgh, even the declration screams that there is something wrong considering it _either_ targets @ctx or @task.
Quoted 2 lines
Just pass @ctx, so it either use
ctx->submitter_taskorreq->task, hmm?I actually since changed the above to use a common helper, so was scratching my head a bit over your comment as it can't really work in that setup without needing to check for whether ->submitter_task is set or not. But I do agree this would be nicer, so I'll just return to using the separate helpers for this and it should fall out nicely. The only odd caller is the
MSG_RINGside, so makes sense to have it a bit more separate rather than try and fold it in with the regular side of usingtask_work.Quoted 3 lines
A side note, it's a dangerous game, I told it before. At least it would've been nice to abuse lockdep in a form of:
Quoted 8 lines
Quoted 6 lines
io_req_task_complete(req, tw, ctx) { lockdep_assert(req->ctx == ctx); ... }but we don't have @ctx there, maybe we'll add it to tw later.
Agree, but a separate thing imho.
It's not in a sense that condition couldn't have happened before and the patch opening all possibilities.
We actually have @ctx via struct io_tctx_node, so considering fallback it would probably be:
lockdep_assert(!current->io_uring ||
current->io_uring->ctx == req->ctx);
Signature · 1 line
Pavel Begunkov
Jens Axboe
maintainerRe: [PATCH 1/3] io_uring: add remote task_work execution helper
Quoted 91 lines · Pavel Begunkov
Quoted 81 lines · Jens Axboe
Quoted 65 lines · Pavel Begunkov
Quoted 50 lines · Jens Axboe
All our
task_workhandling is targeted at the state in theio_kiocbitself, which is what it is being used for. However,MSG_RINGrolls its owntask_workhandling, ignoring how that is usually done.In preparation for switching
MSG_RINGto be able to use the normaltask_workhandling, addio_req_task_work_add_remote()which allows the caller to pass in the targetio_ring_ctxand task.Signed-off-by Jens Axboe <[email protected]>--- io_uring/io_uring.c | 27 +++++++++++++++++++-------- io_uring/io_uring.h | 2 ++ 2 files changed, 21 insertions(+), 8 deletions(-)io_uring/io_uring.c +7
-2@@ -1241,9 +1241,10 @@ void tctx_task_work(struct callback_head *cb)WARN_ON_ONCE (ret); } -static inline void io_req_local_work_add(struct io_kiocb *req, unsigned tw_flags) +static inline void io_req_local_work_add(struct io_kiocb *req, + struct io_ring_ctx *ctx, + unsigned tw_flags) { -struct io_ring_ctx *ctx = req->ctx;unsigned nr_wait, nr_tw, nr_tw_prev;unsigned long flags; @@ -1291,9 +1292,10 @@ static inline void io_req_local_work_add(struct io_kiocb *req, unsigned tw_flags wake_up_state(ctx->submitter_task, TASK_INTERRUPTIBLE); } -static void io_req_normal_work_add(struct io_kiocb *req) +static void io_req_normal_work_add(struct io_kiocb *req, + struct task_struct *task) { -struct io_uring_task *tctx = req->task->io_uring; +struct io_uring_task *tctx = task->io_uring;struct io_ring_ctx *ctx = req->ctx;unsigned long flags;bool was_empty;@@ -1319,7 +1321,7 @@ static void io_req_normal_work_add(struct io_kiocb *req)return ; } - if (likely(!task_work_add(req->task, &tctx->task_work, ctx->notify_method))) +if (likely (!task_work_add (task, &tctx->task_work, ctx->notify_method)))return ;io_fallback_tw (tctx, false);@@ -1328,9 +1330,18 @@ static void io_req_normal_work_add(struct io_kiocb *req)void __io_req_task_work_add (struct io_kiocb *req,unsigned flags)io_uring/io_uring.c static void io_req_normal_work_add(struct io_kiocb *req) · line 1339
Quoted 10 lines
io_uring/io_uring.c +7
-1{if (req->ctx->flags &IORING_SETUP_DEFER_TASKRUN ) -io_req_local_work_add (req, flags); +io_req_local_work_add (req, req->ctx, flags); +else +io_req_normal_work_add (req, req->task); +} + +void io_req_task_work_add_remote (struct io_kiocb *req,struct task_struct *task, +struct io_ring_ctx *ctx,unsigned flags)Urgh, even the declration screams that there is something wrong considering it _either_ targets @ctx or @task.
Quoted 2 lines
Just pass @ctx, so it either use
ctx->submitter_taskorreq->task, hmm?I actually since changed the above to use a common helper, so was scratching my head a bit over your comment as it can't really work in that setup without needing to check for whether ->submitter_task is set or not. But I do agree this would be nicer, so I'll just return to using the separate helpers for this and it should fall out nicely. The only odd caller is the
MSG_RINGside, so makes sense to have it a bit more separate rather than try and fold it in with the regular side of usingtask_work.Quoted 3 lines
A side note, it's a dangerous game, I told it before. At least it would've been nice to abuse lockdep in a form of:
Quoted 8 lines
Quoted 6 lines
io_req_task_complete(req, tw, ctx) { lockdep_assert(req->ctx == ctx); ... }but we don't have @ctx there, maybe we'll add it to tw later.
Agree, but a separate thing imho.
Quoted 8 lines
It's not in a sense that condition couldn't have happened before and the patch opening all possibilities.
We actually have @ctx via struct
io_tctx_node, so considering fallback it would probably be:lockdep_assert(!current->io_uring ||
current->io_uring->ctx == req->ctx);
That's not a bad idea. I did run all the testing verifying the ctx, and it all appears fine. But adding the check is a good idea in general. Want to send it?
Signature · 1 line
Jens Axboe
Jens Axboe
maintainerPATCH 2/3 io_uring/msg_ring: cleanup posting to IOPOLL vs !IOPOLL ring
In reply to Jens AxboeMove the posting outside the checking and locking, it's cleaner that way.
--- io_uring/msg_ring.c | 10 ++++------ 1 file changed, 4 insertions(+), 6 deletions(-)
io_uring/msg_ring.c +4 -6
@@ -147,13 +147,11 @@ static int io_msg_ring_data(struct io_kiocb *req, unsigned int issue_flags)if (target_ctx->flags &IORING_SETUP_IOPOLL ) {if (unlikely (io_double_lock_ctx (target_ctx, issue_flags)))return -EAGAIN ; -if (io_post_aux_cqe (target_ctx, msg->user_data, msg->len, flags)) - ret =0 ; -io_double_unlock_ctx (target_ctx); - }else { -if (io_post_aux_cqe (target_ctx, msg->user_data, msg->len, flags)) - ret =0 ; } +if (io_post_aux_cqe (target_ctx, msg->user_data, msg->len, flags)) + ret =0 ; +if (target_ctx->flags &IORING_SETUP_IOPOLL ) +io_double_unlock_ctx (target_ctx);return ret; }
Signature · 1 line
2.43.0
Re: [PATCH 2/3] io_uring/msg_ring: cleanup posting to IOPOLL vs !IOPOLL ring
Quoted 13 lines · Jens Axboe
Move the posting outside the checking and locking, it's cleaner that way.
Signed-off-by Jens Axboe <[email protected]>--- io_uring/msg_ring.c | 10 ++++------ 1 file changed, 4 insertions(+), 6 deletions(-)io_uring/msg_ring.c +0
-0
io_uring/msg_ring.c static int io_msg_ring_data(struct io_kiocb *req, unsigned int issue_flags) · line 150
Quoted 10 lines
io_uring/msg_ring.c +0
-6@@ -147,13 +147,11 @@ static int io_msg_ring_data(struct io_kiocb *req, unsigned int issue_flags)if (target_ctx->flags &IORING_SETUP_IOPOLL ) {if (unlikely (io_double_lock_ctx (target_ctx, issue_flags)))return -EAGAIN ; -if (io_post_aux_cqe (target_ctx, msg->user_data, msg->len, flags)) - ret =0 ; -io_double_unlock_ctx (target_ctx); - }else { -if (io_post_aux_cqe (target_ctx, msg->user_data, msg->len, flags)) - ret =0 ; }
A side note, maybe we should just get rid of double locking, it's always horrible, and always do the job via tw. With DEFER_TASKRUN it only benefits when rings and bound to the same task => never for any sane use case, so it's only about !DEFER_TASKRUN. Simpler but also more predictable for general latency and so on since you need to wait/grab two locks.
Quoted 7 lines
io_uring/msg_ring.c +4
-0+if (io_post_aux_cqe (target_ctx, msg->user_data, msg->len, flags)) + ret =0 ; +if (target_ctx->flags &IORING_SETUP_IOPOLL ) +io_double_unlock_ctx (target_ctx);return ret; }
Signature · 1 line
Pavel Begunkov
Jens Axboe
maintainerRe: [PATCH 2/3] io_uring/msg_ring: cleanup posting to IOPOLL vs !IOPOLL ring
Quoted 22 lines · Pavel Begunkov
Quoted 21 lines · Jens Axboe
Move the posting outside the checking and locking, it's cleaner that way.
Signed-off-by Jens Axboe <[email protected]>--- io_uring/msg_ring.c | 10 ++++------ 1 file changed, 4 insertions(+), 6 deletions(-)io_uring/msg_ring.c +0
-5@@ -147,13 +147,11 @@ static int io_msg_ring_data(struct io_kiocb *req, unsigned int issue_flags)if (target_ctx->flags &IORING_SETUP_IOPOLL ) {if (unlikely (io_double_lock_ctx (target_ctx, issue_flags)))return -EAGAIN ; -if (io_post_aux_cqe (target_ctx, msg->user_data, msg->len, flags)) - ret =0 ; -io_double_unlock_ctx (target_ctx); - }else { -if (io_post_aux_cqe (target_ctx, msg->user_data, msg->len, flags))
Quoted 8 lines
io_uring/msg_ring.c static int io_msg_ring_data(struct io_kiocb *req, unsigned int issue_flags) · line 150
Quoted 2 lines
io_uring/msg_ring.c +0
-1- ret =0 ; }A side note, maybe we should just get rid of double locking, it's always horrible, and always do the job via tw. With
DEFER_TASKRUNit only benefits when rings and bound to the same task => never for any sane use case, so it's only about !DEFER_TASKRUN. Simpler but also more predictable for general latency and so on since you need to wait/grab two locks.
It's not the prettiest, but at least for !DEFER_TASKRUN it's a LOT more efficient than punting through task_work... This is more of a case of DEFER_TASKRUN not being able to do this well, as we have strict requirements on CQE posting.
The function is a bit misnamed imho, as it's not double locking, it's just grabbing the target ctx lock. Should be io_lock_target_ctx() or something like that.
Signature · 1 line
Jens Axboe
Jens Axboe
maintainerPATCH 3/3 io_uring/msg_ring: improve handling of target CQE posting
In reply to Jens AxboeUse the exported helper for queueing task_work, rather than rolling our own.
This improves peak performance of message passing by about 5x in some basic testing, with 2 threads just sending messages to each other. Before this change, it was capped at around 700K/sec, with the change it's at over 4M/sec.
--- io_uring/msg_ring.c | 27 ++++++++++----------------- 1 file changed, 10 insertions(+), 17 deletions(-)
io_uring/msg_ring.c +10 -17
@@ -11,9 +11,9 @@#include "io_uring.h" #include "rsrc.h" #include "filetable.h" +#include "refs.h" #include "msg_ring.h" -/* All valid masks for MSG_RING */ #define IORING_MSG_RING_MASK (IORING_MSG_RING_CQE_SKIP | \ IORING_MSG_RING_FLAGS_PASS)
@@ -21,7 +21,6 @@struct io_msg {struct file *file;struct file *src_file; -struct callback_head tw;u64 user_data;u32 len;u32 cmd;
@@ -73,26 +72,20 @@ static inline bool io_msg_need_remote(struct io_ring_ctx *target_ctx)return current != target_ctx->submitter_task; } -static int io_msg_exec_remote (struct io_kiocb *req,task_work_func_t func) +static int io_msg_exec_remote (struct io_kiocb *req,io_req_tw_func_t func) {struct io_ring_ctx *ctx = req->file->private_data; - struct io_msg *msg = io_kiocb_to_cmd(req, struct io_msg);struct task_struct *task =READ_ONCE (ctx->submitter_task); -if (unlikely (!task)) -return -EOWNERDEAD ; - -init_task_work (&msg->tw, func); -if (task_work_add (ctx->submitter_task, &msg->tw,TWA_SIGNAL )) -return -EOWNERDEAD ; - +__io_req_set_refcount (req,2 ); + req->io_task_work.func = func; +io_req_task_work_add_remote (req, task, ctx,IOU_F_TWQ_LAZY_WAKE );return IOU_ISSUE_SKIP_COMPLETE ; } -static void io_msg_tw_complete (struct callback_head *head) +static void io_msg_tw_complete (struct io_kiocb *req,struct io_tw_state *ts) { - struct io_msg *msg = container_of(head, struct io_msg, tw); -struct io_kiocb *req =cmd_to_io_kiocb (msg); + struct io_msg *msg = io_kiocb_to_cmd(req, struct io_msg);struct io_ring_ctx *target_ctx = req->file->private_data;int ret =0 ;
@@ -120,6 +113,7 @@ static void io_msg_tw_complete(struct callback_head *head)if (ret <0 )req_set_fail (req); +req_ref_put_and_test (req);io_req_queue_tw_complete (req, ret); }
@@ -205,16 +199,15 @@ static int io_msg_install_complete(struct io_kiocb *req, unsigned int issue_flagreturn ret; } -static void io_msg_tw_fd_complete (struct callback_head *head) +static void io_msg_tw_fd_complete (struct io_kiocb *req,struct io_tw_state *ts) { - struct io_msg *msg = container_of(head, struct io_msg, tw); -struct io_kiocb *req =cmd_to_io_kiocb (msg);int ret = -EOWNERDEAD ;if (!(current->flags &PF_EXITING )) ret =io_msg_install_complete (req,IO_URING_F_UNLOCKED );if (ret <0 )req_set_fail (req); +req_ref_put_and_test (req);io_req_queue_tw_complete (req, ret); }
Signature · 1 line
2.43.0
Re: [PATCH 3/3] io_uring/msg_ring: improve handling of target CQE posting
Quoted 45 lines · Jens Axboe
Use the exported helper for queueing
task_work, rather than rolling our own.This improves peak performance of message passing by about 5x in some basic testing, with 2 threads just sending messages to each other. Before this change, it was capped at around 700K/sec, with the change it's at over 4M/sec.
Signed-off-by Jens Axboe <[email protected]>--- io_uring/msg_ring.c | 27 ++++++++++----------------- 1 file changed, 10 insertions(+), 17 deletions(-)io_uring/msg_ring.c +2
-4@@ -11,9 +11,9 @@#include "io_uring.h" #include "rsrc.h" #include "filetable.h" +#include "refs.h" #include "msg_ring.h" -/* All valid masks for MSG_RING */ #define IORING_MSG_RING_MASK (IORING_MSG_RING_CQE_SKIP | \ IORING_MSG_RING_FLAGS_PASS)@@ -21,7 +21,6 @@struct io_msg {struct file *file;struct file *src_file; -struct callback_head tw;u64 user_data;u32 len;u32 cmd;@@ -73,26 +72,20 @@ static inline bool io_msg_need_remote(struct io_ring_ctx *target_ctx)return current != target_ctx->submitter_task; } -static int io_msg_exec_remote (struct io_kiocb *req,task_work_func_t func) +static int io_msg_exec_remote (struct io_kiocb *req,io_req_tw_func_t func) {struct io_ring_ctx *ctx = req->file->private_data; - struct io_msg *msg = io_kiocb_to_cmd(req, struct io_msg);
io_uring/msg_ring.c static inline bool io_msg_need_remote(struct io_ring_ctx *target_ctx) · line 80
Quoted 10 lines
io_uring/msg_ring.c +1
-7struct task_struct *task =READ_ONCE (ctx->submitter_task); -if (unlikely (!task)) -return -EOWNERDEAD ; - -init_task_work (&msg->tw, func); -if (task_work_add (ctx->submitter_task, &msg->tw,TWA_SIGNAL )) -return -EOWNERDEAD ; - +__io_req_set_refcount (req,2 );
I'd argue it's better avoid any more req refcount users, I'd be more happy it it dies out completely at some point.
Why it's even needed here? You pass it via tw to post a CQE/etc and then pass it back via another tw hop to complete IIRC, the ownership is clear. At least it worth a comment.
Quoted 41 lines
io_uring/msg_ring.c +7
-6+ req->io_task_work.func = func; +io_req_task_work_add_remote (req, task, ctx,IOU_F_TWQ_LAZY_WAKE );return IOU_ISSUE_SKIP_COMPLETE ; } -static void io_msg_tw_complete (struct callback_head *head) +static void io_msg_tw_complete (struct io_kiocb *req,struct io_tw_state *ts) { - struct io_msg *msg = container_of(head, struct io_msg, tw); -struct io_kiocb *req =cmd_to_io_kiocb (msg); + struct io_msg *msg = io_kiocb_to_cmd(req, struct io_msg);struct io_ring_ctx *target_ctx = req->file->private_data;int ret =0 ;@@ -120,6 +113,7 @@ static void io_msg_tw_complete(struct callback_head *head)if (ret <0 )req_set_fail (req); +req_ref_put_and_test (req);io_req_queue_tw_complete (req, ret); }@@ -205,16 +199,15 @@ static int io_msg_install_complete(struct io_kiocb *req, unsigned int issue_flagreturn ret; } -static void io_msg_tw_fd_complete (struct callback_head *head) +static void io_msg_tw_fd_complete (struct io_kiocb *req,struct io_tw_state *ts) { - struct io_msg *msg = container_of(head, struct io_msg, tw); -struct io_kiocb *req =cmd_to_io_kiocb (msg);int ret = -EOWNERDEAD ;if (!(current->flags &PF_EXITING )) ret =io_msg_install_complete (req,IO_URING_F_UNLOCKED );if (ret <0 )req_set_fail (req); +req_ref_put_and_test (req);io_req_queue_tw_complete (req, ret); }
Signature · 1 line
Pavel Begunkov
Jens Axboe
maintainerRe: [PATCH 3/3] io_uring/msg_ring: improve handling of target CQE posting
Quoted 57 lines · Pavel Begunkov
Quoted 42 lines · Jens Axboe
Use the exported helper for queueing
task_work, rather than rolling our own.This improves peak performance of message passing by about 5x in some basic testing, with 2 threads just sending messages to each other. Before this change, it was capped at around 700K/sec, with the change it's at over 4M/sec.
Signed-off-by Jens Axboe <[email protected]>--- io_uring/msg_ring.c | 27 ++++++++++----------------- 1 file changed, 10 insertions(+), 17 deletions(-)io_uring/msg_ring.c +2
-1@@ -11,9 +11,9 @@#include "io_uring.h" #include "rsrc.h" #include "filetable.h" +#include "refs.h" #include "msg_ring.h" -/* All valid masks for MSG_RING */ #define IORING_MSG_RING_MASK (IORING_MSG_RING_CQE_SKIP | \ IORING_MSG_RING_FLAGS_PASS)@@ -21,7 +21,6 @@struct io_msg {struct file *file;struct file *src_file; -struct callback_head tw;u64 user_data;u32 len;u32 cmd;@@ -73,26 +72,20 @@ static inline bool io_msg_need_remote(struct io_ring_ctx *target_ctx)return current != target_ctx->submitter_task; } -static int io_msg_exec_remote(struct io_kiocb *req, task_work_func_t func) +static int io_msg_exec_remote(struct io_kiocb *req, io_req_tw_func_t func) {struct io_ring_ctx *ctx = req->file->private_data;io_uring/msg_ring.c static inline bool io_msg_need_remote(struct io_ring_ctx *target_ctx) · line 80
Quoted 10 lines
io_uring/msg_ring.c +1
-7- struct io_msg *msg = io_kiocb_to_cmd(req, struct io_msg);struct task_struct *task =READ_ONCE (ctx->submitter_task); - if (unlikely(!task)) - return -EOWNERDEAD; - -init_task_work (&msg->tw, func); -if (task_work_add (ctx->submitter_task, &msg->tw,TWA_SIGNAL )) -return -EOWNERDEAD ; - +__io_req_set_refcount (req,2 );I'd argue it's better avoid any more req refcount users, I'd be more happy it it dies out completely at some point.
Quoted 3 lines
Why it's even needed here? You pass it via tw to post a CQE/etc and then pass it back via another tw hop to complete IIRC, the ownership is clear. At least it worth a comment.
It's not, it was more documentation than anything else. But I agree that we should just avoid it, I'll kill it.
Signature · 1 line
Jens Axboe
Re: [PATCH 3/3] io_uring/msg_ring: improve handling of target CQE posting
Quoted 58 lines · Jens Axboe
Quoted 57 lines · Pavel Begunkov
Quoted 42 lines · Jens Axboe
Use the exported helper for queueing
task_work, rather than rolling our own.This improves peak performance of message passing by about 5x in some basic testing, with 2 threads just sending messages to each other. Before this change, it was capped at around 700K/sec, with the change it's at over 4M/sec.
Signed-off-by Jens Axboe <[email protected]>--- io_uring/msg_ring.c | 27 ++++++++++----------------- 1 file changed, 10 insertions(+), 17 deletions(-)io_uring/msg_ring.c +2
-1@@ -11,9 +11,9 @@#include "io_uring.h" #include "rsrc.h" #include "filetable.h" +#include "refs.h" #include "msg_ring.h" -/* All valid masks for MSG_RING */ #define IORING_MSG_RING_MASK (IORING_MSG_RING_CQE_SKIP | \ IORING_MSG_RING_FLAGS_PASS)@@ -21,7 +21,6 @@struct io_msg {struct file *file;struct file *src_file; -struct callback_head tw;u64 user_data;u32 len;u32 cmd;@@ -73,26 +72,20 @@ static inline bool io_msg_need_remote(struct io_ring_ctx *target_ctx)return current != target_ctx->submitter_task; } -static int io_msg_exec_remote(struct io_kiocb *req, task_work_func_t func) +static int io_msg_exec_remote(struct io_kiocb *req, io_req_tw_func_t func) {struct io_ring_ctx *ctx = req->file->private_data;io_uring/msg_ring.c static inline bool io_msg_need_remote(struct io_ring_ctx *target_ctx) · line 80
Quoted 10 lines
io_uring/msg_ring.c +1
-7- struct io_msg *msg = io_kiocb_to_cmd(req, struct io_msg);struct task_struct *task =READ_ONCE (ctx->submitter_task); - if (unlikely(!task)) - return -EOWNERDEAD; - -init_task_work (&msg->tw, func); -if (task_work_add (ctx->submitter_task, &msg->tw,TWA_SIGNAL )) -return -EOWNERDEAD ; - +__io_req_set_refcount (req,2 );I'd argue it's better avoid any more req refcount users, I'd be more happy it it dies out completely at some point.
Quoted 6 lines
Quoted 3 lines
Why it's even needed here? You pass it via tw to post a CQE/etc and then pass it back via another tw hop to complete IIRC, the ownership is clear. At least it worth a comment.
It's not, it was more documentation than anything else. But I agree that we should just avoid it, I'll kill it.
Great, it was confusing and I don't think it's even correct. In case it comes with refcounting enabled you'd get only 1 ref instead of desired 2. See how io_wq_submit_work() does it. Probably it's better to kill the "__" set refs helper.
Signature · 1 line
Pavel Begunkov
Jens Axboe
maintainerRe: [PATCH 3/3] io_uring/msg_ring: improve handling of target CQE posting
Quoted 63 lines · Pavel Begunkov
Quoted 62 lines · Jens Axboe
Quoted 60 lines · Pavel Begunkov
Quoted 42 lines · Jens Axboe
Use the exported helper for queueing
task_work, rather than rolling our own.This improves peak performance of message passing by about 5x in some basic testing, with 2 threads just sending messages to each other. Before this change, it was capped at around 700K/sec, with the change it's at over 4M/sec.
Signed-off-by Jens Axboe <[email protected]>--- io_uring/msg_ring.c | 27 ++++++++++----------------- 1 file changed, 10 insertions(+), 17 deletions(-)io_uring/msg_ring.c +2
-1@@ -11,9 +11,9 @@#include "io_uring.h" #include "rsrc.h" #include "filetable.h" +#include "refs.h" #include "msg_ring.h" -/* All valid masks for MSG_RING */ #define IORING_MSG_RING_MASK (IORING_MSG_RING_CQE_SKIP | \ IORING_MSG_RING_FLAGS_PASS)@@ -21,7 +21,6 @@struct io_msg {struct file *file;struct file *src_file; -struct callback_head tw;u64 user_data;u32 len;u32 cmd;@@ -73,26 +72,20 @@ static inline bool io_msg_need_remote(struct io_ring_ctx *target_ctx)return current != target_ctx->submitter_task; } -static int io_msg_exec_remote(struct io_kiocb *req, task_work_func_t func) +static int io_msg_exec_remote(struct io_kiocb *req, io_req_tw_func_t func) {struct io_ring_ctx *ctx = req->file->private_data;io_uring/msg_ring.c static inline bool io_msg_need_remote(struct io_ring_ctx *target_ctx) · line 80
Quoted 10 lines
io_uring/msg_ring.c +1
-7- struct io_msg *msg = io_kiocb_to_cmd(req, struct io_msg);struct task_struct *task =READ_ONCE (ctx->submitter_task); - if (unlikely(!task)) - return -EOWNERDEAD; - -init_task_work (&msg->tw, func); -if (task_work_add (ctx->submitter_task, &msg->tw,TWA_SIGNAL )) -return -EOWNERDEAD ; - +__io_req_set_refcount (req,2 );I'd argue it's better avoid any more req refcount users, I'd be more happy it it dies out completely at some point.
Why it's even needed here? You pass it via tw to post a CQE/etc and then pass it back via another tw hop to complete IIRC, the ownership is clear. At least it worth a comment.
Quoted 7 lines
Quoted 2 lines
It's not, it was more documentation than anything else. But I agree that we should just avoid it, I'll kill it.
Great, it was confusing and I don't think it's even correct. In case it comes with refcounting enabled you'd get only 1 ref instead of desired 2. See how
io_wq_submit_work()does it. Probably it's better to kill the "__" set refs helper.
Yeah, I think there's a bit of room for cleanups on the refs side. But thankfully it's not very prevalent in the code base.
Signature · 1 line
Jens Axboe