further optimise drain
cover further optimise drain
On top of "[PATCH 5.14 00/12] for-next optimisations"
The first two further optimise non-drain and rare-drain cases, so the overhead of it on the hot/generic path is minimal. With those, I'm more or less happy about draining.
3/3 inlines io_commit_cqring(). I didn't want to bloat the binary with it before, but now the hot path is shrinked enough.
Pavel Begunkov (3):
io_uring: switch !DRAIN fast path when possible
io_uring: shove more drain bits out of hot path
io_uring: optimise io_commit_cqring()
fs/io_uring.c | 68 ++++++++++++++++++++++++++++----------------------- 1 file changed, 38 insertions(+), 30 deletions(-)
Signature · 1 line
2.31.1
PATCH 1/3 io_uring: switch !DRAIN fast path when possible
->drain_used is one way, which is not optimal if users use DRAIN but very rarely. However, we can just clear it in io_drain_req() when all drained before requests are gone. Also rename the flag to reflect the change and be more clear about it.
--- fs/io_uring.c | 14 ++++++++------ 1 file changed, 8 insertions(+), 6 deletions(-)
fs/io_uring.c +8 -6
@@ -352,7 +352,7 @@ struct io_ring_ctx { unsigned int eventfd_async: 1; unsigned int restricted: 1; unsigned int off_timeout_used: 1; - unsigned int drain_used: 1; + unsigned int drain_active: 1; }____cacheline_aligned_in_smp ;/* submission data */
@@ -1346,10 +1346,10 @@ static void io_flush_timeouts(struct io_ring_ctx *ctx)static void io_commit_cqring (struct io_ring_ctx *ctx) { -if (unlikely (ctx->off_timeout_used || ctx->drain_used)) { +if (unlikely (ctx->off_timeout_used || ctx->drain_active)) {if (ctx->off_timeout_used)io_flush_timeouts (ctx); -if (ctx->drain_used) +if (ctx->drain_active)io_queue_deferred (ctx); }/* order cqe stores with ring update */
@@ -6004,8 +6004,10 @@ static bool io_drain_req(struct io_kiocb *req)/* Still need defer if there is pending req in defer list. */ if (likely (list_empty_careful (&ctx->defer_list) && - !(req->flags &REQ_F_IO_DRAIN ))) + !(req->flags &REQ_F_IO_DRAIN ))) { + ctx->drain_active = false;return false; + } seq =io_get_sequence (req);/* Still a chance to pass the sequence check */
@@ -6446,7 +6448,7 @@ static void __io_queue_sqe(struct io_kiocb *req)static inline void io_queue_sqe (struct io_kiocb *req) { -if (unlikely (req->ctx->drain_used) &&io_drain_req (req)) +if (unlikely (req->ctx->drain_active) &&io_drain_req (req))return ;if (likely (!(req->flags &REQ_F_FORCE_ASYNC ))) {
@@ -6572,7 +6574,7 @@ static int io_submit_sqe(struct io_ring_ctx *ctx, struct io_kiocb *req, }if (unlikely (req->flags &REQ_F_IO_DRAIN )) { - ctx->drain_used = true; + ctx->drain_active = true;/* * Taking sequential execution of a link, draining both sides
Signature · 1 line
2.31.1
PATCH 2/3 io_uring: shove more drain bits out of hot path
In reply to Pavel BegunkovPlace all drain_next logic into io_drain_req(), so it's never executed if there was no drained requests before. The only thing we need is to set ->drain_active if we see a request with IOSQE_IO_DRAIN, do that in io_init_req() where flags are definitely in registers.
Also, all drain-related code is encapsulated in io_drain_req(), makes it cleaner.
--- fs/io_uring.c | 42 ++++++++++++++++++++++-------------------- 1 file changed, 22 insertions(+), 20 deletions(-)
fs/io_uring.c +22 -20
@@ -5997,11 +5997,31 @@ static u32 io_get_sequence(struct io_kiocb *req)static bool io_drain_req (struct io_kiocb *req) { +struct io_kiocb *pos;struct io_ring_ctx *ctx = req->ctx;struct io_defer_entry *de;int ret;u32 seq; +/* +* If we need to drain a request in the middle of a link, drain the +* head request and the next request/link after the current link. +* Considering sequential execution of links, IOSQE_IO_DRAIN will be +* maintained for every request of our link. +*/ +if (ctx->drain_next) { + req->flags |=REQ_F_IO_DRAIN ; + ctx->drain_next = false; + } +/* not interested in head, start from the first linked */ +io_for_each_link (pos, req->link) { +if (pos->flags &REQ_F_IO_DRAIN ) { + ctx->drain_next = true; + req->flags |=REQ_F_IO_DRAIN ; +break ; + } + } +/* Still need defer if there is pending req in defer list. */ if (likely (list_empty_careful (&ctx->defer_list) && !(req->flags &REQ_F_IO_DRAIN ))) {
@@ -6522,6 +6542,8 @@ static int io_init_req(struct io_ring_ctx *ctx, struct io_kiocb *req,if ((sqe_flags &IOSQE_BUFFER_SELECT ) && !io_op_defs[req->opcode].buffer_select)return -EOPNOTSUPP ; +if (unlikely (sqe_flags &IOSQE_IO_DRAIN )) + ctx->drain_active = true; personality =READ_ONCE (sqe->personality);if (personality) {
@@ -6573,22 +6595,6 @@ static int io_submit_sqe(struct io_ring_ctx *ctx, struct io_kiocb *req,return ret; } -if (unlikely (req->flags &REQ_F_IO_DRAIN )) { - ctx->drain_active = true; - -/* -* Taking sequential execution of a link, draining both sides -* of the link also fullfils IOSQE_IO_DRAIN semantics for all -* requests in the link. So, it drains the head and the -* next after the link request. The last one is done via -* drain_next flag to persist the effect across calls. -*/ -if (link->head) { - link->head->flags |=REQ_F_IO_DRAIN ; - ctx->drain_next =1 ; - } - } - ret =io_req_prep (req, sqe);if (unlikely (ret)) goto fail_req;
@@ -6621,10 +6627,6 @@ static int io_submit_sqe(struct io_ring_ctx *ctx, struct io_kiocb *req,io_queue_sqe (head); } }else { -if (unlikely (ctx->drain_next)) { - req->flags |=REQ_F_IO_DRAIN ; - ctx->drain_next =0 ; - }if (req->flags & (REQ_F_LINK |REQ_F_HARDLINK )) { link->head = req; link->last = req;
Signature · 1 line
2.31.1
PATCH 3/3 io_uring: optimise io_commit_cqring()
In reply to Pavel BegunkovIn most cases io_commit_cqring() is just an smp_store_release(), and it's hot enough, especially for IRQ rw, to want it to save on a function call. Mark it inline and extract a non-inlined slow path doing drain and timeout flushing. The inlined part is pretty slim to not cause binary bloating.
--- fs/io_uring.c | 18 +++++++++++------- 1 file changed, 11 insertions(+), 7 deletions(-)
fs/io_uring.c +11 -7
@@ -1344,14 +1344,18 @@ static void io_flush_timeouts(struct io_ring_ctx *ctx) ctx->cq_last_tm_flush = seq; } -static void io_commit_cqring (struct io_ring_ctx *ctx) +static void __io_commit_cqring_flush (struct io_ring_ctx *ctx) { -if (unlikely (ctx->off_timeout_used || ctx->drain_active)) { -if (ctx->off_timeout_used) -io_flush_timeouts (ctx); -if (ctx->drain_active) -io_queue_deferred (ctx); - } +if (ctx->off_timeout_used) +io_flush_timeouts (ctx); +if (ctx->drain_active) +io_queue_deferred (ctx); +} + +static inline void io_commit_cqring (struct io_ring_ctx *ctx) +{ +if (unlikely (ctx->off_timeout_used || ctx->drain_active)) +__io_commit_cqring_flush (ctx);/* order cqe stores with ring update */ smp_store_release (&ctx->rings->cq.tail, ctx->cached_cq_tail); }
Signature · 1 line
2.31.1
Quoted 5 lines · Pavel Begunkov
On top of "[PATCH 5.14 00/12] for-next optimisations"
The first two further optimise non-drain and rare-drain cases, so the overhead of it on the hot/generic path is minimal. With those, I'm more or less happy about draining.
Applied, thanks.
Signature · 1 line
Jens Axboe