allocated requests based drain and fixes
v2 cover allocated requests based drain and fixes
Patches 1-7 are fixes and preparations. Patch 8 implements draining based on the number of allocated requests, which is simpler and remove cq_extra accounting overhead.
v2: fix executing still drained reqs on spurious calls to
io_queue_deferred() (Patch 2)
Only clear ->drain_active after flushing, if a non drain request got flushed.
Pavel Begunkov (8):
io_uring: account drain memory to cgroup
io_uring: fix spurious drain flushing
io_uring: simplify drain ret passing
io_uring: remove drain prealloc checks
io_uring: consolidate drain seq checking
io_uring: open code io_account_cq_overflow()
io_uring: count allocated requests
io_uring: drain based on allocates reqs
include/linux/io_uring_types.h | 3 +- io_uring/io_uring.c | 137 ++++++++++++++------------------- io_uring/io_uring.h | 3 +- 3 files changed, 62 insertions(+), 81 deletions(-)
Signature · 1 line
2.49.0
PATCH v2 1/8 io_uring: account drain memory to cgroup
Account drain allocations against memcg. It's not a big problem as each such allocation is paired with a request, which is accounted, but it's nicer to follow the limits more closely.
--- io_uring/io_uring.c | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-)
io_uring/io_uring.c +1 -1
@@ -1672,7 +1672,7 @@ static __cold void io_drain_req(struct io_kiocb *req)spin_unlock (&ctx->completion_lock);io_prep_async_link (req); - de =kmalloc (sizeof (*de),GFP_KERNEL ); + de =kmalloc (sizeof (*de),GFP_KERNEL_ACCOUNT );if (!de) { ret = -ENOMEM ;io_req_defer_failed (req, ret);
Signature · 1 line
2.49.0
PATCH v2 2/8 io_uring: fix spurious drain flushing
In reply to Pavel Begunkovio_queue_deferred() is not tolerant to spurious calls not completing some requests. You can have an inflight drain-marked request and another request that came after and got queued into the drain list. Now, if io_queue_deferred() is called before the first request completes, it'll check the 2nd req with req_need_defer(), find that there is no drain flag set, and queue it for execution.
To make io_queue_deferred() work, it should at least check sequences for the first request, and then we need also need to check if there is another drain request creating another bubble.
--- io_uring/io_uring.c | 14 +++++++++++++- 1 file changed, 13 insertions(+), 1 deletion(-)
io_uring/io_uring.c +13 -1
@@ -559,18 +559,30 @@ void io_req_queue_iowq(struct io_kiocb *req)io_req_task_work_add (req); } +static bool io_drain_defer_seq (struct io_kiocb *req,u32 seq) +{ +struct io_ring_ctx *ctx = req->ctx; + +return seq +READ_ONCE (ctx->cq_extra) != ctx->cached_cq_tail; +} + static __cold noinline void io_queue_deferred(struct io_ring_ctx *ctx) { +bool drain_seen = false, first = true; +spin_lock (&ctx->completion_lock);while (!list_empty (&ctx->defer_list)) {struct io_defer_entry *de =list_first_entry (&ctx->defer_list, struct io_defer_entry, list); -if (req_need_defer (de->req, de->seq)) + drain_seen |= de->req->flags &REQ_F_IO_DRAIN ; +if ((drain_seen || first) &&io_drain_defer_seq (de->req, de->seq))break ; +list_del_init (&de->list);io_req_task_queue (de->req);kfree (de); + first = false; }spin_unlock (&ctx->completion_lock); }
Signature · 1 line
2.49.0
PATCH v2 3/8 io_uring: simplify drain ret passing
In reply to Pavel Begunkov"ret" in io_drain_req() is only used in one place, remove it and pass -ENOMEM directly.
--- io_uring/io_uring.c | 4 +--- 1 file changed, 1 insertion(+), 3 deletions(-)
io_uring/io_uring.c +1 -3
@@ -1669,7 +1669,6 @@ static __cold void io_drain_req(struct io_kiocb *req) {struct io_ring_ctx *ctx = req->ctx;struct io_defer_entry *de; -int ret;u32 seq =io_get_sequence (req);/* Still need defer if there is pending req in defer list. */
@@ -1686,8 +1685,7 @@ static __cold void io_drain_req(struct io_kiocb *req)io_prep_async_link (req); de =kmalloc (sizeof (*de),GFP_KERNEL_ACCOUNT );if (!de) { - ret = -ENOMEM ; -io_req_defer_failed (req, ret); +io_req_defer_failed (req, -ENOMEM );return ; }
Signature · 1 line
2.49.0
PATCH v2 4/8 io_uring: remove drain prealloc checks
In reply to Pavel BegunkovCurrently io_drain_req() has two steps. The first is fast path checking sequence numbers. The second is allocations, rechecking and actual queuing. Further simplify it by removing the first step.
--- io_uring/io_uring.c | 15 +++------------ 1 file changed, 3 insertions(+), 12 deletions(-)
io_uring/io_uring.c +3 -12
@@ -1671,17 +1671,6 @@ static __cold void io_drain_req(struct io_kiocb *req)struct io_defer_entry *de;u32 seq =io_get_sequence (req); -/* Still need defer if there is pending req in defer list. */ -spin_lock (&ctx->completion_lock); -if (!req_need_defer (req, seq) &&list_empty_careful (&ctx->defer_list)) { -spin_unlock (&ctx->completion_lock); -queue: - ctx->drain_active = false; -io_req_task_queue (req); -return ; - } -spin_unlock (&ctx->completion_lock); -io_prep_async_link (req); de =kmalloc (sizeof (*de),GFP_KERNEL_ACCOUNT );if (!de) {
@@ -1693,7 +1682,9 @@ static __cold void io_drain_req(struct io_kiocb *req)if (!req_need_defer (req, seq) &&list_empty (&ctx->defer_list)) {spin_unlock (&ctx->completion_lock);kfree (de); - goto queue; + ctx->drain_active = false; +io_req_task_queue (req); +return ; }trace_io_uring_defer (req);
Signature · 1 line
2.49.0
PATCH v2 5/8 io_uring: consolidate drain seq checking
In reply to Pavel BegunkovWe check sequences when queuing drained requests as well when flushing them. Instead, always queue and immediately try to flush, so that all seq handling can be kept contained in the flushing code.
--- io_uring/io_uring.c | 45 +++++++++++++++++---------------------------- 1 file changed, 17 insertions(+), 28 deletions(-)
io_uring/io_uring.c +17 -28
@@ -389,17 +389,6 @@ static void io_account_cq_overflow(struct io_ring_ctx *ctx) ctx->cq_extra--; } -static bool req_need_defer (struct io_kiocb *req,u32 seq) -{ -if (unlikely (req->flags &REQ_F_IO_DRAIN )) { -struct io_ring_ctx *ctx = req->ctx; - -return seq +READ_ONCE (ctx->cq_extra) != ctx->cached_cq_tail; - } - -return false; -} -static void io_clean_op (struct io_kiocb *req) {if (unlikely (req->flags &REQ_F_BUFFER_SELECTED ))
@@ -566,11 +555,10 @@ static bool io_drain_defer_seq(struct io_kiocb *req, u32 seq)return seq +READ_ONCE (ctx->cq_extra) != ctx->cached_cq_tail; } -static __cold noinline void io_queue_deferred(struct io_ring_ctx *ctx) +static __cold noinline void __io_queue_deferred(struct io_ring_ctx *ctx) {bool drain_seen = false, first = true; -spin_lock (&ctx->completion_lock);while (!list_empty (&ctx->defer_list)) {struct io_defer_entry *de =list_first_entry (&ctx->defer_list, struct io_defer_entry, list);
@@ -584,7 +572,12 @@ static __cold noinline void io_queue_deferred(struct io_ring_ctx *ctx)kfree (de); first = false; } -spin_unlock (&ctx->completion_lock); +} + +static __cold noinline void io_queue_deferred(struct io_ring_ctx *ctx) +{ +guard (spinlock)(&ctx->completion_lock); +__io_queue_deferred (ctx); }void __io_commit_cqring_flush (struct io_ring_ctx *ctx)
@@ -1668,30 +1661,26 @@ static __cold void io_drain_req(struct io_kiocb *req) __must_hold(&ctx->uring_lock) {struct io_ring_ctx *ctx = req->ctx; +bool drain = req->flags &IOSQE_IO_DRAIN ;struct io_defer_entry *de; -u32 seq =io_get_sequence (req); -io_prep_async_link (req); de =kmalloc (sizeof (*de),GFP_KERNEL_ACCOUNT );if (!de) {io_req_defer_failed (req, -ENOMEM );return ; } -spin_lock (&ctx->completion_lock); -if (!req_need_defer (req, seq) &&list_empty (&ctx->defer_list)) { -spin_unlock (&ctx->completion_lock); -kfree (de); - ctx->drain_active = false; -io_req_task_queue (req); -return ; - } - +io_prep_async_link (req);trace_io_uring_defer (req); de->req = req; - de->seq = seq; -list_add_tail (&de->list, &ctx->defer_list); -spin_unlock (&ctx->completion_lock); + de->seq =io_get_sequence (req); + + scoped_guard(spinlock, &ctx->completion_lock) { +list_add_tail (&de->list, &ctx->defer_list); +__io_queue_deferred (ctx); +if (!drain &&list_empty (&ctx->defer_list)) + ctx->drain_active = false; + } } static bool io_assign_file(struct io_kiocb *req, const struct io_issue_def *def,
Signature · 1 line
2.49.0
PATCH v2 6/8 io_uring: open code io_account_cq_overflow()
In reply to Pavel Begunkovio_account_cq_overflow() doesn't help explaining what's going on in there, and it'll become even smaller with following patches, so open code it.
--- io_uring/io_uring.c | 13 ++++--------- 1 file changed, 4 insertions(+), 9 deletions(-)
io_uring/io_uring.c +4 -9
@@ -381,14 +381,6 @@ static __cold struct io_ring_ctx *io_ring_ctx_alloc(struct io_uring_params *p)return NULL ; } -static void io_account_cq_overflow (struct io_ring_ctx *ctx) -{ -struct io_rings *r = ctx->rings; - -WRITE_ONCE (r->cq_overflow,READ_ONCE (r->cq_overflow) +1 ); - ctx->cq_extra--; -} -static void io_clean_op (struct io_kiocb *req) {if (unlikely (req->flags &REQ_F_BUFFER_SELECTED ))
@@ -742,12 +734,15 @@ static bool io_cqring_event_overflow(struct io_ring_ctx *ctx, u64 user_data, ocqe =kmalloc (ocq_size,GFP_ATOMIC | __GFP_ACCOUNT);trace_io_uring_cqe_overflow (ctx, user_data, res, cflags, ocqe);if (!ocqe) { +struct io_rings *r = ctx->rings; +/* * If we're in ring overflow flush mode, or in task cancel mode, * or cannot allocate an overflow entry, then we need to drop it * on the floor. */ -io_account_cq_overflow (ctx); +WRITE_ONCE (r->cq_overflow,READ_ONCE (r->cq_overflow) +1 ); + ctx->cq_extra--;set_bit (IO_CHECK_CQ_DROPPED_BIT , &ctx->check_cq);return false; }
Signature · 1 line
2.49.0
PATCH v2 7/8 io_uring: count allocated requests
In reply to Pavel BegunkovKeep track of the number requests a ring currently has allocated (and not freed), it'll be needed in the next patch.
--- include/linux/io_uring_types.h | 1 + io_uring/io_uring.c | 9 ++++++++- 2 files changed, 9 insertions(+), 1 deletion(-)
include/linux/io_uring_types.h +1 -0
@@ -435,6 +435,7 @@ struct io_ring_ctx {/* protected by ->completion_lock */ unsigned evfd_last_cq_tail; +unsigned nr_req_allocated;/* * Protection for resize vs mmap races - both the mmap and resize
io_uring/io_uring.c +8 -1
@@ -954,6 +954,8 @@ __cold bool __io_alloc_req_refill(struct io_ring_ctx *ctx) }percpu_ref_get_many (&ctx->refs, ret); + ctx->nr_req_allocated += ret; +while (ret--) {struct io_kiocb *req = reqs[ret];
@@ -2691,8 +2693,10 @@ static void io_req_caches_free(struct io_ring_ctx *ctx)kmem_cache_free (req_cachep, req); nr++; } -if (nr) +if (nr) { + ctx->nr_req_allocated -= nr;percpu_ref_put_many (&ctx->refs, nr); + }mutex_unlock (&ctx->uring_lock); }
@@ -2729,6 +2733,9 @@ static __cold void io_ring_ctx_free(struct io_ring_ctx *ctx)percpu_ref_exit (&ctx->refs);free_uid (ctx->user);io_req_caches_free (ctx); + +WARN_ON_ONCE (ctx->nr_req_allocated); +if (ctx->hash_map)io_wq_put_hash (ctx->hash_map);io_napi_free (ctx);
Signature · 1 line
2.49.0
PATCH v2 8/8 io_uring: drain based on allocates reqs
In reply to Pavel BegunkovDon't rely on CQ sequence numbers for draining, as it has become messy and needs cq_extra adjustments. Instead, base it on the number of allocated requests and only allow flushing when all requests are in the drain list.
As a result, cq_extra is gone, no overhead for its accounting in aux cqe posting, less bloating as it was inlined before, and it's in general simpler than trying to track where we should bump it and where it should be put back like in cases of overflow. Also, it'll likely help with cleaning and unifying some of the CQ posting helpers.
--- include/linux/io_uring_types.h | 2 +- io_uring/io_uring.c | 83 +++++++++++++++------------------- io_uring/io_uring.h | 3 +- 3 files changed, 38 insertions(+), 50 deletions(-)
include/linux/io_uring_types.h +1 -1
@@ -341,7 +341,6 @@ struct io_ring_ctx {unsigned cached_cq_tail;unsigned cq_entries;struct io_ev_fd __rcu *io_ev_fd; -unsigned cq_extra;void *cq_wait_arg;size_t cq_wait_size;
@@ -417,6 +416,7 @@ struct io_ring_ctx {struct callback_head poll_wq_task_work;struct list_head defer_list; +unsigned nr_drained;struct io_alloc_cache msg_cache;spinlock_t msg_lock;
io_uring/io_uring.c +36 -47
214 lines not on this page · raw message
io_uring/io_uring.h +1 -2
@@ -196,7 +196,6 @@ static inline bool io_defer_get_uncommited_cqe(struct io_ring_ctx *ctx, {io_lockdep_assert_cq_locked (ctx); - ctx->cq_extra++; ctx->submit_state.cq_flush = true;return io_get_cqe (ctx, cqe_ret); }
@@ -414,7 +413,7 @@ static inline void io_req_complete_defer(struct io_kiocb *req)static inline void io_commit_cqring_flush (struct io_ring_ctx *ctx) { -if (unlikely (ctx->off_timeout_used || ctx->drain_active || +if (unlikely (ctx->off_timeout_used || ctx->has_evfd || ctx->poll_activated))__io_commit_cqring_flush (ctx); }
Signature · 1 line
2.49.0
Re: [PATCH v2 8/8] io_uring: drain based on allocates reqs
Quoted 10 lines · Pavel Begunkov
Don't rely on CQ sequence numbers for draining, as it has become messy and needs
cq_extraadjustments. Instead, base it on the number of allocated requests and only allow flushing when all requests are in the drain list.As a result,
cq_extrais gone, no overhead for its accounting in aux cqe posting, less bloating as it was inlined before, and it's in general simpler than trying to track where we should bump it and where it should be put back like in cases of overflow. Also, it'll likely help with cleaning and unifying some of the CQ posting helpers.
This patch breaks the make W=1 build. Please, always test your changes with make W=1. See below the details.
...
io_uring/io_uring.c io_req_flags_t io_file_get_flags(struct file *file)
Quoted 9 lines
static
__coldvoidio_drain_req(struct io_kiocb *req)io_uring/io_uring.c +2
-0__must_hold(&ctx->uring_lock) {struct io_ring_ctx *ctx = req->ctx;bool drain = req->flags &IOSQE_IO_DRAIN ;struct io_defer_entry *de; +struct io_kiocb *tmp; +int nr =0 ;
Defined and assigned...
io_uring/io_uring.c static __cold void io_drain_req(struct io_kiocb *req) · line 1662
Quoted 9 lines
io_uring/io_uring.c +2
-0de =kmalloc (sizeof (*de),GFP_KERNEL_ACCOUNT );if (!de) {@@ -1667,17 +1658,17 @@ static __cold void io_drain_req(struct io_kiocb *req)return ; } +io_for_each_link (tmp, req) + nr++;
...just incremented...
And that seems it. Does the above have any side-effects? Or is it just a dead code (a.k.a. leftovers from the rebase/upcoming work)?
In any case, please make use of nr or drop it completely.
io_uring/io_uring.c:1649:6: error: variable 'nr' set but not used [-Werror,-Wunused-but-set-variable]
1649 | int nr = 0;
| ^
1 error generated.
Quoted 17 lines
io_uring/io_uring.c +5
-7io_prep_async_link (req);trace_io_uring_defer (req); de->req = req; - de->seq =io_get_sequence (req); - scoped_guard(spinlock, &ctx->completion_lock) { -list_add_tail (&de->list, &ctx->defer_list); -__io_queue_deferred (ctx); -if (!drain &&list_empty (&ctx->defer_list)) - ctx->drain_active = false; - } + ctx->nr_drained +=io_linked_nr (req); +list_add_tail (&de->list, &ctx->defer_list); +io_queue_deferred (ctx); +if (!drain &&list_empty (&ctx->defer_list)) + ctx->drain_active = false; }
Signature · 2 lines
With Best Regards, Andy Shevchenko
Re: [PATCH v2 8/8] io_uring: drain based on allocates reqs
Quoted 6 lines · Andy Shevchenko
Quoted 5 lines · Pavel Begunkov
Don't rely on CQ sequence numbers for draining, as it has become messy and needs
cq_extraadjustments. Instead, base it on the number of allocated requests and only allow flushing when all requests are in the drain list.
Quoted 7 lines
Quoted 5 lines
As a result,
cq_extrais gone, no overhead for its accounting in aux cqe posting, less bloating as it was inlined before, and it's in general simpler than trying to track where we should bump it and where it should be put back like in cases of overflow. Also, it'll likely help with cleaning and unifying some of the CQ posting helpers.This patch breaks the
make W=1build. Please, always test your changes with
That's good advice, even if unsolicited, but it doesn't help with the issue. Not every compiler complain. I'd also say, things like that should warn by default, and they usually do, at least for basic "unused variable" cases.
Quoted 1 line
make W=1. See below the details.
Yes, it's dead code and has already been fixed a day ago.
Signature · 1 line
Pavel Begunkov
Quoted 8 lines · Pavel Begunkov
Patches 1-7 are fixes and preparations. Patch 8 implements draining based on the number of allocated requests, which is simpler and remove
cq_extraaccounting overhead.v2: fix executing still drained reqs on spurious calls to
io_queue_deferred()(Patch 2)[...]
Applied, thanks!
[1/8] io_uring: account drain memory to cgroup
commit: f979c20547e72568e3c793bc92c7522bc3166246
[2/8] io_uring: fix spurious drain flushing
commit: fde04c7e2775feb0746301e0ef86a04d3598c3fe
[3/8] io_uring: simplify drain ret passing
commit: 05b334110fdc85f536d7dd573120d573801fb2d1
[4/8] io_uring: remove drain prealloc checks
commit: e91e4f692f7993d5d192228c5f8a9a2e12ff5250
[5/8] io_uring: consolidate drain seq checking
commit: 19a94da447f832ee614f8f5532d31c1c70061520
[6/8] io_uring: open code io_account_cq_overflow()
commit: b0c8a6401fbca91da4fe0dc10d61a770f1581e45
[7/8] io_uring: count allocated requests
commit: 63de899cb6220357dea9d0f4e5aa459ff5193bb0
[8/8] io_uring: drain based on allocates reqs
commit: 55c28f431651973385c17081bdcdadf8b5c6da91Best regards,
Signature · 1 line
Jens Axboe