cq wait refactoring rebase
cover cq wait refactoring rebase
Rebase of 6.3, i.e. recent CQ waiting refactoring series on top of just sent 6.2 patch ("io_uring: fix CQ waiting timeout handling").
Apart from that there are 2 more patches on top, 9/10 squeezes an extra 1% of perf for one of my tests.
Pavel Begunkov (10):
io_uring: rearrange defer list checks
io_uring: don't iterate cq wait fast path
io_uring: kill io_run_task_work_ctx
io_uring: move defer tw task checks
io_uring: parse check_cq out of wq waiting
io_uring: mimimise io_cqring_wait_schedule
io_uring: simplify io_has_work
io_uring: set TASK_RUNNING right after schedule
io_uring: optimise non-timeout waiting
io_uring: keep timeout in io_wait_queue
io_uring/io_uring.c | 137 +++++++++++++++++++++++--------------------- io_uring/io_uring.h | 25 ++------ 2 files changed, 77 insertions(+), 85 deletions(-)
Signature · 1 line
2.38.1
PATCH 3/10 io_uring: kill io_run_task_work_ctx
There is only one user of io_run_task_work_ctx(), inline it.
--- io_uring/io_uring.c | 6 +++++- io_uring/io_uring.h | 20 -------------------- 2 files changed, 5 insertions(+), 21 deletions(-)
io_uring/io_uring.c +5 -1
@@ -2452,7 +2452,11 @@ static int io_wake_function(struct wait_queue_entry *curr, unsigned int mode,int io_run_task_work_sig (struct io_ring_ctx *ctx) { -if (io_run_task_work_ctx (ctx) >0 ) +if (!llist_empty (&ctx->work_llist)) { +if (io_run_local_work (ctx) >0 ) +return 1 ; + } +if (io_run_task_work () >0 )return 1 ;if (task_sigpending (current))return -EINTR ;
io_uring/io_uring.h +0 -20
@@ -269,26 +269,6 @@ static inline bool io_task_work_pending(struct io_ring_ctx *ctx)return task_work_pending (current) || !wq_list_empty (&ctx->work_llist); } -static inline int io_run_task_work_ctx (struct io_ring_ctx *ctx) -{ -int ret =0 ; -int ret2; - -if (!llist_empty (&ctx->work_llist)) - ret =io_run_local_work (ctx); - -/* want to run this after in case more is added */ - ret2 =io_run_task_work (); - -/* Try propagate error in favour of if tasks were run, -* but still make sure to run them if requested -*/ -if (ret >=0 ) - ret += ret2; - -return ret; -} -static inline int io_run_local_work_locked (struct io_ring_ctx *ctx) {bool locked;
Signature · 1 line
2.38.1
PATCH 2/10 io_uring: don't iterate cq wait fast path
In reply to Pavel BegunkovTask work runners keep running until all queues tw items are exhausted. It's also rare for defer tw to queue normal tw and vise versa. Taking it into account, there is only a dim chance that further iterating the io_cqring_wait() fast path will get us anything and so we can remove the loop there.
--- io_uring/io_uring.c | 18 ++++++++---------- 1 file changed, 8 insertions(+), 10 deletions(-)
io_uring/io_uring.c +8 -10
@@ -2507,18 +2507,16 @@ static int io_cqring_wait(struct io_ring_ctx *ctx, int min_events,if (!io_allowed_run_tw (ctx))return -EEXIST ; - -do { -/* always run at least 1 task work to process local work */ - ret =io_run_task_work_ctx (ctx); +if (!llist_empty (&ctx->work_llist)) { + ret =io_run_local_work (ctx);if (ret <0 )return ret; -io_cqring_overflow_flush (ctx); - -/* if user messes with these they will just get an early return */ -if (__io_cqring_events_user (ctx) >= min_events) -return 0 ; - }while (ret >0 ); + } +io_run_task_work (); +io_cqring_overflow_flush (ctx); +/* if user messes with these they will just get an early return */ +if (__io_cqring_events_user (ctx) >= min_events) +return 0 ;if (sig) {#ifdef CONFIG_COMPAT
Signature · 1 line
2.38.1
PATCH 7/10 io_uring: simplify io_has_work
In reply to Pavel Begunkov->work_llist should never be non-empty for a non DEFER_TASKRUN ring, so we can safely skip checking the flag.
--- io_uring/io_uring.c | 3 +-- 1 file changed, 1 insertion(+), 2 deletions(-)
io_uring/io_uring.c +1 -2
@@ -2416,8 +2416,7 @@ struct io_wait_queue {static inline bool io_has_work (struct io_ring_ctx *ctx) {return test_bit (IO_CHECK_CQ_OVERFLOW_BIT , &ctx->check_cq) || - ((ctx->flags &IORING_SETUP_DEFER_TASKRUN ) && - !llist_empty (&ctx->work_llist)); + !llist_empty (&ctx->work_llist); }static inline bool io_should_wake (struct io_wait_queue *iowq)
Signature · 1 line
2.38.1
PATCH 1/10 io_uring: rearrange defer list checks
In reply to Pavel BegunkovThere should be nothing in the ->work_llist for non DEFER_TASKRUN rings, so we can skip flag checks and test the list emptiness directly. Also move it out of io_run_local_work() for inlining.
--- io_uring/io_uring.c | 3 --- io_uring/io_uring.h | 2 +- 2 files changed, 1 insertion(+), 4 deletions(-)
io_uring/io_uring.c +0 -3
@@ -1338,9 +1338,6 @@ int io_run_local_work(struct io_ring_ctx *ctx)bool locked;int ret; -if (llist_empty (&ctx->work_llist)) -return 0 ; -__set_current_state (TASK_RUNNING ); locked =mutex_trylock (&ctx->uring_lock); ret =__io_run_local_work (ctx, &locked);
io_uring/io_uring.h +1 -1
@@ -274,7 +274,7 @@ static inline int io_run_task_work_ctx(struct io_ring_ctx *ctx)int ret =0 ;int ret2; -if (ctx->flags &IORING_SETUP_DEFER_TASKRUN ) +if (!llist_empty (&ctx->work_llist)) ret =io_run_local_work (ctx);/* want to run this after in case more is added */
Signature · 1 line
2.38.1
PATCH 5/10 io_uring: parse check_cq out of wq waiting
In reply to Pavel BegunkovWe already avoid flushing overflows in io_cqring_wait_schedule() but only return an error for the outer loop to handle it. Minimise it even further by moving all ->check_cq parsing there.
--- io_uring/io_uring.c | 32 ++++++++++++++++++-------------- 1 file changed, 18 insertions(+), 14 deletions(-)
io_uring/io_uring.c +18 -14
@@ -2468,21 +2468,13 @@ static inline int io_cqring_wait_schedule(struct io_ring_ctx *ctx, ktime_t *timeout) {int ret; -unsigned long check_cq; +if (unlikely (READ_ONCE (ctx->check_cq))) +return 1 ;/* make sure we run task_work before checking for signals */ ret =io_run_task_work_sig (ctx);if (ret ||io_should_wake (iowq))return ret; - - check_cq =READ_ONCE (ctx->check_cq); -if (unlikely (check_cq)) { -/* let the caller flush overflows, retry */ -if (check_cq &BIT (IO_CHECK_CQ_OVERFLOW_BIT )) -return 1 ; -if (check_cq &BIT (IO_CHECK_CQ_DROPPED_BIT )) -return -EBADR ; - }if (!schedule_hrtimeout (timeout,HRTIMER_MODE_ABS ))return -ETIME ;
@@ -2548,13 +2540,25 @@ static int io_cqring_wait(struct io_ring_ctx *ctx, int min_events,trace_io_uring_cqring_wait (ctx, min_events); do { - if (test_bit(IO_CHECK_CQ_OVERFLOW_BIT, &ctx->check_cq)) { - finish_wait(&ctx->cq_wait, &iowq.wq); - io_cqring_do_overflow_flush(ctx); - } + unsigned long check_cq; + prepare_to_wait_exclusive(&ctx->cq_wait, &iowq.wq, TASK_INTERRUPTIBLE); ret = io_cqring_wait_schedule(ctx, &iowq, &timeout); + + check_cq = READ_ONCE(ctx->check_cq); + if (unlikely(check_cq)) { + /* let the caller flush overflows, retry */ + if (check_cq & BIT(IO_CHECK_CQ_OVERFLOW_BIT)) { + finish_wait(&ctx->cq_wait, &iowq.wq); + io_cqring_do_overflow_flush(ctx); + } + if (check_cq & BIT(IO_CHECK_CQ_DROPPED_BIT)) { + ret = -EBADR; + break; + } + } + if (__io_cqring_events_user(ctx) >= min_events) break; cond_resched();
Signature · 1 line
2.38.1
PATCH 4/10 io_uring: move defer tw task checks
In reply to Pavel BegunkovMost places that want to run local tw explicitly and in advance check if they are allowed to do so. Don't rely on a similar check in __io_run_local_work(), leave it as a just-in-case warning and make sure callers checks capabilities themselves.
--- io_uring/io_uring.c | 15 ++++++--------- io_uring/io_uring.h | 5 +++++ 2 files changed, 11 insertions(+), 9 deletions(-)
io_uring/io_uring.c +6 -9
@@ -1296,14 +1296,13 @@ int __io_run_local_work(struct io_ring_ctx *ctx, bool *locked)struct llist_node *node;struct llist_node fake;struct llist_node *current_final =NULL ; -int ret; +int ret =0 ;unsigned int loops =1 ; -if (unlikely (ctx->submitter_task != current)) +if (WARN_ON_ONCE (ctx->submitter_task != current))return -EEXIST ; node =io_llist_xchg (&ctx->work_llist, &fake); - ret =0 ; again:while (node != current_final) {struct llist_node *next = node->next;
@@ -2511,11 +2510,8 @@ static int io_cqring_wait(struct io_ring_ctx *ctx, int min_events,if (!io_allowed_run_tw (ctx))return -EEXIST ; -if (!llist_empty (&ctx->work_llist)) { - ret =io_run_local_work (ctx); -if (ret <0 ) -return ret; - } +if (!llist_empty (&ctx->work_llist)) +io_run_local_work (ctx);io_run_task_work ();io_cqring_overflow_flush (ctx);/* if user messes with these they will just get an early return */
@@ -3052,7 +3048,8 @@ static __cold bool io_uring_try_cancel_requests(struct io_ring_ctx *ctx, } } -if (ctx->flags &IORING_SETUP_DEFER_TASKRUN ) +if ((ctx->flags &IORING_SETUP_DEFER_TASKRUN ) && +io_allowed_defer_tw_run (ctx)) ret |=io_run_local_work (ctx) >0 ; ret |=io_cancel_defer_files (ctx, task, cancel_all);mutex_lock (&ctx->uring_lock);
io_uring/io_uring.h +5 -0
@@ -352,6 +352,11 @@ static inline struct io_kiocb *io_alloc_req(struct io_ring_ctx *ctx) return container_of(node, struct io_kiocb, comp_list); } +static inline bool io_allowed_defer_tw_run (struct io_ring_ctx *ctx) +{ +return likely (ctx->submitter_task == current); +} +static inline bool io_allowed_run_tw (struct io_ring_ctx *ctx) { return likely(!(ctx->flags & IORING_SETUP_DEFER_TASKRUN) ||
Signature · 1 line
2.38.1
PATCH 8/10 io_uring: set TASK_RUNNING right after schedule
In reply to Pavel BegunkovInstead of constantly watching that the state of the task is running before executing tw or taking locks in io_cqring_wait(), switch it back to TASK_RUNNING immediately.
--- io_uring/io_uring.c | 5 ++--- 1 file changed, 2 insertions(+), 3 deletions(-)
io_uring/io_uring.c +2 -3
@@ -2541,6 +2541,7 @@ static int io_cqring_wait(struct io_ring_ctx *ctx, int min_events, ret =io_cqring_wait_schedule (ctx, &iowq, &timeout);if (ret <0 )break ; +__set_current_state (TASK_RUNNING );/* * Run task_work after scheduling and before io_should_wake(). * If we got woken because of task_work being processed, run it
@@ -2553,10 +2554,8 @@ static int io_cqring_wait(struct io_ring_ctx *ctx, int min_events, check_cq =READ_ONCE (ctx->check_cq);if (unlikely (check_cq)) {/* let the caller flush overflows, retry */ -if (check_cq &BIT (IO_CHECK_CQ_OVERFLOW_BIT )) { -finish_wait (&ctx->cq_wait, &iowq.wq); +if (check_cq &BIT (IO_CHECK_CQ_OVERFLOW_BIT ))io_cqring_do_overflow_flush (ctx); - }if (check_cq &BIT (IO_CHECK_CQ_DROPPED_BIT )) { ret = -EBADR ;break ;
Signature · 1 line
2.38.1
PATCH 9/10 io_uring: optimise non-timeout waiting
In reply to Pavel BegunkovUnlike the jiffy scheduling version, schedule_hrtimeout() jumps a few functions before getting into schedule() even if there is no actual timeout needed. Some tests showed that it takes up to 1% of CPU cycles.
--- io_uring/io_uring.c | 4 +++- 1 file changed, 3 insertions(+), 1 deletion(-)
io_uring/io_uring.c +3 -1
@@ -2476,7 +2476,9 @@ static inline int io_cqring_wait_schedule(struct io_ring_ctx *ctx,return -EINTR ;if (unlikely (io_should_wake (iowq)))return 0 ; -if (!schedule_hrtimeout (timeout,HRTIMER_MODE_ABS )) +if (*timeout ==KTIME_MAX ) +schedule (); +else if (!schedule_hrtimeout (timeout,HRTIMER_MODE_ABS ))return -ETIME ;return 0 ; }
Signature · 1 line
2.38.1
PATCH 6/10 io_uring: mimimise io_cqring_wait_schedule
In reply to Pavel Begunkovio_cqring_wait_schedule() is called after we started waiting on the cq wq and set the state to TASK_INTERRUPTIBLE, for that reason we have to constantly worry whether we has returned the state back to running or not. Leave only quick checks in io_cqring_wait_schedule() and move the rest including running task work to the callers. Note, we run tw in the loop after the sched checks because of the fast path in the beginning of the function.
--- io_uring/io_uring.c | 39 +++++++++++++++++++++++---------------- 1 file changed, 23 insertions(+), 16 deletions(-)
io_uring/io_uring.c +23 -16
@@ -2467,24 +2467,19 @@ static inline int io_cqring_wait_schedule(struct io_ring_ctx *ctx, struct io_wait_queue *iowq, ktime_t *timeout) { - int ret; - if (unlikely(READ_ONCE(ctx->check_cq))) return 1; -/* make sure we run task_work before checking for signals */ - ret =io_run_task_work_sig (ctx); -if (ret ||io_should_wake (iowq)) -return ret; + if (unlikely(!llist_empty(&ctx->work_llist))) + return 1; + if (unlikely(test_thread_flag(TIF_NOTIFY_SIGNAL))) + return 1; + if (unlikely(task_sigpending(current))) + return -EINTR; + if (unlikely(io_should_wake(iowq))) + return 0; if (!schedule_hrtimeout(timeout, HRTIMER_MODE_ABS)) return -ETIME; - -/* -* Run task_work after scheduling. If we got woken because of -* task_work being processed, run it now rather than let the caller -* do another wait loop. -*/ - ret =io_run_task_work_sig (ctx); -return ret <0 ? ret :1 ; + return 0; }/*
@@ -2545,6 +2540,16 @@ static int io_cqring_wait(struct io_ring_ctx *ctx, int min_events,prepare_to_wait_exclusive (&ctx->cq_wait, &iowq.wq,TASK_INTERRUPTIBLE ); ret =io_cqring_wait_schedule (ctx, &iowq, &timeout); +if (ret <0 ) +break ; +/* +* Run task_work after scheduling and before io_should_wake(). +* If we got woken because of task_work being processed, run it +* now rather than let the caller do another wait loop. +*/ +io_run_task_work (); +if (!llist_empty (&ctx->work_llist)) +io_run_local_work (ctx); check_cq =READ_ONCE (ctx->check_cq);if (unlikely (check_cq)) {
@@ -2559,10 +2564,12 @@ static int io_cqring_wait(struct io_ring_ctx *ctx, int min_events, } } -if (__io_cqring_events_user (ctx) >= min_events) +if (io_should_wake (&iowq)) { + ret =0 ;break ; + }cond_resched (); - }while (ret >0 ); + }while (1 );finish_wait (&ctx->cq_wait, &iowq.wq);restore_saved_sigmask_unless (ret == -EINTR );
Signature · 1 line
2.38.1
PATCH 10/10 io_uring: keep timeout in io_wait_queue
In reply to Pavel BegunkovMove waiting timeout into io_wait_queue
--- io_uring/io_uring.c | 28 ++++++++++++++-------------- 1 file changed, 14 insertions(+), 14 deletions(-)
io_uring/io_uring.c +14 -14
@@ -2411,6 +2411,7 @@ struct io_wait_queue {struct io_ring_ctx *ctx;unsigned cq_tail;unsigned nr_timeouts; +ktime_t timeout; };static inline bool io_has_work (struct io_ring_ctx *ctx)
@@ -2463,8 +2464,7 @@ int io_run_task_work_sig(struct io_ring_ctx *ctx)/* when returns >0, the caller should retry */ static inline int io_cqring_wait_schedule (struct io_ring_ctx *ctx, -struct io_wait_queue *iowq, -ktime_t *timeout) +struct io_wait_queue *iowq) {if (unlikely (READ_ONCE (ctx->check_cq)))return 1 ;
@@ -2476,9 +2476,9 @@ static inline int io_cqring_wait_schedule(struct io_ring_ctx *ctx,return -EINTR ;if (unlikely (io_should_wake (iowq)))return 0 ; -if (*timeout ==KTIME_MAX ) +if (iowq->timeout ==KTIME_MAX )schedule (); -else if (!schedule_hrtimeout (timeout,HRTIMER_MODE_ABS )) +else if (!schedule_hrtimeout (&iowq->timeout,HRTIMER_MODE_ABS ))return -ETIME ;return 0 ; }
@@ -2493,7 +2493,6 @@ static int io_cqring_wait(struct io_ring_ctx *ctx, int min_events, {struct io_wait_queue iowq;struct io_rings *rings = ctx->rings; -ktime_t timeout =KTIME_MAX ;int ret;if (!io_allowed_run_tw (ctx))
@@ -2519,20 +2518,21 @@ static int io_cqring_wait(struct io_ring_ctx *ctx, int min_events,return ret; } -if (uts) { -struct timespec64 ts; - -if (get_timespec64 (&ts, uts)) -return -EFAULT ; - timeout =ktime_add_ns (timespec64_to_ktime (ts),ktime_get_ns ()); - } -init_waitqueue_func_entry (&iowq.wq, io_wake_function); iowq.wq.private = current;INIT_LIST_HEAD (&iowq.wq.entry); iowq.ctx = ctx; iowq.nr_timeouts =atomic_read (&ctx->cq_timeouts); iowq.cq_tail =READ_ONCE (ctx->rings->cq.head) + min_events; + iowq.timeout =KTIME_MAX ; + +if (uts) { +struct timespec64 ts; + +if (get_timespec64 (&ts, uts)) +return -EFAULT ; + iowq.timeout =ktime_add_ns (timespec64_to_ktime (ts),ktime_get_ns ()); + }trace_io_uring_cqring_wait (ctx, min_events); do {
@@ -2540,7 +2540,7 @@ static int io_cqring_wait(struct io_ring_ctx *ctx, int min_events,prepare_to_wait_exclusive (&ctx->cq_wait, &iowq.wq,TASK_INTERRUPTIBLE ); - ret =io_cqring_wait_schedule (ctx, &iowq, &timeout); + ret =io_cqring_wait_schedule (ctx, &iowq);if (ret <0 )break ;__set_current_state (TASK_RUNNING );
Signature · 1 line
2.38.1
Quoted 18 lines · Pavel Begunkov
Rebase of 6.3, i.e. recent CQ waiting refactoring series on top of just sent 6.2 patch ("
io_uring: fix CQ waiting timeout handling").Apart from that there are 2 more patches on top, 9/10 squeezes an extra 1% of perf for one of my tests.
Pavel Begunkov (10):
io_uring: rearrange defer list checks
io_uring: don't iterate cq wait fast path
io_uring: killio_run_task_work_ctx
io_uring: move defer tw task checks
io_uring: parsecheck_cqout of wq waiting
io_uring: mimimiseio_cqring_wait_schedule
io_uring: simplifyio_has_work
io_uring: setTASK_RUNNINGright after schedule
io_uring: optimise non-timeout waiting
io_uring: keep timeout inio_wait_queue
Quoted 1 line
[...]
Applied, thanks!
[01/10] io_uring: rearrange defer list checks
commit: d6d505fb8cafd1d78ca438fcd2d9a460c9695a1a
[02/10] io_uring: don't iterate cq wait fast path
commit: ba5636ba85eb7b01bcbb03b8855ec444f78779ed
[03/10] io_uring: kill io_run_task_work_ctx
commit: a95f8ea66f4804c9faeb51bf496008d09d43bf41
[04/10] io_uring: move defer tw task checks
commit: 07ef9bf92577095d2e6612b010267f6bd5139936
[05/10] io_uring: parse check_cq out of wq waiting
commit: 4a42b391367e7df55f8fd5f7a4e42cdf796ef36a
[06/10] io_uring: mimimise io_cqring_wait_schedule
commit: 4fbb75ef0d9ed027d098394227596d91c59e246a
[07/10] io_uring: simplify io_has_work
commit: 8d1f973f5025d62f88a9b3e4406c74c857ef49de
[08/10] io_uring: set TASK_RUNNING right after schedule
commit: 2fc7bfa59e195d2a1fed638a35dde7ab794f0178
[09/10] io_uring: optimise non-timeout waiting
commit: dc53cdbd5a63ef78c1acc34b2fda535bbcffd5c5
[10/10] io_uring: keep timeout in io_wait_queue
commit: a367ffbe03a6da8d97f5f8057114bada90ccaea8Best regards,
Signature · 1 line
Jens Axboe