task work cleanups
Jens Axboe
maintainercover task work cleanups
Hi,
This patchset gets rid of using llist for handling task_work, for both local and normal task_work. Instead of a lockless list, a normal io_wq_work_list is used and protected by a spinlock. I've done some benchmarking with this, and I only see wins with this - the act of adding or iterating task_work is the same cost, but we get rid of the need to reverse the task_work list, which can be substantial for bursty applications or just generally busy task_work usages.
Patch 2 implements io_wq_work_list handling for deferred task_work, and patch 4 does the same for normal task_work. Patch 6 then also switches SQPOLL to use this scheme, which eliminates the passing around of io_wq_work_node for its retry logic.
Outside of cleaning up this code, it also enables us to potentially implement task_work run capping for normal task_work in the future.
Git tree can be found here:
https://git.kernel.dk/cgit/linux/log/?h=io_uring-defer-tw
include/linux/io_uring_types.h | 17 +- io_uring/io_uring.c | 293 ++++++++++++++++++--------------- io_uring/io_uring.h | 20 ++- io_uring/slist.h | 16 ++ io_uring/sqpoll.c | 20 ++- io_uring/tctx.c | 3 +- 6 files changed, 211 insertions(+), 158 deletions(-)
Signature · 1 line
Jens Axboe
Jens Axboe
maintainerPATCH 1/6 io_uring: make task_work pending check dependent on ring type
There's no need to check for generic task_work for DEFER_TASKRUN, if we have local task_work pending. This avoids dipping into the huge task_struct, if we have normal task_work pending.
--- io_uring/io_uring.h | 4 +++- 1 file changed, 3 insertions(+), 1 deletion(-)
io_uring/io_uring.h +3 -1
@@ -354,7 +354,9 @@ static inline bool io_local_work_pending(struct io_ring_ctx *ctx)static inline bool io_task_work_pending (struct io_ring_ctx *ctx) { -return task_work_pending (current) ||io_local_work_pending (ctx); +if (ctx->flags &IORING_SETUP_DEFER_TASKRUN &&io_local_work_pending (ctx)) +return true; +return task_work_pending (current); }static inline void io_tw_lock (struct io_ring_ctx *ctx,struct io_tw_state *ts)
Signature · 1 line
2.45.2
Jens Axboe
maintainerPATCH 2/6 io_uring: replace defer task_work llist with io_wq_work_list
In reply to Jens AxboeAdd a spinlock for the list, and replace the lockless llist with the work list instead. This avoids needing to reverse items in the list before running them, as the io_wq_work_list is FIFO by nature whereas the llist is LIFO.
--- include/linux/io_uring_types.h | 13 ++- io_uring/io_uring.c | 194 ++++++++++++++++----------------- io_uring/io_uring.h | 2 +- 3 files changed, 104 insertions(+), 105 deletions(-)
include/linux/io_uring_types.h +8 -5
@@ -335,8 +335,9 @@ struct io_ring_ctx {* regularly bounce b/w CPUs. */ struct { -struct llist_head work_llist; -struct llist_head retry_llist; +struct io_wq_work_list work_list; +spinlock_t work_lock; +int work_items;unsigned long check_cq;atomic_t cq_wait_nr;atomic_t cq_timeouts;
@@ -566,7 +567,11 @@ enum {typedef void (*io_req_tw_func_t )(struct io_kiocb *req,struct io_tw_state *ts);struct io_task_work { -struct llist_node node; +/* DEFER_TASKRUN uses work_node, regular task_work node */ +union { +struct io_wq_work_node work_node; +struct llist_node node; + };io_req_tw_func_t func; };
@@ -622,8 +627,6 @@ struct io_kiocb {*/ u16 buf_index; -unsigned nr_tw; -/* REQ_F_* flags */ io_req_flags_t flags;
io_uring/io_uring.c +95 -99
275 lines not on this page · raw message
io_uring/io_uring.h +1 -1
@@ -349,7 +349,7 @@ static inline int io_run_task_work(void)static inline bool io_local_work_pending (struct io_ring_ctx *ctx) { -return !llist_empty (&ctx->work_llist) || !llist_empty (&ctx->retry_llist); +return READ_ONCE (ctx->work_list.first); }static inline bool io_task_work_pending (struct io_ring_ctx *ctx)
Signature · 1 line
2.45.2
Jens Axboe
maintainerPATCH 3/6 io_uring/slist: add list-to-list list splice helper
In reply to Jens AxboeAdd a helper to splice a source list to a destination list.
--- io_uring/slist.h | 16 ++++++++++++++++ 1 file changed, 16 insertions(+)
io_uring/slist.h +16 -0
@@ -85,6 +85,22 @@ static inline bool wq_list_splice(struct io_wq_work_list *list,return false; } +static inline bool wq_list_splice_list (struct io_wq_work_list *src, +struct io_wq_work_list *dst) +{ +bool ret = false; + +if (wq_list_empty (dst)) { + *dst = *src; + }else { + dst->last->next = src->first; + dst->last = src->last; + ret = true; + } +INIT_WQ_LIST (src); +return false; +} +static inline void wq_stack_add_head (struct io_wq_work_node *node,struct io_wq_work_node *stack) {
Signature · 1 line
2.45.2
Jens Axboe
maintainerPATCH 4/6 io_uring: switch non-defer task_work to io_wq_work_list
In reply to Jens AxboeSwitch the normal task_work to io_wq_work_list as well, to both unify with defer task_work, but also to avoid needing to reverse the ordering of the list when running it.
Note that this still keeps the manual retry list for SQPOLL task_work. That could go away as well, as now the task_work list is fully ordered and SQPOLL could just leave entries on there when it chops up the running of the list.
--- include/linux/io_uring_types.h | 14 ++- io_uring/io_uring.c | 167 ++++++++++++++++++++------------- io_uring/io_uring.h | 6 +- io_uring/sqpoll.c | 8 +- io_uring/tctx.c | 3 +- 5 files changed, 116 insertions(+), 82 deletions(-)
include/linux/io_uring_types.h +6 -8
@@ -102,7 +102,8 @@ struct io_uring_task {struct percpu_counter inflight;struct {/* task_work */ -struct llist_head task_list; +struct io_wq_work_list task_list; +spinlock_t task_lock;struct callback_head task_work; }____cacheline_aligned_in_smp ; };
@@ -390,8 +391,9 @@ struct io_ring_ctx {struct mm_struct *mm_account;/* ctx exit and cancelation */ -struct llist_head fallback_llist; -struct delayed_work fallback_work; +struct io_wq_work_list fallback_list; +spinlock_t fallback_lock; +struct work_struct fallback_work;struct work_struct exit_work;struct list_head tctx_list;struct completion ref_comp;
@@ -567,11 +569,7 @@ enum {typedef void (*io_req_tw_func_t )(struct io_kiocb *req,struct io_tw_state *ts);struct io_task_work { -/* DEFER_TASKRUN uses work_node, regular task_work node */ -union { -struct io_wq_work_node work_node; -struct llist_node node; - }; +struct io_wq_work_node node;io_req_tw_func_t func; };
io_uring/io_uring.c +100 -67
269 lines not on this page · raw message
io_uring/io_uring.h +4 -2
@@ -91,8 +91,10 @@ void io_req_task_queue(struct io_kiocb *req);void io_req_task_complete (struct io_kiocb *req,struct io_tw_state *ts);void io_req_task_queue_fail (struct io_kiocb *req,int ret);void io_req_task_submit (struct io_kiocb *req,struct io_tw_state *ts); -struct llist_node *io_handle_tw_list (struct llist_node *node,unsigned int *count,unsigned int max_entries); -struct llist_node *tctx_task_work_run (struct io_uring_task *tctx,unsigned int max_entries,unsigned int *count); +struct io_wq_work_node *io_handle_tw_list (struct io_wq_work_node *node, +unsigned int *count,unsigned int max_entries); +struct io_wq_work_node *tctx_task_work_run (struct io_uring_task *tctx, +unsigned int max_entries,unsigned int *count);void tctx_task_work (struct callback_head *cb);__cold void io_uring_cancel_generic (bool cancel_all,struct io_sq_data *sqd); int io_uring_alloc_task_context(struct task_struct *task,
io_uring/sqpoll.c +4 -4
@@ -221,7 +221,7 @@ static bool io_sqd_handle_event(struct io_sq_data *sqd)* than we were asked to process. Newly queued task_work isn't run until the * retry list has been fully processed. */ -static unsigned int io_sq_tw (struct llist_node **retry_list,int max_entries) +static unsigned int io_sq_tw (struct io_wq_work_node **retry_list,int max_entries) {struct io_uring_task *tctx = current->io_uring;unsigned int count =0 ;
@@ -239,11 +239,11 @@ static unsigned int io_sq_tw(struct llist_node **retry_list, int max_entries)return count; } -static bool io_sq_tw_pending (struct llist_node *retry_list) +static bool io_sq_tw_pending (struct io_wq_work_node *retry_list) {struct io_uring_task *tctx = current->io_uring; -return retry_list || !llist_empty (&tctx->task_list); +return retry_list ||READ_ONCE (tctx->task_list.first); }static void io_sq_update_worktime (struct io_sq_data *sqd,struct rusage *start)
@@ -259,7 +259,7 @@ static void io_sq_update_worktime(struct io_sq_data *sqd, struct rusage *start)static int io_sq_thread (void *data) { -struct llist_node *retry_list =NULL ; +struct io_wq_work_node *retry_list =NULL ;struct io_sq_data *sqd = data;struct io_ring_ctx *ctx;struct rusage start;
io_uring/tctx.c +2 -1
@@ -87,7 +87,8 @@ __cold int io_uring_alloc_task_context(struct task_struct *task,atomic_set (&tctx->in_cancel,0 );atomic_set (&tctx->inflight_tracked,0 ); task->io_uring = tctx; -init_llist_head (&tctx->task_list); +INIT_WQ_LIST (&tctx->task_list); +spin_lock_init (&tctx->task_lock);init_task_work (&tctx->task_work, tctx_task_work);return 0 ; }
Signature · 1 line
2.45.2
Jens Axboe
maintainerPATCH 5/6 io_uring: add __tctx_task_work_run() helper
In reply to Jens AxboeMost use cases only care about running all of the task_work, and they don't need the node passed back or the work capped. Rename the existing helper to __tctx_task_work_run(), and add a wrapper around that for the more basic use cases.
--- io_uring/io_uring.c | 18 ++++++++++++------ io_uring/io_uring.h | 9 +++------ io_uring/sqpoll.c | 2 +- 3 files changed, 16 insertions(+), 13 deletions(-)
io_uring/io_uring.c +12 -6
@@ -1137,9 +1137,9 @@ static void io_fallback_tw(struct io_uring_task *tctx, bool sync)__io_fallback_tw (&tctx->task_list, &tctx->task_lock, sync); } -struct io_wq_work_node *tctx_task_work_run (struct io_uring_task *tctx, -unsigned int max_entries, -unsigned int *count) +struct io_wq_work_node *__tctx_task_work_run (struct io_uring_task *tctx, +unsigned int max_entries, +unsigned int *count) {struct io_wq_work_node *node;
@@ -1167,14 +1167,20 @@ struct io_wq_work_node *tctx_task_work_run(struct io_uring_task *tctx,return node; } +unsigned int tctx_task_work_run (struct io_uring_task *tctx) +{ +unsigned int count =0 ; + +__tctx_task_work_run (tctx,UINT_MAX , &count); +return count; +} +void tctx_task_work (struct callback_head *cb) {struct io_uring_task *tctx; -unsigned int count =0 ; tctx = container_of(cb, struct io_uring_task, task_work); -if (tctx_task_work_run (tctx,UINT_MAX , &count)) -WARN_ON_ONCE (1 ); +tctx_task_work_run (tctx); } static inline void io_req_local_work_add(struct io_kiocb *req,
io_uring/io_uring.h +3 -6
@@ -93,8 +93,9 @@ void io_req_task_queue_fail(struct io_kiocb *req, int ret);void io_req_task_submit (struct io_kiocb *req,struct io_tw_state *ts);struct io_wq_work_node *io_handle_tw_list (struct io_wq_work_node *node,unsigned int *count,unsigned int max_entries); -struct io_wq_work_node *tctx_task_work_run (struct io_uring_task *tctx, +struct io_wq_work_node *__tctx_task_work_run (struct io_uring_task *tctx,unsigned int max_entries,unsigned int *count); +unsigned int tctx_task_work_run (struct io_uring_task *tctx);void tctx_task_work (struct callback_head *cb);__cold void io_uring_cancel_generic (bool cancel_all,struct io_sq_data *sqd); int io_uring_alloc_task_context(struct task_struct *task,
@@ -332,12 +333,8 @@ static inline int io_run_task_work(void)resume_user_mode_work (NULL ); }if (current->io_uring) { -unsigned int count =0 ; -__set_current_state (TASK_RUNNING ); -tctx_task_work_run (current->io_uring,UINT_MAX , &count); -if (count) - ret = true; + ret =tctx_task_work_run (current->io_uring) !=0 ; } }if (task_work_pending (current)) {
io_uring/sqpoll.c +1 -1
@@ -232,7 +232,7 @@ static unsigned int io_sq_tw(struct io_wq_work_node **retry_list, int max_entrie goto out; max_entries -= count; } - *retry_list =tctx_task_work_run (tctx, max_entries, &count); + *retry_list =__tctx_task_work_run (tctx, max_entries, &count); out:if (task_work_pending (current))task_work_run ();
Signature · 1 line
2.45.2
Jens Axboe
maintainerPATCH 6/6 io_uring: make __tctx_task_work_run() take an io_wq_work_list
In reply to Jens AxboeThe normal task_work logic doesn't really need it, as it always runs all of the pending work. But for SQPOLL, it can now pass in its retry_list which simplifies the tracking of split up task_work running.
This avoids passing io_wq_work_node around. Rather than pass in a list, SQPOLL could re-add the leftover items to the generic task_work list. But that requires re-locking the task_lock and using task_list for that, whereas having a separate retry list allows for skipping those steps. The downside is that now two lists need checking, but that's now it was before as well.
--- io_uring/io_uring.c | 36 ++++++++++++++++-------------------- io_uring/io_uring.h | 9 +++++---- io_uring/sqpoll.c | 20 +++++++++++--------- 3 files changed, 32 insertions(+), 33 deletions(-)
io_uring/io_uring.c +16 -20
@@ -1044,20 +1044,20 @@ static void ctx_flush_and_put(struct io_ring_ctx *ctx, struct io_tw_state *ts)/* * Run queued task_work, returning the number of entries processed in *count. * If more entries than max_entries are available, stop processing once this -* is reached and return the rest of the list. +* is reached. */ -struct io_wq_work_node *io_handle_tw_list (struct io_wq_work_node *node, -unsigned int *count, -unsigned int max_entries) +void io_handle_tw_list (struct io_wq_work_list *list,unsigned int *count, +unsigned int max_entries) {struct io_ring_ctx *ctx =NULL ;struct io_tw_state ts = { }; do { - struct io_wq_work_node *next = node->next; + struct io_wq_work_node *node = list->first; struct io_kiocb *req = container_of(node, struct io_kiocb, io_task_work.node); + list->first = node->next; if (req->ctx != ctx) { ctx_flush_and_put(ctx, &ts); ctx = req->ctx;
@@ -1067,17 +1067,15 @@ struct io_wq_work_node *io_handle_tw_list(struct io_wq_work_node *node,INDIRECT_CALL_2 (req->io_task_work.func, io_poll_task_func, io_req_rw_complete, req, &ts); - node = next; (*count)++;if (unlikely (need_resched ())) {ctx_flush_and_put (ctx, &ts); ctx =NULL ;cond_resched (); } - }while (node && *count < max_entries); + }while (list->first && *count < max_entries);ctx_flush_and_put (ctx, &ts); -return node; } static __cold void __io_fallback_schedule(struct io_ring_ctx *ctx,
@@ -1137,41 +1135,39 @@ static void io_fallback_tw(struct io_uring_task *tctx, bool sync)__io_fallback_tw (&tctx->task_list, &tctx->task_lock, sync); } -struct io_wq_work_node *__tctx_task_work_run (struct io_uring_task *tctx, -unsigned int max_entries, -unsigned int *count) +void __tctx_task_work_run (struct io_uring_task *tctx, +struct io_wq_work_list *list, +unsigned int max_entries,unsigned int *count) { -struct io_wq_work_node *node; -if (unlikely (current->flags &PF_EXITING )) {io_fallback_tw (tctx, true); -return NULL ; +return ; }if (!READ_ONCE (tctx->task_list.first)) -return NULL ; +return ;spin_lock_irq (&tctx->task_lock); - node = tctx->task_list.first; + *list = tctx->task_list;INIT_WQ_LIST (&tctx->task_list);spin_unlock_irq (&tctx->task_lock); -if (node) - node =io_handle_tw_list (node, count, max_entries); +if (!wq_list_empty (list)) +io_handle_tw_list (list, count, max_entries);/* relaxed read is enough as only the task itself sets ->in_cancel */ if (unlikely (atomic_read (&tctx->in_cancel)))io_uring_drop_tctx_refs (current);trace_io_uring_task_work_run (tctx, *count); -return node; }unsigned int tctx_task_work_run (struct io_uring_task *tctx) { +struct io_wq_work_list list;unsigned int count =0 ; -__tctx_task_work_run (tctx,UINT_MAX , &count); +__tctx_task_work_run (tctx, &list,UINT_MAX , &count);return count; }
io_uring/io_uring.h +5 -4
@@ -91,10 +91,11 @@ void io_req_task_queue(struct io_kiocb *req);void io_req_task_complete (struct io_kiocb *req,struct io_tw_state *ts);void io_req_task_queue_fail (struct io_kiocb *req,int ret);void io_req_task_submit (struct io_kiocb *req,struct io_tw_state *ts); -struct io_wq_work_node *io_handle_tw_list (struct io_wq_work_node *node, -unsigned int *count,unsigned int max_entries); -struct io_wq_work_node *__tctx_task_work_run (struct io_uring_task *tctx, -unsigned int max_entries,unsigned int *count); +void io_handle_tw_list (struct io_wq_work_list *list,unsigned int *count, +unsigned int max_entries); +void __tctx_task_work_run (struct io_uring_task *tctx, +struct io_wq_work_list *list,unsigned int max_entries, +unsigned int *count);unsigned int tctx_task_work_run (struct io_uring_task *tctx);void tctx_task_work (struct callback_head *cb);__cold void io_uring_cancel_generic (bool cancel_all,struct io_sq_data *sqd);
io_uring/sqpoll.c +11 -9
@@ -221,29 +221,29 @@ static bool io_sqd_handle_event(struct io_sq_data *sqd)* than we were asked to process. Newly queued task_work isn't run until the * retry list has been fully processed. */ -static unsigned int io_sq_tw (struct io_wq_work_node **retry_list,int max_entries) +static unsigned int io_sq_tw (struct io_wq_work_list *retry_list,int max_entries) {struct io_uring_task *tctx = current->io_uring;unsigned int count =0 ; -if (*retry_list) { - *retry_list =io_handle_tw_list (*retry_list, &count, max_entries); +if (!wq_list_empty (retry_list)) { +io_handle_tw_list (retry_list, &count, max_entries);if (count >= max_entries) goto out; max_entries -= count; } - *retry_list =__tctx_task_work_run (tctx, max_entries, &count); +__tctx_task_work_run (tctx, retry_list, max_entries, &count); out:if (task_work_pending (current))task_work_run ();return count; } -static bool io_sq_tw_pending (struct io_wq_work_node *retry_list) +static bool io_sq_tw_pending (struct io_wq_work_list *retry_list) {struct io_uring_task *tctx = current->io_uring; -return retry_list ||READ_ONCE (tctx->task_list.first); +return !wq_list_empty (retry_list) || !wq_list_empty (&tctx->task_list); }static void io_sq_update_worktime (struct io_sq_data *sqd,struct rusage *start)
@@ -259,7 +259,7 @@ static void io_sq_update_worktime(struct io_sq_data *sqd, struct rusage *start)static int io_sq_thread (void *data) { -struct io_wq_work_node *retry_list =NULL ; +struct io_wq_work_list retry_list;struct io_sq_data *sqd = data;struct io_ring_ctx *ctx;struct rusage start;
@@ -292,6 +292,7 @@ static int io_sq_thread(void *data)audit_uring_entry (IORING_OP_NOP );audit_uring_exit (true,0 ); +INIT_WQ_LIST (&retry_list);mutex_lock (&sqd->lock);while (1 ) {bool cap_entries, sqt_spin = false;
@@ -332,7 +333,8 @@ static int io_sq_thread(void *data) }prepare_to_wait (&sqd->wait, &wait,TASK_INTERRUPTIBLE ); -if (!io_sqd_events_pending (sqd) && !io_sq_tw_pending (retry_list)) { +if (!io_sqd_events_pending (sqd) && + !io_sq_tw_pending (&retry_list)) {bool needs_sched = true;list_for_each_entry (ctx, &sqd->ctx_list, sqd_list) {
@@ -371,7 +373,7 @@ static int io_sq_thread(void *data) timeout = jiffies + sqd->sq_thread_idle; } -if (retry_list) +if (!wq_list_empty (&retry_list))io_sq_tw (&retry_list,UINT_MAX );io_uring_cancel_generic (true, sqd);
Signature · 1 line
2.45.2
Re: [PATCH 2/6] io_uring: replace defer task_work llist with io_wq_work_list
In reply to Jens AxboeOn 11/22/24 16:12, Jens Axboe wrote:
...
Quoted 14 lines
static inline void io_req_local_work_add(struct
io_kiocb*req,
structio_ring_ctx*ctx,io_uring/io_uring.c +3
-3- unsigned flags) + unsigned tw_flags) { -unsigned nr_wait, nr_tw, nr_tw_prev; -struct llist_node *head; +unsigned nr_tw, nr_tw_prev, nr_wait; +unsigned long flags;/* See comment above IO_CQ_WAKE_INIT */ BUILD_BUG_ON (IO_CQ_WAKE_FORCE <=IORING_MAX_CQ_ENTRIES );/*
io_uring/io_uring.c void tctx_task_work(struct callback_head *cb) · line 1157
Quoted 10 lines
io_uring/io_uring.c +3
-4-* We don't know how many reuqests is there in the link and whether -* they can even be queued lazily, fall back to non-lazy. +* We don't know how many requests are in the link and whether they can +* even be queued lazily, fall back to non-lazy. */ if (req->flags & (REQ_F_LINK |REQ_F_HARDLINK )) - flags &= ~IOU_F_TWQ_LAZY_WAKE ; + tw_flags &= ~IOU_F_TWQ_LAZY_WAKE ; -guard (rcu)();
protects against ctx->task deallocation, see a comment in io_ring_exit_work() -> synchronize_rcu()
io_uring/io_uring.c void tctx_task_work(struct callback_head *cb) · line 1159
Quoted 3 lines
io_uring/io_uring.c +3
-0+spin_lock_irqsave (&ctx->work_lock, flags); +wq_list_add_tail (&req->io_task_work.work_node, &ctx->work_list); + nr_tw_prev = ctx->work_items++;
Is there a good reason why it changes the semantics of what's stored across adds? It was assigning a corrected nr_tw, this one will start heavily spamming with wake_up() in some cases.
Quoted 60 lines
io_uring/io_uring.c +13
-35+spin_unlock_irqrestore (&ctx->work_lock, flags); - head =READ_ONCE (ctx->work_llist.first); -do { - nr_tw_prev =0 ; -if (head) { -struct io_kiocb *first_req =container_of (head, - struct io_kiocb, - io_task_work.node); -/* -* Might be executed at any moment, rely on -* SLAB_TYPESAFE_BY_RCU to keep it alive. -*/ - nr_tw_prev =READ_ONCE (first_req->nr_tw); - } - -/* -* Theoretically, it can overflow, but that's fine as one of -* previous adds should've tried to wake the task. -*/ - nr_tw = nr_tw_prev +1 ; -if (!(flags &IOU_F_TWQ_LAZY_WAKE )) - nr_tw =IO_CQ_WAKE_FORCE ; - - req->nr_tw = nr_tw; - req->io_task_work.node.next = head; - }while (!try_cmpxchg (&ctx->work_llist.first, &head, - &req->io_task_work.node)); - -/* -* cmpxchg implies a full barrier, which pairs with the barrier -* in set_current_state() on the io_cqring_wait() side. It's used -* to ensure that either we see updated ->cq_wait_nr, or waiters -* going to sleep will observe the work added to the list, which -* is similar to the wait/wawke task state sync. -*/ + nr_tw = nr_tw_prev +1 ; +if (!(tw_flags &IOU_F_TWQ_LAZY_WAKE )) + nr_tw =IO_CQ_WAKE_FORCE ; -if (!head) { +if (!nr_tw_prev) {if (ctx->flags &IORING_SETUP_TASKRUN_FLAG )atomic_or (IORING_SQ_TASKRUN , &ctx->rings->sq_flags);if (ctx->has_evfd)io_eventfd_signal (ctx); } +/* +* We need a barrier after unlock, which pairs with the barrier +* in set_current_state() on the io_cqring_wait() side. It's used +* to ensure that either we see updated ->cq_wait_nr, or waiters +* going to sleep will observe the work added to the list, which +* is similar to the wait/wake task state sync. +*/ +smp_mb (); nr_wait =atomic_read (&ctx->cq_wait_nr);/* not enough or no one is waiting */ if (nr_tw < nr_wait)@@ -1253,11 +1233,27 @@ void io_req_task_work_add_remote(struct io_kiocb *req, struct io_ring_ctx *ctx,
Signature · 1 line
Pavel Begunkov
Jens Axboe
maintainerRe: [PATCH 2/6] io_uring: replace defer task_work llist with io_wq_work_list
Quoted 24 lines · Pavel Begunkov
On 11/22/24 16:12, Jens Axboe wrote:
...Quoted 11 lines
static inline void io_req_local_work_add(struct
io_kiocb*req,
structio_ring_ctx*ctx,io_uring/io_uring.c +3
-3- unsigned flags) + unsigned tw_flags) { -unsigned nr_wait, nr_tw, nr_tw_prev; -struct llist_node *head; +unsigned nr_tw, nr_tw_prev, nr_wait; +unsigned long flags;/* See comment above IO_CQ_WAKE_INIT */ BUILD_BUG_ON (IO_CQ_WAKE_FORCE <=IORING_MAX_CQ_ENTRIES );io_uring/io_uring.c void tctx_task_work(struct callback_head *cb)
Quoted 10 lines
io_uring/io_uring.c +3
-3/* -* We don't know how many reuqests is there in the link and whether -* they can even be queued lazily, fall back to non-lazy. +* We don't know how many requests are in the link and whether they can +* even be queued lazily, fall back to non-lazy. */ if (req->flags & (REQ_F_LINK |REQ_F_HARDLINK )) - flags &= ~IOU_F_TWQ_LAZY_WAKE ; + tw_flags &= ~IOU_F_TWQ_LAZY_WAKE ; -guard (rcu)();
Quoted 2 lines
protects against
ctx->taskdeallocation, see a comment inio_ring_exit_work()->synchronize_rcu()
Yeah that's just an editing mistake.
Quoted 8 lines
io_uring/io_uring.c void tctx_task_work(struct callback_head *cb) · line 1159
Quoted 3 lines
io_uring/io_uring.c +3
-0+spin_lock_irqsave (&ctx->work_lock, flags); +wq_list_add_tail (&req->io_task_work.work_node, &ctx->work_list); + nr_tw_prev = ctx->work_items++;Is there a good reason why it changes the semantics of what's stored across adds? It was assigning a corrected
nr_tw, this one will start heavily spamming withwake_up()in some cases.
Not sure I follow, how so? nr_tw_prev will be the previous count, just like before. Except we won't need to dig into the list to find it, we have it readily available. nr_tw will be the current code, or force wake if needed. As before.
Signature · 1 line
Jens Axboe
Re: [PATCH 2/6] io_uring: replace defer task_work llist with io_wq_work_list
Quoted 39 lines · Jens Axboe
Quoted 24 lines · Pavel Begunkov
On 11/22/24 16:12, Jens Axboe wrote:
...Quoted 11 lines
static inline void io_req_local_work_add(struct
io_kiocb*req,
structio_ring_ctx*ctx,io_uring/io_uring.c +3
-3- unsigned flags) + unsigned tw_flags) { -unsigned nr_wait, nr_tw, nr_tw_prev; -struct llist_node *head; +unsigned nr_tw, nr_tw_prev, nr_wait; +unsigned long flags;/* See comment above IO_CQ_WAKE_INIT */ BUILD_BUG_ON (IO_CQ_WAKE_FORCE <=IORING_MAX_CQ_ENTRIES );io_uring/io_uring.c void tctx_task_work(struct callback_head *cb)
Quoted 10 lines
io_uring/io_uring.c +3
-3/* -* We don't know how many reuqests is there in the link and whether -* they can even be queued lazily, fall back to non-lazy. +* We don't know how many requests are in the link and whether they can +* even be queued lazily, fall back to non-lazy. */ if (req->flags & (REQ_F_LINK |REQ_F_HARDLINK )) - flags &= ~IOU_F_TWQ_LAZY_WAKE ; + tw_flags &= ~IOU_F_TWQ_LAZY_WAKE ; -guard (rcu)();Quoted 2 lines
protects against
ctx->taskdeallocation, see a comment inio_ring_exit_work()->synchronize_rcu()Yeah that's just an editing mistake.
Quoted 8 lines
io_uring/io_uring.c void tctx_task_work(struct callback_head *cb) · line 1159
Quoted 3 lines
io_uring/io_uring.c +3
-0+spin_lock_irqsave (&ctx->work_lock, flags); +wq_list_add_tail (&req->io_task_work.work_node, &ctx->work_list); + nr_tw_prev = ctx->work_items++;Is there a good reason why it changes the semantics of what's stored across adds? It was assigning a corrected
nr_tw, this one will start heavily spamming withwake_up()in some cases.
Quoted 4 lines
Not sure I follow, how so?
nr_tw_prevwill be the previous count, just like before. Except we won't need to dig into the list to find it, we have it readily available.nr_twwill be the current code, or force wake if needed. As before.
The problem is what it stores, not how and where. Before req->nr_tw could've been set to IO_CQ_WAKE_FORCE, in which case following requests are not going to attempt waking up the task, now work_items is just a counter.
Let's say you've got a bunch of non-lazy adds coming close to each other. The first sets IO_CQ_WAKE_FORCE and wakes the task, and others just queue themselves in the list. Now, every single one of them will try to wake_up() as long as ->cq_wait_nr is large enough.
Signature · 1 line
Pavel Begunkov
Jens Axboe
maintainerRe: [PATCH 2/6] io_uring: replace defer task_work llist with io_wq_work_list
Quoted 50 lines · Pavel Begunkov
Quoted 39 lines · Jens Axboe
Quoted 24 lines · Pavel Begunkov
On 11/22/24 16:12, Jens Axboe wrote:
...Quoted 11 lines
static inline void io_req_local_work_add(struct
io_kiocb*req,
structio_ring_ctx*ctx,io_uring/io_uring.c +3
-3- unsigned flags) + unsigned tw_flags) { -unsigned nr_wait, nr_tw, nr_tw_prev; -struct llist_node *head; +unsigned nr_tw, nr_tw_prev, nr_wait; +unsigned long flags;/* See comment above IO_CQ_WAKE_INIT */ BUILD_BUG_ON (IO_CQ_WAKE_FORCE <=IORING_MAX_CQ_ENTRIES );io_uring/io_uring.c void tctx_task_work(struct callback_head *cb)
Quoted 10 lines
io_uring/io_uring.c +3
-3/* -* We don't know how many reuqests is there in the link and whether -* they can even be queued lazily, fall back to non-lazy. +* We don't know how many requests are in the link and whether they can +* even be queued lazily, fall back to non-lazy. */ if (req->flags & (REQ_F_LINK |REQ_F_HARDLINK )) - flags &= ~IOU_F_TWQ_LAZY_WAKE ; + tw_flags &= ~IOU_F_TWQ_LAZY_WAKE ; -guard (rcu)();Quoted 2 lines
protects against
ctx->taskdeallocation, see a comment inio_ring_exit_work()->synchronize_rcu()Yeah that's just an editing mistake.
Quoted 8 lines
io_uring/io_uring.c void tctx_task_work(struct callback_head *cb) · line 1159
Quoted 3 lines
io_uring/io_uring.c +3
-0+spin_lock_irqsave (&ctx->work_lock, flags); +wq_list_add_tail (&req->io_task_work.work_node, &ctx->work_list); + nr_tw_prev = ctx->work_items++;Is there a good reason why it changes the semantics of what's stored across adds? It was assigning a corrected
nr_tw, this one will start heavily spamming withwake_up()in some cases.Quoted 4 lines
Not sure I follow, how so?
nr_tw_prevwill be the previous count, just like before. Except we won't need to dig into the list to find it, we have it readily available.nr_twwill be the current code, or force wake if needed. As before.The problem is what it stores, not how and where. Before
req->nr_twcould've been set toIO_CQ_WAKE_FORCE, in which case following requests are not going to attempt waking up the task, nowwork_itemsis just a counter.
Quoted 5 lines
Let's say you've got a bunch of non-lazy adds coming close to each other. The first sets
IO_CQ_WAKE_FORCEand wakes the task, and others just queue themselves in the list. Now, every single one of them will try towake_up()as long as ->cq_wait_nr is large enough.
If we really care about the non-lazy path as much, we can just use the same storing scheme as we did in req->nr_tw, except in ->work_items instead. Not a big deal imho.
Signature · 1 line
Jens Axboe
Re: [PATCH 2/6] io_uring: replace defer task_work llist with io_wq_work_list
Quoted 51 lines · Jens Axboe
Quoted 50 lines · Pavel Begunkov
Quoted 39 lines · Jens Axboe
Quoted 24 lines · Pavel Begunkov
On 11/22/24 16:12, Jens Axboe wrote:
...Quoted 11 lines
static inline void io_req_local_work_add(struct
io_kiocb*req,
structio_ring_ctx*ctx,io_uring/io_uring.c +3
-3- unsigned flags) + unsigned tw_flags) { -unsigned nr_wait, nr_tw, nr_tw_prev; -struct llist_node *head; +unsigned nr_tw, nr_tw_prev, nr_wait; +unsigned long flags;/* See comment above IO_CQ_WAKE_INIT */ BUILD_BUG_ON (IO_CQ_WAKE_FORCE <=IORING_MAX_CQ_ENTRIES );io_uring/io_uring.c void tctx_task_work(struct callback_head *cb)
Quoted 10 lines
io_uring/io_uring.c +3
-3/* -* We don't know how many reuqests is there in the link and whether -* they can even be queued lazily, fall back to non-lazy. +* We don't know how many requests are in the link and whether they can +* even be queued lazily, fall back to non-lazy. */ if (req->flags & (REQ_F_LINK |REQ_F_HARDLINK )) - flags &= ~IOU_F_TWQ_LAZY_WAKE ; + tw_flags &= ~IOU_F_TWQ_LAZY_WAKE ; -guard (rcu)();Quoted 2 lines
protects against
ctx->taskdeallocation, see a comment inio_ring_exit_work()->synchronize_rcu()Yeah that's just an editing mistake.
Quoted 8 lines
io_uring/io_uring.c void tctx_task_work(struct callback_head *cb) · line 1159
Quoted 3 lines
io_uring/io_uring.c +3
-0+spin_lock_irqsave (&ctx->work_lock, flags); +wq_list_add_tail (&req->io_task_work.work_node, &ctx->work_list); + nr_tw_prev = ctx->work_items++;Is there a good reason why it changes the semantics of what's stored across adds? It was assigning a corrected
nr_tw, this one will start heavily spamming withwake_up()in some cases.Quoted 4 lines
Not sure I follow, how so?
nr_tw_prevwill be the previous count, just like before. Except we won't need to dig into the list to find it, we have it readily available.nr_twwill be the current code, or force wake if needed. As before.The problem is what it stores, not how and where. Before
req->nr_twcould've been set toIO_CQ_WAKE_FORCE, in which case following requests are not going to attempt waking up the task, nowwork_itemsis just a counter.
Quoted 7 lines
Quoted 5 lines
Let's say you've got a bunch of non-lazy adds coming close to each other. The first sets
IO_CQ_WAKE_FORCEand wakes the task, and others just queue themselves in the list. Now, every single one of them will try towake_up()as long as ->cq_wait_nr is large enough.If we really care about the non-lazy path as much, we can just use the
Well, it's all linked requests, some of sendzc notif until I optimise it, maybe something else?
Quoted 2 lines
same storing scheme as we did in
req->nr_tw, except in ->work_items instead. Not a big deal imho.
Yes please. It wouldn't be great sneaking them in the same commit either way.
Signature · 1 line
Pavel Begunkov