Misc fixes and cleanups
Jens Axboe
maintainercover Misc fixes and cleanups
Hi,
No real theme to these patches, but sending them out together to make my life a bit easier.
Patch 1 is just a cleanup.
Patch 2 moves io-wq work struct flags to atomics. Again no real reason to do this outside of making KxSAN happier, but it's prudent to do and has no real downside.
Patch 3 cleans up a fix that went into 6.10-rc, removing a __set_current_state() that is bogus and fixing up the additional one to simply use the usual finish_wait() helper.
Signature · 1 line
Jens Axboe
Jens Axboe
maintainerPATCH 1/3 io_uring: use 'state' consistently
__io_submit_flush_completions() assigns ctx->submit_state to a local variable and uses it in all but one spot, switch that forgotten statement to using 'state' as well.
--- io_uring/io_uring.c | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-)
io_uring/io_uring.c +1 -1
@@ -1390,7 +1390,7 @@ void __io_submit_flush_completions(struct io_ring_ctx *ctx) }__io_cq_unlock_post (ctx); -if (!wq_list_empty (&ctx->submit_state.compl_reqs)) { +if (!wq_list_empty (&state->compl_reqs)) {io_free_batch_list (ctx, state->compl_reqs.first);INIT_WQ_LIST (&state->compl_reqs); }
Signature · 1 line
2.43.0
Jens Axboe
maintainerPATCH 2/3 io_uring/io-wq: make io_wq_work flags atomic
In reply to Jens AxboeFrom: Jens Axboe <[email protected]>
The work flags can be set/accessed from different tasks, both the originator of the request, and the io-wq workers. While modifications aren't concurrent, it still makes KMSAN unhappy. There's no real downside to just making the flag reading/manipulation use proper atomics here.
--- include/linux/io_uring_types.h | 2 +- io_uring/io-wq.c | 19 ++++++++++--------- io_uring/io-wq.h | 2 +- io_uring/io_uring.c | 12 ++++++------ 4 files changed, 18 insertions(+), 17 deletions(-)
include/linux/io_uring_types.h +1 -1
@@ -50,7 +50,7 @@ struct io_wq_work_list {struct io_wq_work {struct io_wq_work_node list; -unsigned flags; +atomic_t flags;/* place it here instead of io_kiocb as it fills padding and saves 4B */ int cancel_seq; };
io_uring/io-wq.c +10 -9
@@ -159,7 +159,7 @@ static inline struct io_wq_acct *io_get_acct(struct io_wq *wq, bool bound)static inline struct io_wq_acct *io_work_get_acct (struct io_wq *wq,struct io_wq_work *work) { -return io_get_acct (wq, !(work->flags &IO_WQ_WORK_UNBOUND )); +return io_get_acct (wq, !(atomic_read (&work->flags) &IO_WQ_WORK_UNBOUND )); }static inline struct io_wq_acct *io_wq_get_acct (struct io_worker *worker)
@@ -451,7 +451,7 @@ static void __io_worker_idle(struct io_wq *wq, struct io_worker *worker)static inline unsigned int io_get_work_hash (struct io_wq_work *work) { -return work->flags >>IO_WQ_HASH_SHIFT ; +return atomic_read (&work->flags) >>IO_WQ_HASH_SHIFT ; }static bool io_wait_on_hash (struct io_wq *wq,unsigned int hash)
@@ -592,8 +592,9 @@ static void io_worker_handle_work(struct io_wq_acct *acct, next_hashed =wq_next_work (work); -if (unlikely (do_kill) && (work->flags &IO_WQ_WORK_UNBOUND )) - work->flags |=IO_WQ_WORK_CANCEL ; +if (do_kill && + (atomic_read (&work->flags) &IO_WQ_WORK_UNBOUND )) +atomic_or (IO_WQ_WORK_CANCEL , &work->flags); wq->do_work (work);io_assign_current_work (worker,NULL );
@@ -891,7 +892,7 @@ static bool io_wq_worker_wake(struct io_worker *worker, void *data)static void io_run_cancel (struct io_wq_work *work,struct io_wq *wq) {do { - work->flags |=IO_WQ_WORK_CANCEL ; +atomic_or (IO_WQ_WORK_CANCEL , &work->flags); wq->do_work (work); work = wq->free_work (work); }while (work);
@@ -926,7 +927,7 @@ static bool io_wq_work_match_item(struct io_wq_work *work, void *data)void io_wq_enqueue (struct io_wq *wq,struct io_wq_work *work) {struct io_wq_acct *acct =io_work_get_acct (wq, work); -unsigned long work_flags = work->flags; +unsigned int work_flags =atomic_read (&work->flags);struct io_cb_cancel_data match = { .fn = io_wq_work_match_item, .data = work,
@@ -939,7 +940,7 @@ void io_wq_enqueue(struct io_wq *wq, struct io_wq_work *work)* been marked as one that should not get executed, cancel it here. */ if (test_bit (IO_WQ_BIT_EXIT , &wq->state) || - (work->flags &IO_WQ_WORK_CANCEL )) { + (work_flags &IO_WQ_WORK_CANCEL )) {io_run_cancel (work, wq);return ; }
@@ -982,7 +983,7 @@ void io_wq_hash_work(struct io_wq_work *work, void *val)unsigned int bit; bit =hash_ptr (val,IO_WQ_HASH_ORDER ); - work->flags |= (IO_WQ_WORK_HASHED | (bit <<IO_WQ_HASH_SHIFT )); +atomic_or (IO_WQ_WORK_HASHED | (bit <<IO_WQ_HASH_SHIFT ), &work->flags); } static bool __io_wq_worker_cancel(struct io_worker *worker,
@@ -990,7 +991,7 @@ static bool __io_wq_worker_cancel(struct io_worker *worker, struct io_wq_work *work) {if (work && match->fn (work, match->data)) { - work->flags |=IO_WQ_WORK_CANCEL ; +atomic_or (IO_WQ_WORK_CANCEL , &work->flags);__set_notify_signal (worker->task);return true; }
io_uring/io-wq.h +1 -1
@@ -56,7 +56,7 @@ bool io_wq_worker_stopped(void);static inline bool io_wq_is_hashed (struct io_wq_work *work) { -return work->flags &IO_WQ_WORK_HASHED ; +return atomic_read (&work->flags) &IO_WQ_WORK_HASHED ; }typedef bool (work_cancel_fn )(struct io_wq_work *,void *);
io_uring/io_uring.c +6 -6
@@ -462,9 +462,9 @@ static void io_prep_async_work(struct io_kiocb *req) } req->work.list.next =NULL ; - req->work.flags =0 ; +atomic_set (&req->work.flags,0 );if (req->flags &REQ_F_FORCE_ASYNC ) - req->work.flags |=IO_WQ_WORK_CONCURRENT ; +atomic_or (IO_WQ_WORK_CONCURRENT , &req->work.flags);if (req->file && !(req->flags &REQ_F_FIXED_FILE )) req->flags |=io_file_get_flags (req->file);
@@ -480,7 +480,7 @@ static void io_prep_async_work(struct io_kiocb *req)io_wq_hash_work (&req->work,file_inode (req->file)); }else if (!req->file || !S_ISBLK (file_inode (req->file)->i_mode)) {if (def->unbound_nonreg_file) - req->work.flags |=IO_WQ_WORK_UNBOUND ; +atomic_or (IO_WQ_WORK_UNBOUND , &req->work.flags); } }
@@ -520,7 +520,7 @@ static void io_queue_iowq(struct io_kiocb *req)* worker for it). */ if (WARN_ON_ONCE (!same_thread_group (req->task, current))) - req->work.flags |=IO_WQ_WORK_CANCEL ; +atomic_or (IO_WQ_WORK_CANCEL , &req->work.flags);trace_io_uring_queue_async_work (req,io_wq_is_hashed (&req->work));io_wq_enqueue (tctx->io_wq, &req->work);
@@ -1736,14 +1736,14 @@ void io_wq_submit_work(struct io_wq_work *work)io_arm_ltimeout (req);/* either cancelled or io-wq is dying, so don't touch tctx->iowq */ -if (work->flags &IO_WQ_WORK_CANCEL ) { +if (atomic_read (&work->flags) &IO_WQ_WORK_CANCEL ) { fail:io_req_task_queue_fail (req, err);return ; }if (!io_assign_file (req, def, issue_flags)) { err = -EBADF ; - work->flags |=IO_WQ_WORK_CANCEL ; +atomic_or (IO_WQ_WORK_CANCEL , &work->flags); goto fail; }
Signature · 1 line
2.43.0
Jens Axboe
maintainerPATCH 3/3 io_uring/rsrc: remove redundant __set_current_state() post schedule()
In reply to Jens AxboeWe're guaranteed to be in a TASK_RUNNING state post schedule, so we never need to set the state after that. While in there, remove the other __set_current_state() as well, and just call finish_wait() when we now we're going to break anyway. This is easier to grok than manual __set_current_state() calls.
--- io_uring/rsrc.c | 3 +-- 1 file changed, 1 insertion(+), 2 deletions(-)
io_uring/rsrc.c +1 -2
@@ -224,7 +224,7 @@ __cold static int io_rsrc_ref_quiesce(struct io_rsrc_data *data, ret =io_run_task_work_sig (ctx);if (ret <0 ) { -__set_current_state (TASK_RUNNING ); +finish_wait (&ctx->rsrc_quiesce_wq, &we);mutex_lock (&ctx->uring_lock);if (list_empty (&ctx->rsrc_ref_list)) ret =0 ;
@@ -232,7 +232,6 @@ __cold static int io_rsrc_ref_quiesce(struct io_rsrc_data *data, }schedule (); -__set_current_state (TASK_RUNNING );mutex_lock (&ctx->uring_lock); ret =0 ; }while (!list_empty (&ctx->rsrc_ref_list));
Signature · 1 line
2.43.0