support hashing for linked requests
cover support hashing for linked requests
That's it, honour hashing for dependant works in io-wq.
Pavel Begunkov (3):
io-wq: don't reshed if there is no work
io-wq: split hashing and enqueueing
io-wq: hash dependant works
fs/io-wq.c | 49 ++++++++++++++++++++++++++++++------------------- fs/io-wq.h | 7 ++++++- fs/io_uring.c | 24 ++++++++++-------------- 3 files changed, 46 insertions(+), 34 deletions(-)
Signature · 1 line
2.24.0
PATCH 1/3 io-wq: don't reshed if there is no work
This little tweak restores the behaviour that was before the recent io_worker_handle_work() optimisation patches. It makes the function do cond_resched() and flush_signals() only if there is an actual work to execute.
--- fs/io-wq.c | 10 ++++++---- 1 file changed, 6 insertions(+), 4 deletions(-)
fs/io-wq.c +6 -4
@@ -458,10 +458,12 @@ static void io_impersonate_work(struct io_worker *worker,static void io_assign_current_work (struct io_worker *worker,struct io_wq_work *work) { -/* flush pending signals before assigning new work */ -if (signal_pending (current)) -flush_signals (current); -cond_resched (); +if (work) { +/* flush pending signals before assigning new work */ +if (signal_pending (current)) +flush_signals (current); +cond_resched (); + }spin_lock_irq (&worker->lock); worker->cur_work = work;
Signature · 1 line
2.24.0
PATCH 3/3 io-wq: hash dependant works
In reply to Pavel BegunkovEnable io-wq hashing stuff for dependant works simply by re-enqueueing such requests.
--- fs/io-wq.c | 25 +++++++++++++++++++------ 1 file changed, 19 insertions(+), 6 deletions(-)
fs/io-wq.c +19 -6
@@ -375,11 +375,17 @@ static bool __io_worker_idle(struct io_wqe *wqe, struct io_worker *worker)return __io_worker_unuse (wqe, worker); } -static struct io_wq_work *io_get_next_work (struct io_wqe *wqe,unsigned *hash) +static inline unsigned int io_get_work_hash (struct io_wq_work *work) +{ +return work->flags >>IO_WQ_HASH_SHIFT ; +} + +static struct io_wq_work *io_get_next_work (struct io_wqe *wqe) __must_hold(wqe->lock) {struct io_wq_work_node *node, *prev;struct io_wq_work *work; +unsigned int hash;wq_list_for_each (node, prev, &wqe->work_list) { work = container_of(node, struct io_wq_work, list);
@@ -391,9 +397,9 @@ static struct io_wq_work *io_get_next_work(struct io_wqe *wqe, unsigned *hash) }/* hashed, can run if not already running */ - *hash = work->flags >>IO_WQ_HASH_SHIFT ; -if (!(wqe->hash_map &BIT (*hash))) { - wqe->hash_map |=BIT (*hash); + hash =io_get_work_hash (work); +if (!(wqe->hash_map &BIT (hash))) { + wqe->hash_map |=BIT (hash);wq_node_del (&wqe->work_list, node, prev);return work; }
@@ -470,15 +476,17 @@ static void io_assign_current_work(struct io_worker *worker,spin_unlock_irq (&worker->lock); } +static void io_wqe_enqueue (struct io_wqe *wqe,struct io_wq_work *work); +static void io_worker_handle_work (struct io_worker *worker) __releases(wqe->lock) {struct io_wqe *wqe = worker->wqe;struct io_wq *wq = wqe->wq; -unsigned hash =-1U ; do { struct io_wq_work *work; + unsigned int hash; get_next: /* * If we got some work, mark us as busy. If we didn't, but
@@ -487,7 +495,7 @@ static void io_worker_handle_work(struct io_worker *worker)* can't make progress, any work completion or insertion will * clear the stalled flag. */ - work =io_get_next_work (wqe, &hash); + work =io_get_next_work (wqe);if (work)__io_worker_busy (wqe, worker, work);else if (!wq_list_empty (&wqe->work_list))
@@ -511,11 +519,16 @@ static void io_worker_handle_work(struct io_worker *worker) work->flags |=IO_WQ_WORK_CANCEL ; old_work = work; + hash =io_get_work_hash (work); work->func (&work); work = (old_work == work) ?NULL : work;io_assign_current_work (worker, work); wq->free_work (old_work); +if (work &&io_wq_is_hashed (work)) { +io_wqe_enqueue (wqe, work); + work =NULL ; + }if (hash !=-1U ) {spin_lock_irq (&wqe->lock); wqe->hash_map &= ~BIT_ULL (hash);
Signature · 1 line
2.24.0
PATCH 2/3 io-wq: split hashing and enqueueing
In reply to Pavel BegunkovIt's a preparation patch removing io_wq_enqueue_hashed(), which now should be done by io_wq_hash_work() + io_wq_enqueue().
Also, set hash value for dependant works, and do it as late as possible, because req->file can be unavailable before. This hash will be ignored by io-wq.
--- fs/io-wq.c | 14 +++++--------- fs/io-wq.h | 7 ++++++- fs/io_uring.c | 24 ++++++++++-------------- 3 files changed, 21 insertions(+), 24 deletions(-)
fs/io-wq.c +5 -9
@@ -385,7 +385,7 @@ static struct io_wq_work *io_get_next_work(struct io_wqe *wqe, unsigned *hash) work = container_of(node, struct io_wq_work, list);/* not hashed, can run anytime */ -if (!(work->flags &IO_WQ_WORK_HASHED )) { +if (!io_wq_is_hashed (work)) {wq_node_del (&wqe->work_list, node, prev);return work; }
@@ -795,19 +795,15 @@ void io_wq_enqueue(struct io_wq *wq, struct io_wq_work *work) }/* -* Enqueue work, hashed by some key. Work items that hash to the same value -* will not be done in parallel. Used to limit concurrent writes, generally -* hashed by inode. +* Work items that hash to the same value will not be done in parallel. +* Used to limit concurrent writes, generally hashed by inode. */ -void io_wq_enqueue_hashed (struct io_wq *wq,struct io_wq_work *work,void *val) +void io_wq_hash_work (struct io_wq_work *work,void *val) { -struct io_wqe *wqe = wq->wqes[numa_node_id ()]; -unsigned bit; - +unsigned int bit; bit =hash_ptr (val,IO_WQ_HASH_ORDER ); work->flags |= (IO_WQ_WORK_HASHED | (bit <<IO_WQ_HASH_SHIFT )); -io_wqe_enqueue (wqe, work); }static bool io_wqe_worker_send_sig (struct io_worker *worker,void *data)
fs/io-wq.h +6 -1
@@ -94,7 +94,12 @@ bool io_wq_get(struct io_wq *wq, struct io_wq_data *data);void io_wq_destroy (struct io_wq *wq);void io_wq_enqueue (struct io_wq *wq,struct io_wq_work *work); -void io_wq_enqueue_hashed (struct io_wq *wq,struct io_wq_work *work,void *val); +void io_wq_hash_work (struct io_wq_work *work,void *val); + +static inline bool io_wq_is_hashed (struct io_wq_work *work) +{ +return work->flags &IO_WQ_WORK_HASHED ; +}void io_wq_cancel_all (struct io_wq *wq);enum io_wq_cancel io_wq_cancel_work (struct io_wq *wq,struct io_wq_work *cwork);
fs/io_uring.c +10 -14
@@ -1040,15 +1040,14 @@ static inline void io_req_work_drop_env(struct io_kiocb *req) } } -static inline bool io_prep_async_work (struct io_kiocb *req, +static inline void io_prep_async_work (struct io_kiocb *req,struct io_kiocb **link) {const struct io_op_def *def = &io_op_defs[req->opcode]; -bool do_hashed = false;if (req->flags &REQ_F_ISREG ) {if (def->hash_reg_file) - do_hashed = true; +io_wq_hash_work (&req->work,file_inode (req->file)); }else {if (def->unbound_nonreg_file) req->work.flags |=IO_WQ_WORK_UNBOUND ;
@@ -1057,25 +1056,18 @@ static inline bool io_prep_async_work(struct io_kiocb *req,io_req_work_grab_env (req, def); *link =io_prep_linked_timeout (req); -return do_hashed; }static inline void io_queue_async_work (struct io_kiocb *req) {struct io_ring_ctx *ctx = req->ctx;struct io_kiocb *link; -bool do_hashed; - do_hashed =io_prep_async_work (req, &link); +io_prep_async_work (req, &link); -trace_io_uring_queue_async_work (ctx, do_hashed, req, &req->work, - req->flags); -if (!do_hashed) { -io_wq_enqueue (ctx->io_wq, &req->work); - }else { -io_wq_enqueue_hashed (ctx->io_wq, &req->work, -file_inode (req->file)); - } +trace_io_uring_queue_async_work (ctx,io_wq_is_hashed (&req->work), req, + &req->work, req->flags); +io_wq_enqueue (ctx->io_wq, &req->work);if (link)io_queue_linked_timeout (link);
@@ -1582,6 +1574,10 @@ static void io_link_work_cb(struct io_wq_work **workptr)static void io_wq_assign_next (struct io_wq_work **workptr,struct io_kiocb *nxt) {struct io_kiocb *link; +const struct io_op_def *def = &io_op_defs[nxt->opcode]; + +if ((nxt->flags &REQ_F_ISREG ) && def->hash_reg_file) +io_wq_hash_work (&nxt->work,file_inode (nxt->file)); *workptr = &nxt->work; link =io_prep_linked_timeout (nxt);
Signature · 1 line
2.24.0
Jens Axboe
maintainerQuoted 1 line · Pavel Begunkov
That's it, honour hashing for dependant works in io-wq.
Looks good to me, applied for 5.7, thanks.
Signature · 1 line
Jens Axboe