moving grab_env() later before punt
PATCH 1/4 io_uring: don't pass def into io_req_work_grab_env
In reply to Pavel BegunkovRemove struct io_op_def *def parameter from io_req_work_grab_env(), it's trivially deducible from req->opcode and fast. The API is cleaner this way, and also helps the complier to understand that it's a real constant and could be register-cached.
--- fs/io_uring.c | 9 +++++---- 1 file changed, 5 insertions(+), 4 deletions(-)
fs/io_uring.c +5 -4
@@ -1101,9 +1101,10 @@ static void __io_commit_cqring(struct io_ring_ctx *ctx) } } -static inline void io_req_work_grab_env (struct io_kiocb *req, -const struct io_op_def *def) +static inline void io_req_work_grab_env (struct io_kiocb *req) { +const struct io_op_def *def = &io_op_defs[req->opcode]; +if (!req->work.mm && def->needs_mm) {mmgrab (current->mm); req->work.mm = current->mm;
@@ -1161,7 +1162,7 @@ static inline void io_prep_async_work(struct io_kiocb *req, }io_req_init_async (req); -io_req_work_grab_env (req, def); +io_req_work_grab_env (req); *link =io_prep_linked_timeout (req); }
@@ -5232,7 +5233,7 @@ static int io_req_defer_prep(struct io_kiocb *req,if (for_async || (req->flags &REQ_F_WORK_INITIALIZED )) {io_req_init_async (req); -io_req_work_grab_env (req, &io_op_defs[req->opcode]); +io_req_work_grab_env (req); }switch (req->opcode) {
Signature · 1 line
2.24.0
PATCH 2/4 io_uring: do init work in grab_env()
In reply to Pavel BegunkovPlace io_req_init_async() in io_req_work_grab_env() so it won't be forgotten.
--- fs/io_uring.c | 8 +++----- 1 file changed, 3 insertions(+), 5 deletions(-)
fs/io_uring.c +3 -5
@@ -1105,6 +1105,8 @@ static inline void io_req_work_grab_env(struct io_kiocb *req) {const struct io_op_def *def = &io_op_defs[req->opcode]; +io_req_init_async (req); +if (!req->work.mm && def->needs_mm) {mmgrab (current->mm); req->work.mm = current->mm;
@@ -1161,9 +1163,7 @@ static inline void io_prep_async_work(struct io_kiocb *req, req->work.flags |=IO_WQ_WORK_UNBOUND ; } -io_req_init_async (req);io_req_work_grab_env (req); - *link =io_prep_linked_timeout (req); }
@@ -5231,10 +5231,8 @@ static int io_req_defer_prep(struct io_kiocb *req,return ret; } -if (for_async || (req->flags &REQ_F_WORK_INITIALIZED )) { -io_req_init_async (req); +if (for_async || (req->flags &REQ_F_WORK_INITIALIZED ))io_req_work_grab_env (req); - }switch (req->opcode) {case IORING_OP_NOP :
Signature · 1 line
2.24.0
PATCH 3/4 io_uring: factor out grab_env() from defer_prep()
In reply to Pavel BegunkovRemove io_req_work_grab_env() call from io_req_defer_prep(), just call it when neccessary.
--- fs/io_uring.c | 15 +++++++-------- 1 file changed, 7 insertions(+), 8 deletions(-)
fs/io_uring.c +7 -8
@@ -5217,7 +5217,7 @@ static int io_files_update(struct io_kiocb *req, bool force_nonblock, }static int io_req_defer_prep (struct io_kiocb *req, -const struct io_uring_sqe *sqe,bool for_async) +const struct io_uring_sqe *sqe) {ssize_t ret =0 ;
@@ -5231,9 +5231,6 @@ static int io_req_defer_prep(struct io_kiocb *req,return ret; } -if (for_async || (req->flags &REQ_F_WORK_INITIALIZED )) -io_req_work_grab_env (req); -switch (req->opcode) {case IORING_OP_NOP :break ;
@@ -5346,9 +5343,10 @@ static int io_req_defer(struct io_kiocb *req, const struct io_uring_sqe *sqe)if (!req->io) {if (io_alloc_async_ctx (req))return -EAGAIN ; - ret =io_req_defer_prep (req, sqe, true); + ret =io_req_defer_prep (req, sqe);if (ret <0 )return ret; +io_req_work_grab_env (req); }spin_lock_irq (&ctx->completion_lock);
@@ -5960,9 +5958,10 @@ static void io_queue_sqe(struct io_kiocb *req, const struct io_uring_sqe *sqe, ret = -EAGAIN ;if (io_alloc_async_ctx (req)) goto fail_req; - ret =io_req_defer_prep (req, sqe, true); + ret =io_req_defer_prep (req, sqe);if (unlikely (ret <0 )) goto fail_req; +io_req_work_grab_env (req); }/*
@@ -6016,7 +6015,7 @@ static int io_submit_sqe(struct io_kiocb *req, const struct io_uring_sqe *sqe,if (io_alloc_async_ctx (req))return -EAGAIN ; - ret =io_req_defer_prep (req, sqe, false); + ret =io_req_defer_prep (req, sqe);if (ret) {/* fail even hard links since we don't submit */ head->flags |=REQ_F_FAIL_LINK ;
@@ -6043,7 +6042,7 @@ static int io_submit_sqe(struct io_kiocb *req, const struct io_uring_sqe *sqe,if (io_alloc_async_ctx (req))return -EAGAIN ; - ret =io_req_defer_prep (req, sqe, false); + ret =io_req_defer_prep (req, sqe);if (ret) req->flags |=REQ_F_FAIL_LINK ; *link = req;
Signature · 1 line
2.24.0
PATCH 4/4 io_uring: do grab_env() just before punting
In reply to Pavel BegunkovCurrently io_steal_work() is disabled, and every linked request should go through task_work for initialisation. Do io_req_work_grab_env() just before io-wq punting and for the whole link, so any request reachable by io_steal_work() is prepared.
This is also interesting for another reason -- it localises io_req_work_grab_env() into one place just before io-wq punting, helping to to better manage req->work lifetime and add some neat cleanup/optimisations later.
--- fs/io_uring.c | 53 ++++++++++++++++++++++++++++----------------------- 1 file changed, 29 insertions(+), 24 deletions(-)
fs/io_uring.c +29 -24
@@ -1101,7 +1101,7 @@ static void __io_commit_cqring(struct io_ring_ctx *ctx) } } -static inline void io_req_work_grab_env (struct io_kiocb *req) +static void io_req_work_grab_env (struct io_kiocb *req) {const struct io_op_def *def = &io_op_defs[req->opcode];
@@ -1150,8 +1150,7 @@ static inline void io_req_work_drop_env(struct io_kiocb *req) } } -static inline void io_prep_async_work (struct io_kiocb *req, -struct io_kiocb **link) +static void io_prep_async_work (struct io_kiocb *req) {const struct io_op_def *def = &io_op_defs[req->opcode];
@@ -1164,15 +1163,22 @@ static inline void io_prep_async_work(struct io_kiocb *req, }io_req_work_grab_env (req); - *link =io_prep_linked_timeout (req); } -static inline void io_queue_async_work (struct io_kiocb *req) +static void io_prep_async_link (struct io_kiocb *req) { -struct io_ring_ctx *ctx = req->ctx; -struct io_kiocb *link; +struct io_kiocb *cur; -io_prep_async_work (req, &link); +io_prep_async_work (req); +if (req->flags &REQ_F_LINK_HEAD ) +list_for_each_entry (cur, &req->link_list, link_list) +io_prep_async_work (cur); +} + +static void __io_queue_async_work (struct io_kiocb *req) +{ +struct io_ring_ctx *ctx = req->ctx; +struct io_kiocb *link =io_prep_linked_timeout (req);trace_io_uring_queue_async_work (ctx,io_wq_is_hashed (&req->work), req, &req->work, req->flags);
@@ -1182,6 +1188,13 @@ static inline void io_queue_async_work(struct io_kiocb *req)io_queue_linked_timeout (link); } +static void io_queue_async_work (struct io_kiocb *req) +{ +/* init ->work of the whole link before punting */ +io_prep_async_link (req); +__io_queue_async_work (req); +} +static void io_kill_timeout (struct io_kiocb *req) {int ret;
@@ -1215,7 +1228,8 @@ static void __io_queue_deferred(struct io_ring_ctx *ctx)if (req_need_defer (req))break ;list_del_init (&req->list); -io_queue_async_work (req); +/* punt-init is done before queueing for defer */ +__io_queue_async_work (req); }while (!list_empty (&ctx->defer_list)); }
@@ -1774,7 +1788,7 @@ static void io_put_req(struct io_kiocb *req)static struct io_wq_work *io_steal_work (struct io_kiocb *req) { -struct io_kiocb *nxt =NULL ; +struct io_kiocb *timeout, *nxt =NULL ;/* * A ref is owned by io-wq in which context we're. So, if that's the
@@ -1788,18 +1802,10 @@ static struct io_wq_work *io_steal_work(struct io_kiocb *req)if (!nxt)return NULL ; -if ((nxt->flags &REQ_F_ISREG ) && io_op_defs[nxt->opcode].hash_reg_file) -io_wq_hash_work (&nxt->work,file_inode (nxt->file)); - -io_req_task_queue (nxt); -/* -* If we're going to return actual work, here should be timeout prep: -* -* link = io_prep_linked_timeout(nxt); -* if (link) -* nxt->flags |= REQ_F_QUEUE_TIMEOUT; -*/ -return NULL ; + timeout =io_prep_linked_timeout (nxt); +if (timeout) + nxt->flags |=REQ_F_QUEUE_TIMEOUT ; +return &nxt->work; }/*
@@ -5346,8 +5352,8 @@ static int io_req_defer(struct io_kiocb *req, const struct io_uring_sqe *sqe) ret =io_req_defer_prep (req, sqe);if (ret <0 )return ret; -io_req_work_grab_env (req); } +io_prep_async_link (req);spin_lock_irq (&ctx->completion_lock);if (!req_need_defer (req) &&list_empty (&ctx->defer_list)) {
@@ -5961,7 +5967,6 @@ static void io_queue_sqe(struct io_kiocb *req, const struct io_uring_sqe *sqe, ret =io_req_defer_prep (req, sqe);if (unlikely (ret <0 )) goto fail_req; -io_req_work_grab_env (req); }/*
Signature · 1 line
2.24.0
cover moving grab_env() later before punt
This is a first part of patches cleaning up after recent link/tast_work quick fixes. The main idea intention here is to make io_steal_work() functional again. That's done in [4/4], others are preps.
[4/4] may have its dragons, and even though I tested it long enough, would love someone to take a closer look / test as well.
note: on top of "cleanup for req_free/find_next"
Pavel Begunkov (4):
io_uring: don't pass def into io_req_work_grab_env
io_uring: do init work in grab_env()
io_uring: factor out grab_env() from defer_prep()
io_uring: do grab_env() just before punting
fs/io_uring.c | 73 +++++++++++++++++++++++++++------------------------ 1 file changed, 38 insertions(+), 35 deletions(-)
Signature · 1 line
2.24.0
Jens Axboe
maintainerQuoted 6 lines · Pavel Begunkov
This is a first part of patches cleaning up after recent link/tast_work quick fixes. The main idea intention here is to make
io_steal_work()functional again. That's done in [4/4], others are preps.[4/4] may have its dragons, and even though I tested it long enough, would love someone to take a closer look / test as well.
1-3 are straight forward, but I also didn't find any issues with 4 both reviewing it and also testing it. So I'll apply this series, thanks!
Signature · 1 line
Jens Axboe