async punting improvements for io_uring
v2 cover async punting improvements for io_uring
This cleans up io-wq punting paths, doing small fixes and removing unnecessary logic from different submission paths. The last patch is a resubmission after a rebase bundled into this patchset.
v2:
- remove pid-related comment, as it's fixed separately
- make ("add missing io_req_cancelled()") first in the series, so it may be picked for 5.6
Pavel Begunkov (5):
io_uring: add missing io_req_cancelled()
io_uring: remove REQ_F_MUST_PUNT
io_uring: don't call work.func from sync ctx
io_uring: don't do full *prep_worker() from io-wq
io_uring: remove req->in_async
fs/io_uring.c | 123 ++++++++++++++++++++++++++------------------------ 1 file changed, 65 insertions(+), 58 deletions(-)
Signature · 1 line
2.24.0
PATCH v2 1/5 io_uring: add missing io_req_cancelled()
fallocate_finish() is missing cancellation check. Add it. It's safe to do that, as only flags setup and sqe fields copy are done before it gets into __io_fallocate().
--- fs/io_uring.c | 4 ++++ 1 file changed, 4 insertions(+)
fs/io_uring.c +4 -0
@@ -2517,6 +2517,9 @@ static void io_fallocate_finish(struct io_wq_work **workptr)struct io_kiocb *nxt =NULL ;int ret; +if (io_req_cancelled (req)) +return ; + ret =vfs_fallocate (req->file, req->sync.mode, req->sync.off, req->sync.len);if (ret <0 )
@@ -2904,6 +2907,7 @@ static void io_close_finish(struct io_wq_work **workptr) struct io_kiocb *req = container_of(*workptr, struct io_kiocb, work);struct io_kiocb *nxt =NULL ; +/* not cancellable, don't do io_req_cancelled() */ __io_close_finish (req, &nxt);if (nxt)io_wq_assign_next (workptr, nxt);
Signature · 1 line
2.24.0
Jens Axboe
maintainerRe: [PATCH v2 1/5] io_uring: add missing io_req_cancelled()
Quoted 3 lines · Pavel Begunkov
fallocate_finish()is missing cancellation check. Add it. It's safe to do that, as only flags setup and sqe fields copy are done before it gets into__io_fallocate().
Thanks, I added this one to the 5.6 mix.
Going to be sporadic this next week, but I hope I can get to your 5.7 material anyway.
Signature · 1 line
Jens Axboe
Re: [PATCH v2 1/5] io_uring: add missing io_req_cancelled()
Quoted 10 lines · Jens Axboe
Quoted 3 lines · Pavel Begunkov
fallocate_finish()is missing cancellation check. Add it. It's safe to do that, as only flags setup and sqe fields copy are done before it gets into__io_fallocate().Thanks, I added this one to the 5.6 mix.
Going to be sporadic this next week, but I hope I can get to your 5.7 material anyway.
Sure, there is plenty of time
Signature · 1 line
Pavel Begunkov
PATCH v2 2/5 io_uring: remove REQ_F_MUST_PUNT
In reply to Pavel BegunkovREQ_F_MUST_PUNT is needed to ignore REQ_F_NOWAIT and thus always punt async if have got -EAGAIN. It's enough to clear REQ_F_NOWAIT instead of having yet another flag.
--- fs/io_uring.c | 18 +++++++----------- 1 file changed, 7 insertions(+), 11 deletions(-)
fs/io_uring.c +7 -11
@@ -477,7 +477,6 @@ enum {REQ_F_LINK_TIMEOUT_BIT ,REQ_F_TIMEOUT_BIT ,REQ_F_ISREG_BIT , -REQ_F_MUST_PUNT_BIT ,REQ_F_TIMEOUT_NOSEQ_BIT ,REQ_F_COMP_LOCKED_BIT , REQ_F_NEED_CLEANUP_BIT,
@@ -514,8 +513,6 @@ enum {REQ_F_TIMEOUT =BIT (REQ_F_TIMEOUT_BIT ),/* regular file */ REQ_F_ISREG =BIT (REQ_F_ISREG_BIT ), -/* must be punted even for NONBLOCK */ -REQ_F_MUST_PUNT =BIT (REQ_F_MUST_PUNT_BIT ),/* no timeout sequence */ REQ_F_TIMEOUT_NOSEQ = BIT(REQ_F_TIMEOUT_NOSEQ_BIT),/* completion under lock */
@@ -2252,11 +2249,11 @@ static int io_read(struct io_kiocb *req, struct io_kiocb **nxt, req->result = io_size;/* -* If the file doesn't support async, mark it as REQ_F_MUST_PUNT so -* we know to async punt it even if it was opened O_NONBLOCK +* If the file doesn't support async, async punt it even if it +* was opened O_NONBLOCK */ if (force_nonblock && !io_file_supports_async (req->file)) { - req->flags |=REQ_F_MUST_PUNT ; + req->flags &= ~REQ_F_NOWAIT ; goto copy_iov; }
@@ -2341,11 +2338,11 @@ static int io_write(struct io_kiocb *req, struct io_kiocb **nxt, req->result = io_size;/* -* If the file doesn't support async, mark it as REQ_F_MUST_PUNT so -* we know to async punt it even if it was opened O_NONBLOCK +* If the file doesn't support async, async punt it even if it +* was opened O_NONBLOCK */ if (force_nonblock && !io_file_supports_async (req->file)) { - req->flags |=REQ_F_MUST_PUNT ; + req->flags &= ~REQ_F_NOWAIT ; goto copy_iov; }
@@ -4725,8 +4722,7 @@ static void __io_queue_sqe(struct io_kiocb *req, const struct io_uring_sqe *sqe)* We async punt it if the file wasn't marked NOWAIT, or if the file * doesn't support non-blocking read/write attempts */ -if (ret == -EAGAIN && (!(req->flags &REQ_F_NOWAIT ) || - (req->flags &REQ_F_MUST_PUNT ))) { +if (ret == -EAGAIN && !(req->flags &REQ_F_NOWAIT )) { punt:if (io_op_defs[req->opcode].file_table) { ret =io_grab_files (req);
Signature · 1 line
2.24.0
PATCH v2 3/5 io_uring: don't call work.func from sync ctx
In reply to Pavel BegunkovMany operations define custom work.func before getting into an io-wq. There are several points against:
- it calls io_wq_assign_next() from outside io-wq, that may be confusing
- sync context would go unnecessary through io_req_cancelled()
- prototypes are quite different, so work!=old_work looks strange
- makes async/sync responsibilities fuzzy
- adds extra overhead
Don't call generic path and io-wq handlers from each other, but use helpers instead
--- fs/io_uring.c | 76 +++++++++++++++++++++++++-------------------------- 1 file changed, 38 insertions(+), 38 deletions(-)
fs/io_uring.c +38 -38
@@ -2468,23 +2468,28 @@ static void io_wq_assign_next(struct io_wq_work **workptr, struct io_kiocb *nxt) } } -static void io_fsync_finish (struct io_wq_work **workptr) +static void __io_fsync (struct io_kiocb *req,struct io_kiocb **nxt) { - struct io_kiocb *req = container_of(*workptr, struct io_kiocb, work);loff_t end = req->sync.off + req->sync.len; -struct io_kiocb *nxt =NULL ;int ret; -if (io_req_cancelled (req)) -return ; - ret =vfs_fsync_range (req->file, req->sync.off, end >0 ? end :LLONG_MAX , req->sync.flags &IORING_FSYNC_DATASYNC );if (ret <0 )req_set_fail_links (req);io_cqring_add_event (req, ret); -io_put_req_find_next (req, &nxt); +io_put_req_find_next (req, nxt); +} + +static void io_fsync_finish (struct io_wq_work **workptr) +{ + struct io_kiocb *req = container_of(*workptr, struct io_kiocb, work); +struct io_kiocb *nxt =NULL ; + +if (io_req_cancelled (req)) +return ; +__io_fsync (req, &nxt);if (nxt)io_wq_assign_next (workptr, nxt); }
@@ -2492,26 +2497,18 @@ static void io_fsync_finish(struct io_wq_work **workptr)static int io_fsync (struct io_kiocb *req,struct io_kiocb **nxt,bool force_nonblock) { -struct io_wq_work *work, *old_work; -/* fsync always requires a blocking context */ if (force_nonblock) {io_put_req (req); req->work.func = io_fsync_finish;return -EAGAIN ; } - - work = old_work = &req->work; -io_fsync_finish (&work); -if (work && work != old_work) - *nxt = container_of(work, struct io_kiocb, work); +__io_fsync (req, nxt);return 0 ; } -static void io_fallocate_finish (struct io_wq_work **workptr) +static void __io_fallocate (struct io_kiocb *req,struct io_kiocb **nxt) { - struct io_kiocb *req = container_of(*workptr, struct io_kiocb, work); -struct io_kiocb *nxt =NULL ;int ret;if (io_req_cancelled (req))
@@ -2522,7 +2519,15 @@ static void io_fallocate_finish(struct io_wq_work **workptr)if (ret <0 )req_set_fail_links (req);io_cqring_add_event (req, ret); -io_put_req_find_next (req, &nxt); +io_put_req_find_next (req, nxt); +} + +static void io_fallocate_finish (struct io_wq_work **workptr) +{ + struct io_kiocb *req = container_of(*workptr, struct io_kiocb, work); +struct io_kiocb *nxt =NULL ; + +__io_fallocate (req, &nxt);if (nxt)io_wq_assign_next (workptr, nxt); }
@@ -2542,8 +2547,6 @@ static int io_fallocate_prep(struct io_kiocb *req,static int io_fallocate (struct io_kiocb *req,struct io_kiocb **nxt,bool force_nonblock) { -struct io_wq_work *work, *old_work; -/* fallocate always requiring blocking context */ if (force_nonblock) {io_put_req (req);
@@ -2551,11 +2554,7 @@ static int io_fallocate(struct io_kiocb *req, struct io_kiocb **nxt,return -EAGAIN ; } - work = old_work = &req->work; -io_fallocate_finish (&work); -if (work && work != old_work) - *nxt = container_of(work, struct io_kiocb, work); - +__io_fallocate (req, nxt);return 0 ; }
@@ -2959,21 +2958,27 @@ static int io_prep_sfr(struct io_kiocb *req, const struct io_uring_sqe *sqe)return 0 ; } -static void io_sync_file_range_finish (struct io_wq_work **workptr) +static void __io_sync_file_range (struct io_kiocb *req,struct io_kiocb **nxt) { - struct io_kiocb *req = container_of(*workptr, struct io_kiocb, work); -struct io_kiocb *nxt =NULL ;int ret; -if (io_req_cancelled (req)) -return ; - ret =sync_file_range (req->file, req->sync.off, req->sync.len, req->sync.flags);if (ret <0 )req_set_fail_links (req);io_cqring_add_event (req, ret); -io_put_req_find_next (req, &nxt); +io_put_req_find_next (req, nxt); +} + + +static void io_sync_file_range_finish (struct io_wq_work **workptr) +{ + struct io_kiocb *req = container_of(*workptr, struct io_kiocb, work); +struct io_kiocb *nxt =NULL ; + +if (io_req_cancelled (req)) +return ; +__io_sync_file_range (req, &nxt);if (nxt)io_wq_assign_next (workptr, nxt); }
@@ -2981,8 +2986,6 @@ static void io_sync_file_range_finish(struct io_wq_work **workptr)static int io_sync_file_range (struct io_kiocb *req,struct io_kiocb **nxt,bool force_nonblock) { -struct io_wq_work *work, *old_work; -/* sync_file_range always requires a blocking context */ if (force_nonblock) {io_put_req (req);
@@ -2990,10 +2993,7 @@ static int io_sync_file_range(struct io_kiocb *req, struct io_kiocb **nxt,return -EAGAIN ; } - work = old_work = &req->work; -io_sync_file_range_finish (&work); -if (work && work != old_work) - *nxt = container_of(work, struct io_kiocb, work); +__io_sync_file_range (req, nxt);return 0 ; }
Signature · 1 line
2.24.0
PATCH v2 4/5 io_uring: don't do full *prep_worker() from io-wq
In reply to Pavel Begunkovio_prep_async_worker() called io_wq_assign_next() do many useless checks:io_req_work_grab_env() was already called during prep, and @do_hashed is not ever used. Add io_prep_next_work() -- simplified version, that can be called io-wq.
--- fs/io_uring.c | 13 ++++++++++++- 1 file changed, 12 insertions(+), 1 deletion(-)
fs/io_uring.c +12 -1
@@ -952,6 +952,17 @@ static inline void io_req_work_drop_env(struct io_kiocb *req) } } +static inline void io_prep_next_work (struct io_kiocb *req, +struct io_kiocb **link) +{ +const struct io_op_def *def = &io_op_defs[req->opcode]; + +if (!(req->flags &REQ_F_ISREG ) && def->unbound_nonreg_file) + req->work.flags |=IO_WQ_WORK_UNBOUND ; + + *link =io_prep_linked_timeout (req); +} +static inline bool io_prep_async_work (struct io_kiocb *req,struct io_kiocb **link) {
@@ -2459,7 +2470,7 @@ static void io_wq_assign_next(struct io_wq_work **workptr, struct io_kiocb *nxt) {struct io_kiocb *link; -io_prep_async_work (nxt, &link); +io_prep_next_work (nxt, &link); *workptr = &nxt->work;if (link) { nxt->work.flags |=IO_WQ_WORK_CB ;
Signature · 1 line
2.24.0
PATCH v2 5/5 io_uring: remove req->in_async
In reply to Pavel Begunkovreq->in_async is not really needed, it only prevents propagation of @nxt for fast not-blocked submissions. Remove it.
--- fs/io_uring.c | 12 ++++-------- 1 file changed, 4 insertions(+), 8 deletions(-)
fs/io_uring.c +4 -8
@@ -553,7 +553,6 @@ struct io_kiocb {* llist_node is only used for poll deferred completions */ struct llist_node llist_node; -bool in_async;bool needs_fixed_file;u8 opcode;
@@ -1980,14 +1979,13 @@ static inline void io_rw_done(struct kiocb *kiocb, ssize_t ret) } } -static void kiocb_done (struct kiocb *kiocb,ssize_t ret,struct io_kiocb **nxt, -bool in_async) +static void kiocb_done (struct kiocb *kiocb,ssize_t ret,struct io_kiocb **nxt) { struct io_kiocb *req = container_of(kiocb, struct io_kiocb, rw.kiocb);if (req->flags &REQ_F_CUR_POS ) req->file->f_pos = kiocb->ki_pos; -if (in_async && ret >=0 && kiocb->ki_complete == io_complete_rw) +if (ret >=0 && kiocb->ki_complete == io_complete_rw) *nxt =__io_complete_rw (kiocb, ret);else io_rw_done (kiocb, ret);
@@ -2280,7 +2278,7 @@ static int io_read(struct io_kiocb *req, struct io_kiocb **nxt,/* Catch -EAGAIN return for forced non-blocking submission */ if (!force_nonblock || ret2 != -EAGAIN ) { -kiocb_done (kiocb, ret2, nxt, req->in_async); +kiocb_done (kiocb, ret2, nxt); }else { copy_iov: ret = io_setup_async_rw(req, io_size, iovec,
@@ -2393,7 +2391,7 @@ static int io_write(struct io_kiocb *req, struct io_kiocb **nxt,if (ret2 == -EOPNOTSUPP && (kiocb->ki_flags &IOCB_NOWAIT )) ret2 = -EAGAIN ;if (!force_nonblock || ret2 != -EAGAIN ) { -kiocb_done (kiocb, ret2, nxt, req->in_async); +kiocb_done (kiocb, ret2, nxt); }else { copy_iov: ret = io_setup_async_rw(req, io_size, iovec,
@@ -4530,7 +4528,6 @@ static void io_wq_submit_work(struct io_wq_work **workptr) }if (!ret) { - req->in_async = true; do { ret = io_issue_sqe(req, NULL, &nxt, false); /*
@@ -5066,7 +5063,6 @@ static int io_submit_sqes(struct io_ring_ctx *ctx, unsigned int nr, *mm = ctx->sqo_mm; } - req->in_async = async; req->needs_fixed_file = async;trace_io_uring_submit_sqe (ctx, req->opcode, req->user_data, true, async);
Signature · 1 line
2.24.0