simple cleanups
cover simple cleanups
First two patches moves fixed buffer iter init into rsrs.c and decouples it from rw requests, 3 and 4 are just small unrelated cleanups.
Pavel Begunkov (4):
io_uring: opcode independent fixed buf import
io_uring: move io_import_fixed()
io_uring: consistent naming for inline completion
io_uring: add an warn_once for poll_find
io_uring/io_uring.c | 4 +-- io_uring/io_uring.h | 10 ++++++- io_uring/poll.c | 5 ++++ io_uring/rsrc.c | 60 +++++++++++++++++++++++++++++++++++++++ io_uring/rsrc.h | 3 ++ io_uring/rw.c | 69 +-------------------------------------------- 6 files changed, 80 insertions(+), 71 deletions(-)
Signature · 1 line
2.36.1
PATCH 1/4 io_uring: opcode independent fixed buf import
Fixed buffers are generic infrastructure, make io_import_fixed() opcode agnostic.
--- io_uring/rw.c | 21 +++++++-------------- 1 file changed, 7 insertions(+), 14 deletions(-)
io_uring/rw.c +7 -14
@@ -273,14 +273,15 @@ static int kiocb_done(struct io_kiocb *req, ssize_t ret,return IOU_ISSUE_SKIP_COMPLETE ; } -static int __io_import_fixed (struct io_kiocb *req,int ddir, -struct iov_iter *iter,struct io_mapped_ubuf *imu) +static int io_import_fixed (int ddir,struct iov_iter *iter, +struct io_mapped_ubuf *imu, +u64 buf_addr,size_t len) { -struct io_rw *rw =io_kiocb_to_cmd (req); -size_t len = rw->len; -u64 buf_end, buf_addr = rw->addr; +u64 buf_end;size_t offset; +if (WARN_ON_ONCE (!imu)) +return -EFAULT ;if (unlikely (check_add_overflow (buf_addr, (u64 )len, &buf_end)))return -EFAULT ;/* not inside the mapped region */
@@ -332,14 +333,6 @@ static int __io_import_fixed(struct io_kiocb *req, int ddir,return 0 ; } -static int io_import_fixed (struct io_kiocb *req,int rw,struct iov_iter *iter, -unsigned int issue_flags) -{ -if (WARN_ON_ONCE (!req->imu)) -return -EFAULT ; -return __io_import_fixed (req, rw, iter, req->imu); -} -#ifdef CONFIG_COMPAT static ssize_t io_compat_import (struct io_kiocb *req,struct iovec *iov,unsigned int issue_flags)
@@ -426,7 +419,7 @@ static struct iovec *__io_import_iovec(int ddir, struct io_kiocb *req,ssize_t ret;if (opcode ==IORING_OP_READ_FIXED || opcode ==IORING_OP_WRITE_FIXED ) { - ret =io_import_fixed (req, ddir, iter, issue_flags); + ret =io_import_fixed (ddir, iter, req->imu, rw->addr, rw->len);if (ret)return ERR_PTR (ret);return NULL ;
Signature · 1 line
2.36.1
PATCH 2/4 io_uring: move io_import_fixed()
In reply to Pavel BegunkovMove io_import_fixed() into rsrc.c where it belongs.
--- io_uring/rsrc.c | 60 +++++++++++++++++++++++++++++++++++++++++++++++++ io_uring/rsrc.h | 3 +++ io_uring/rw.c | 60 ------------------------------------------------- 3 files changed, 63 insertions(+), 60 deletions(-)
io_uring/rsrc.c +60 -0
@@ -1307,3 +1307,63 @@ int io_sqe_buffers_register(struct io_ring_ctx *ctx, void __user *arg,io_rsrc_node_switch (ctx,NULL );return ret; } + +int io_import_fixed (int ddir,struct iov_iter *iter, +struct io_mapped_ubuf *imu, +u64 buf_addr,size_t len) +{ +u64 buf_end; +size_t offset; + +if (WARN_ON_ONCE (!imu)) +return -EFAULT ; +if (unlikely (check_add_overflow (buf_addr, (u64 )len, &buf_end))) +return -EFAULT ; +/* not inside the mapped region */ +if (unlikely (buf_addr < imu->ubuf || buf_end > imu->ubuf_end)) +return -EFAULT ; + +/* +* May not be a start of buffer, set size appropriately +* and advance us to the beginning. +*/ + offset = buf_addr - imu->ubuf; +iov_iter_bvec (iter, ddir, imu->bvec, imu->nr_bvecs, offset + len); + +if (offset) { +/* +* Don't use iov_iter_advance() here, as it's really slow for +* using the latter parts of a big fixed buffer - it iterates +* over each segment manually. We can cheat a bit here, because +* we know that: +* +* 1) it's a BVEC iter, we set it up +* 2) all bvecs are PAGE_SIZE in size, except potentially the +* first and last bvec +* +* So just find our index, and adjust the iterator afterwards. +* If the offset is within the first bvec (or the whole first +* bvec, just use iov_iter_advance(). This makes it easier +* since we can just skip the first segment, which may not +* be PAGE_SIZE aligned. +*/ +const struct bio_vec *bvec = imu->bvec; + +if (offset <= bvec->bv_len) { +iov_iter_advance (iter, offset); + }else { +unsigned long seg_skip; + +/* skip first vec */ + offset -= bvec->bv_len; + seg_skip =1 + (offset >>PAGE_SHIFT ); + + iter->bvec = bvec + seg_skip; + iter->nr_segs -= seg_skip; + iter->count -= bvec->bv_len + offset; + iter->iov_offset = offset & ~PAGE_MASK ; + } + } + +return 0 ; +}
io_uring/rsrc.h +3 -0
@@ -56,6 +56,9 @@ int io_queue_rsrc_removal(struct io_rsrc_data *data, unsigned idx,void io_rsrc_node_switch (struct io_ring_ctx *ctx,struct io_rsrc_data *data_to_kill); +int io_import_fixed (int ddir,struct iov_iter *iter, +struct io_mapped_ubuf *imu, +u64 buf_addr,size_t len);void __io_sqe_buffers_unregister (struct io_ring_ctx *ctx);int io_sqe_buffers_unregister (struct io_ring_ctx *ctx);
io_uring/rw.c +0 -60
66 lines not on this page · raw message
Signature · 1 line
2.36.1
PATCH 3/4 io_uring: consistent naming for inline completion
In reply to Pavel BegunkovImprove naming of the inline/deferred completion helper so it's consistent with it's *_post counterpart. Add some comments and extra lockdeps to ensure the locking is done right.
--- io_uring/io_uring.c | 4 ++-- io_uring/io_uring.h | 10 +++++++++- 2 files changed, 11 insertions(+), 3 deletions(-)
io_uring/io_uring.c +2 -2
@@ -1391,7 +1391,7 @@ void io_req_task_complete(struct io_kiocb *req, bool *locked) }if (*locked) -io_req_add_compl_list (req); +io_req_complete_defer (req);else io_req_complete_post (req); }
@@ -1659,7 +1659,7 @@ static int io_issue_sqe(struct io_kiocb *req, unsigned int issue_flags)if (ret ==IOU_OK ) {if (issue_flags &IO_URING_F_COMPLETE_DEFER ) -io_req_add_compl_list (req); +io_req_complete_defer (req);else io_req_complete_post (req); }else if (ret !=IOU_ISSUE_SKIP_COMPLETE )
io_uring/io_uring.h +9 -1
@@ -168,10 +168,18 @@ static inline void io_tw_lock(struct io_ring_ctx *ctx, bool *locked) } } -static inline void io_req_add_compl_list (struct io_kiocb *req) +/* +* Don't complete immediately but use deferred completion infrastructure. +* Protected by ->uring_lock and can only be used either with +* IO_URING_F_COMPLETE_DEFER or inside a tw handler holding the mutex. +*/ +static inline void io_req_complete_defer (struct io_kiocb *req) + __must_hold(&req->ctx->uring_lock) {struct io_submit_state *state = &req->ctx->submit_state; +lockdep_assert_held (&req->ctx->uring_lock); +if (!(req->flags &REQ_F_CQE_SKIP )) state->flush_cqes = true;wq_list_add_tail (&req->comp_list, &state->compl_reqs);
Signature · 1 line
2.36.1
PATCH 4/4 io_uring: add an warn_once for poll_find
In reply to Pavel Begunkovio_poll_remove() expects poll_find() to search only for poll requests and passes a flag for this. Just be a little bit extra cautious considering lots of recent poll/cancellation changes and add a WARN_ON_ONCE checking that we don't get an apoll'ed request.
--- io_uring/poll.c | 5 +++++ 1 file changed, 5 insertions(+)
io_uring/poll.c +5 -0
@@ -824,6 +824,11 @@ int io_poll_remove(struct io_kiocb *req, unsigned int issue_flags) } found: +if (WARN_ON_ONCE (preq->opcode !=IORING_OP_POLL_ADD )) { + ret = -EFAULT ; + goto out; + } +if (poll_update->update_events || poll_update->update_user_data) {/* only mask one event flags, keep behavior flags */ if (poll_update->update_events) {
Signature · 1 line
2.36.1