Move io_buffer_list out of struct io_kiocb
Jens Axboe
maintainercover Move io_buffer_list out of struct io_kiocb
Hi,
One thing that has annoyed me is that struct io_buffer_list is inside struct io_kiocb, as they have potentially drastically different lifetimes. This makes it easy to screw up, even if you think you know what you are doing, as you need to understand the intricacies of provided buffer ring lifetimes.
This patchset adds a struct io_br_sel, which is used for buffer selection, and which also then stores the io_buffer_list whenever it is safe to do so. io_br_sel resides on the stack of the user, and hence cannot leak outside of that scope.
With this, we can also cleanup some of the random recycle points we have in the code base in general.
Should not have any functional changes, unless I screwed up of course. Passes full liburing tests as well.
include/linux/io_uring_types.h | 6 -- io_uring/io_uring.c | 4 +- io_uring/kbuf.c | 67 ++++++++-------- io_uring/kbuf.h | 55 ++++++++----- io_uring/net.c | 139 +++++++++++++++------------------ io_uring/poll.c | 4 - io_uring/rw.c | 56 +++++++------ 7 files changed, 170 insertions(+), 161 deletions(-)
Signature · 1 line
Jens Axboe
Jens Axboe
maintainerPATCH 1/9 io_uring/net: don't use io_net_kbuf_recyle() for non-provided cases
A previous commit used io_net_kbuf_recyle() for any network helper that did IO and needed partial retry. However, that's only needed if the opcode does buffer selection, which isnt support for sendzc, sendmsg_zc, or sendmsg. Just remove them - they don't do any harm, but it is a bit confusing when reading the code.
--- io_uring/net.c | 6 +++--- 1 file changed, 3 insertions(+), 3 deletions(-)
io_uring/net.c +3 -3
@@ -571,7 +571,7 @@ int io_sendmsg(struct io_kiocb *req, unsigned int issue_flags) kmsg->msg.msg_controllen =0 ; kmsg->msg.msg_control =NULL ; sr->done_io += ret; -return io_net_kbuf_recyle (req, kmsg, ret); +return -EAGAIN ; }if (ret == -ERESTARTSYS ) ret = -EINTR ;
@@ -1505,7 +1505,7 @@ int io_send_zc(struct io_kiocb *req, unsigned int issue_flags) zc->len -= ret; zc->buf += ret; zc->done_io += ret; -return io_net_kbuf_recyle (req, kmsg, ret); +return -EAGAIN ; }if (ret == -ERESTARTSYS ) ret = -EINTR ;
@@ -1575,7 +1575,7 @@ int io_sendmsg_zc(struct io_kiocb *req, unsigned int issue_flags)if (ret >0 &&io_net_retry (sock, flags)) { sr->done_io += ret; -return io_net_kbuf_recyle (req, kmsg, ret); +return -EAGAIN ; }if (ret == -ERESTARTSYS ) ret = -EINTR ;
Signature · 1 line
2.50.1
Jens Axboe
maintainerPATCH 2/9 io_uring/net: clarify io_recv_buf_select() return value
In reply to Jens AxboeIt returns 0 on success, less than zero on error.
--- io_uring/net.c | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-)
io_uring/net.c +1 -1
@@ -1199,7 +1199,7 @@ int io_recv(struct io_kiocb *req, unsigned int issue_flags) retry_multishot:if (io_do_buffer_select (req)) { ret =io_recv_buf_select (req, kmsg, &len, issue_flags); -if (unlikely (ret)) { +if (unlikely (ret <0 )) { kmsg->msg.msg_inq =-1 ; goto out_free; }
Signature · 1 line
2.50.1
Jens Axboe
maintainerPATCH 3/9 io_uring/kbuf: pass in struct io_buffer_list to commit/recycle helpers
In reply to Jens AxboeRather than have this implied being in the io_kiocb, pass it in directly so it's immediately obvious where these users of ->buf_list are coming from.
--- io_uring/io_uring.c | 6 +++--- io_uring/kbuf.c | 9 +++++---- io_uring/kbuf.h | 21 +++++++++++++-------- io_uring/net.c | 28 ++++++++++++++-------------- io_uring/poll.c | 6 +++--- io_uring/rw.c | 14 +++++++------- 6 files changed, 45 insertions(+), 39 deletions(-)
io_uring/io_uring.c +3 -3
@@ -1007,7 +1007,7 @@ void io_req_defer_failed(struct io_kiocb *req, s32 res)lockdep_assert_held (&req->ctx->uring_lock);req_set_fail (req); -io_req_set_res (req, res,io_put_kbuf (req, res,IO_URING_F_UNLOCKED )); +io_req_set_res (req, res,io_put_kbuf (req, res, req->buf_list,IO_URING_F_UNLOCKED ));if (def->fail) def->fail (req);io_req_complete_defer (req);
@@ -2025,11 +2025,11 @@ static void io_queue_async(struct io_kiocb *req, unsigned int issue_flags, int rswitch (io_arm_poll_handler (req,0 )) {case IO_APOLL_READY : -io_kbuf_recycle (req,0 ); +io_kbuf_recycle (req, req->buf_list,0 );io_req_task_queue (req);break ;case IO_APOLL_ABORTED : -io_kbuf_recycle (req,0 ); +io_kbuf_recycle (req, req->buf_list,0 );io_queue_iowq (req);break ;case IO_APOLL_OK :
io_uring/kbuf.c +5 -4
@@ -354,9 +354,9 @@ int io_buffers_peek(struct io_kiocb *req, struct buf_sel_arg *arg)return io_provided_buffers_select (req, &arg->max_len, bl, arg->iovs); } -static inline bool __io_put_kbuf_ring (struct io_kiocb *req,int len,int nr) +static inline bool __io_put_kbuf_ring (struct io_kiocb *req, +struct io_buffer_list *bl,int len,int nr) { -struct io_buffer_list *bl = req->buf_list;bool ret = true;if (bl)
@@ -366,7 +366,8 @@ static inline bool __io_put_kbuf_ring(struct io_kiocb *req, int len, int nr)return ret; } -unsigned int __io_put_kbufs (struct io_kiocb *req,int len,int nbufs) +unsigned int __io_put_kbufs (struct io_kiocb *req,struct io_buffer_list *bl, +int len,int nbufs) {unsigned int ret;
@@ -377,7 +378,7 @@ unsigned int __io_put_kbufs(struct io_kiocb *req, int len, int nbufs)return ret; } -if (!__io_put_kbuf_ring (req, len, nbufs)) +if (!__io_put_kbuf_ring (req, bl, len, nbufs)) ret |=IORING_CQE_F_BUF_MORE ;return ret; }
io_uring/kbuf.h +13 -8
@@ -80,14 +80,16 @@ int io_register_pbuf_status(struct io_ring_ctx *ctx, void __user *arg);bool io_kbuf_recycle_legacy (struct io_kiocb *req,unsigned issue_flags);void io_kbuf_drop_legacy (struct io_kiocb *req); -unsigned int __io_put_kbufs (struct io_kiocb *req,int len,int nbufs); +unsigned int __io_put_kbufs (struct io_kiocb *req,struct io_buffer_list *bl, +int len,int nbufs);bool io_kbuf_commit (struct io_kiocb *req,struct io_buffer_list *bl,int len,int nr);struct io_mapped_region *io_pbuf_get_region (struct io_ring_ctx *ctx,unsigned int bgid); -static inline bool io_kbuf_recycle_ring (struct io_kiocb *req) +static inline bool io_kbuf_recycle_ring (struct io_kiocb *req, +struct io_buffer_list *bl) {/* * We don't need to recycle for REQ_F_BUFFER_RING, we can just clear
@@ -96,7 +98,7 @@ static inline bool io_kbuf_recycle_ring(struct io_kiocb *req)* The exception is partial io, that case we should increment bl->head * to monopolize the buffer. */ -if (req->buf_list) { +if (bl) { req->flags &= ~(REQ_F_BUFFER_RING |REQ_F_BUFFERS_COMMIT );return true; }
@@ -110,30 +112,33 @@ static inline bool io_do_buffer_select(struct io_kiocb *req)return !(req->flags & (REQ_F_BUFFER_SELECTED |REQ_F_BUFFER_RING )); } -static inline bool io_kbuf_recycle (struct io_kiocb *req,unsigned issue_flags) +static inline bool io_kbuf_recycle (struct io_kiocb *req,struct io_buffer_list *bl, +unsigned issue_flags) {if (req->flags &REQ_F_BL_NO_RECYCLE )return false;if (req->flags &REQ_F_BUFFER_SELECTED )return io_kbuf_recycle_legacy (req, issue_flags);if (req->flags &REQ_F_BUFFER_RING ) -return io_kbuf_recycle_ring (req); +return io_kbuf_recycle_ring (req, bl);return false; }static inline unsigned int io_put_kbuf (struct io_kiocb *req,int len, +struct io_buffer_list *bl,unsigned issue_flags) {if (!(req->flags & (REQ_F_BUFFER_RING |REQ_F_BUFFER_SELECTED )))return 0 ; -return __io_put_kbufs (req, len,1 ); +return __io_put_kbufs (req, bl, len,1 ); }static inline unsigned int io_put_kbufs (struct io_kiocb *req,int len, -int nbufs,unsigned issue_flags) +int nbufs,struct io_buffer_list *bl, +unsigned issue_flags) {if (!(req->flags & (REQ_F_BUFFER_RING |REQ_F_BUFFER_SELECTED )))return 0 ; -return __io_put_kbufs (req, len, nbufs); +return __io_put_kbufs (req, bl, len, nbufs); }#endif
io_uring/net.c +14 -14
@@ -494,12 +494,12 @@ static int io_bundle_nbufs(struct io_async_msghdr *kmsg, int ret)return nbufs; } -static int io_net_kbuf_recyle (struct io_kiocb *req, +static int io_net_kbuf_recyle (struct io_kiocb *req,struct io_buffer_list *bl,struct io_async_msghdr *kmsg,int len) { req->flags |=REQ_F_BL_NO_RECYCLE ;if (req->flags &REQ_F_BUFFERS_COMMIT ) -io_kbuf_commit (req, req->buf_list, len,io_bundle_nbufs (kmsg, len)); +io_kbuf_commit (req, bl, len,io_bundle_nbufs (kmsg, len));return IOU_RETRY ; }
@@ -512,11 +512,11 @@ static inline bool io_send_finish(struct io_kiocb *req, int *ret,unsigned int cflags;if (!(sr->flags &IORING_RECVSEND_BUNDLE )) { - cflags =io_put_kbuf (req, *ret, issue_flags); + cflags =io_put_kbuf (req, *ret, req->buf_list, issue_flags); goto finish; } - cflags =io_put_kbufs (req, *ret,io_bundle_nbufs (kmsg, *ret), issue_flags); + cflags =io_put_kbufs (req, *ret,io_bundle_nbufs (kmsg, *ret), req->buf_list, issue_flags);if (bundle_finished || req->flags &REQ_F_BL_EMPTY ) goto finish;
@@ -682,7 +682,7 @@ int io_send(struct io_kiocb *req, unsigned int issue_flags) sr->len -= ret; sr->buf += ret; sr->done_io += ret; -return io_net_kbuf_recyle (req, kmsg, ret); +return io_net_kbuf_recyle (req, req->buf_list, kmsg, ret); }if (ret == -ERESTARTSYS ) ret = -EINTR ;
@@ -873,7 +873,7 @@ static inline bool io_recv_finish(struct io_kiocb *req, int *ret,size_t this_ret = *ret - sr->done_io; cflags |=io_put_kbufs (req, this_ret,io_bundle_nbufs (kmsg, this_ret), - issue_flags); + req->buf_list, issue_flags);if (sr->flags &IORING_RECV_RETRY ) cflags = req->cqe.flags | (cflags &CQE_F_MASK );if (sr->mshot_len && *ret >= sr->mshot_len)
@@ -895,7 +895,7 @@ static inline bool io_recv_finish(struct io_kiocb *req, int *ret,return false; } }else { - cflags |=io_put_kbuf (req, *ret, issue_flags); + cflags |=io_put_kbuf (req, *ret, req->buf_list, issue_flags); }/*
@@ -1047,7 +1047,7 @@ int io_recvmsg(struct io_kiocb *req, unsigned int issue_flags)if (req->flags &REQ_F_APOLL_MULTISHOT ) { ret =io_recvmsg_prep_multishot (kmsg, sr, &buf, &len);if (ret) { -io_kbuf_recycle (req, issue_flags); +io_kbuf_recycle (req, req->buf_list, issue_flags);return ret; } }
@@ -1072,13 +1072,13 @@ int io_recvmsg(struct io_kiocb *req, unsigned int issue_flags)if (ret < min_ret) {if (ret == -EAGAIN && force_nonblock) {if (issue_flags &IO_URING_F_MULTISHOT ) -io_kbuf_recycle (req, issue_flags); +io_kbuf_recycle (req, req->buf_list, issue_flags);return IOU_RETRY ; }if (ret >0 &&io_net_retry (sock, flags)) { sr->done_io += ret; -return io_net_kbuf_recyle (req, kmsg, ret); +return io_net_kbuf_recyle (req, req->buf_list, kmsg, ret); }if (ret == -ERESTARTSYS ) ret = -EINTR ;
@@ -1092,7 +1092,7 @@ int io_recvmsg(struct io_kiocb *req, unsigned int issue_flags)else if (sr->done_io) ret = sr->done_io;else -io_kbuf_recycle (req, issue_flags); +io_kbuf_recycle (req, req->buf_list, issue_flags);if (!io_recv_finish (req, &ret, kmsg, mshot_finished, issue_flags)) goto retry_multishot;
@@ -1216,7 +1216,7 @@ int io_recv(struct io_kiocb *req, unsigned int issue_flags)if (ret < min_ret) {if (ret == -EAGAIN && force_nonblock) {if (issue_flags &IO_URING_F_MULTISHOT ) -io_kbuf_recycle (req, issue_flags); +io_kbuf_recycle (req, req->buf_list, issue_flags);return IOU_RETRY ; }
@@ -1224,7 +1224,7 @@ int io_recv(struct io_kiocb *req, unsigned int issue_flags) sr->len -= ret; sr->buf += ret; sr->done_io += ret; -return io_net_kbuf_recyle (req, kmsg, ret); +return io_net_kbuf_recyle (req, req->buf_list, kmsg, ret); }if (ret == -ERESTARTSYS ) ret = -EINTR ;
@@ -1240,7 +1240,7 @@ int io_recv(struct io_kiocb *req, unsigned int issue_flags)else if (sr->done_io) ret = sr->done_io;else -io_kbuf_recycle (req, issue_flags); +io_kbuf_recycle (req, req->buf_list, issue_flags);if (!io_recv_finish (req, &ret, kmsg, mshot_finished, issue_flags)) goto retry_multishot;
io_uring/poll.c +3 -3
@@ -316,10 +316,10 @@ void io_poll_task_func(struct io_kiocb *req, io_tw_token_t tw) ret =io_poll_check_events (req, tw);if (ret ==IOU_POLL_NO_ACTION ) { -io_kbuf_recycle (req,0 ); +io_kbuf_recycle (req, req->buf_list,0 );return ; }else if (ret ==IOU_POLL_REQUEUE ) { -io_kbuf_recycle (req,0 ); +io_kbuf_recycle (req, req->buf_list,0 );__io_poll_execute (req,0 );return ; }
@@ -686,7 +686,7 @@ int io_arm_apoll(struct io_kiocb *req, unsigned issue_flags, __poll_t mask) req->flags |=REQ_F_POLLED ; ipt.pt._qproc = io_async_queue_proc; -io_kbuf_recycle (req, issue_flags); +io_kbuf_recycle (req, req->buf_list, issue_flags); ret =__io_arm_poll_handler (req, &apoll->poll, &ipt, mask, issue_flags);if (ret)
io_uring/rw.c +7 -7
@@ -576,7 +576,7 @@ void io_req_rw_complete(struct io_kiocb *req, io_tw_token_t tw)io_req_io_end (req);if (req->flags & (REQ_F_BUFFER_SELECTED |REQ_F_BUFFER_RING )) - req->cqe.flags |=io_put_kbuf (req, req->cqe.res,0 ); + req->cqe.flags |=io_put_kbuf (req, req->cqe.res, req->buf_list,0 );io_req_rw_cleanup (req,0 );io_req_task_complete (req, tw);
@@ -659,7 +659,7 @@ static int kiocb_done(struct io_kiocb *req, ssize_t ret,* from the submission path. */ io_req_io_end (req); -io_req_set_res (req, final_ret,io_put_kbuf (req, ret, issue_flags)); +io_req_set_res (req, final_ret,io_put_kbuf (req, ret, req->buf_list, issue_flags));io_req_rw_cleanup (req, issue_flags);return IOU_COMPLETE ; }else {
@@ -1049,15 +1049,15 @@ int io_read_mshot(struct io_kiocb *req, unsigned int issue_flags)* Reset rw->len to 0 again to avoid clamping future mshot * reads, in case the buffer size varies. */ -if (io_kbuf_recycle (req, issue_flags)) +if (io_kbuf_recycle (req, req->buf_list, issue_flags)) rw->len =0 ;return IOU_RETRY ; }else if (ret <=0 ) { -io_kbuf_recycle (req, issue_flags); +io_kbuf_recycle (req, req->buf_list, issue_flags);if (ret <0 )req_set_fail (req); }else if (!(req->flags &REQ_F_APOLL_MULTISHOT )) { - cflags =io_put_kbuf (req, ret, issue_flags); + cflags =io_put_kbuf (req, ret, req->buf_list, issue_flags); }else {/* * Any successful return value will keep the multishot read
@@ -1065,7 +1065,7 @@ int io_read_mshot(struct io_kiocb *req, unsigned int issue_flags)* we fail to post a CQE, or multishot is no longer set, then * jump to the termination path. This request is then done. */ - cflags =io_put_kbuf (req, ret, issue_flags); + cflags =io_put_kbuf (req, ret, req->buf_list, issue_flags); rw->len =0 ;/* similarly to above, reset len to 0 */ if (io_req_post_cqe (req, ret, cflags |IORING_CQE_F_MORE )) {
@@ -1362,7 +1362,7 @@ int io_do_iopoll(struct io_ring_ctx *ctx, bool force_nonspin)if (!smp_load_acquire (&req->iopoll_completed))break ; nr_events++; - req->cqe.flags =io_put_kbuf (req, req->cqe.res,0 ); + req->cqe.flags =io_put_kbuf (req, req->cqe.res, req->buf_list,0 );if (req->opcode !=IORING_OP_URING_CMD )io_req_rw_cleanup (req,0 ); }
Signature · 1 line
2.50.1
Jens Axboe
maintainerPATCH 4/9 io_uring/kbuf: introduce struct io_br_sel
In reply to Jens AxboeRather than return addresses directly from buffer selection, add a struct around it. No functional changes in this patch, it's in preparation for storing more buffer related information locally, rather than in struct io_kiocb.
--- io_uring/kbuf.c | 26 +++++++++++++------------- io_uring/kbuf.h | 19 +++++++++++++++++-- io_uring/net.c | 18 +++++++++--------- io_uring/rw.c | 34 ++++++++++++++++++++-------------- 4 files changed, 59 insertions(+), 38 deletions(-)
io_uring/kbuf.c +13 -13
@@ -151,18 +151,18 @@ static int io_provided_buffers_select(struct io_kiocb *req, size_t *len,return 1 ; } -static void __user *io_ring_buffer_select (struct io_kiocb *req,size_t *len, -struct io_buffer_list *bl, -unsigned int issue_flags) +static struct io_br_sel io_ring_buffer_select (struct io_kiocb *req,size_t *len, +struct io_buffer_list *bl, +unsigned int issue_flags) {struct io_uring_buf_ring *br = bl->buf_ring;__u16 tail, head = bl->head; +struct io_br_sel sel = { };struct io_uring_buf *buf; -void __user *ret; tail =smp_load_acquire (&br->tail);if (unlikely (tail == head)) -return NULL ; +return sel;if (head +1 == tail) req->flags |=REQ_F_BL_EMPTY ;
@@ -173,7 +173,7 @@ static void __user *io_ring_buffer_select(struct io_kiocb *req, size_t *len, req->flags |=REQ_F_BUFFER_RING |REQ_F_BUFFERS_COMMIT ; req->buf_list = bl; req->buf_index = buf->bid; - ret =u64_to_user_ptr (buf->addr); + sel.addr =u64_to_user_ptr (buf->addr);if (issue_flags &IO_URING_F_UNLOCKED || !io_file_can_poll (req)) {/*
@@ -189,27 +189,27 @@ static void __user *io_ring_buffer_select(struct io_kiocb *req, size_t *len,io_kbuf_commit (req, bl, *len,1 ); req->buf_list =NULL ; } -return ret; +return sel; } -void __user *io_buffer_select (struct io_kiocb *req,size_t *len, -unsigned buf_group,unsigned int issue_flags) +struct io_br_sel io_buffer_select (struct io_kiocb *req,size_t *len, +unsigned buf_group,unsigned int issue_flags) {struct io_ring_ctx *ctx = req->ctx; +struct io_br_sel sel = { };struct io_buffer_list *bl; -void __user *ret =NULL ;io_ring_submit_lock (req->ctx, issue_flags); bl =io_buffer_get_list (ctx, buf_group);if (likely (bl)) {if (bl->flags &IOBL_BUF_RING ) - ret =io_ring_buffer_select (req, len, bl, issue_flags); + sel =io_ring_buffer_select (req, len, bl, issue_flags);else - ret =io_provided_buffer_select (req, len, bl); + sel.addr =io_provided_buffer_select (req, len, bl); }io_ring_submit_unlock (req->ctx, issue_flags); -return ret; +return sel; }/* cap it at a reasonable 256, will be one page even for 4K */
io_uring/kbuf.h +17 -2
@@ -62,8 +62,23 @@ struct buf_sel_arg {unsigned short partial_map; }; -void __user *io_buffer_select (struct io_kiocb *req,size_t *len, -unsigned buf_group,unsigned int issue_flags); +/* +* Return value from io_buffer_list selection. Just returns the error or +* user address for now, will be extended to return the buffer list in the +* future. +*/ +struct io_br_sel { +/* +* Some selection parts return the user address, others return an error. +*/ +union { +void __user *addr; +ssize_t val; + }; +}; + +struct io_br_sel io_buffer_select (struct io_kiocb *req,size_t *len, +unsigned buf_group,unsigned int issue_flags);int io_buffers_select (struct io_kiocb *req,struct buf_sel_arg *arg,unsigned int issue_flags);int io_buffers_peek (struct io_kiocb *req,struct buf_sel_arg *arg);
io_uring/net.c +9 -9
@@ -1037,22 +1037,22 @@ int io_recvmsg(struct io_kiocb *req, unsigned int issue_flags) retry_multishot:if (io_do_buffer_select (req)) { -void __user *buf; +struct io_br_sel sel;size_t len = sr->len; - buf =io_buffer_select (req, &len, sr->buf_group, issue_flags); -if (!buf) + sel =io_buffer_select (req, &len, sr->buf_group, issue_flags); +if (!sel.addr)return -ENOBUFS ;if (req->flags &REQ_F_APOLL_MULTISHOT ) { - ret =io_recvmsg_prep_multishot (kmsg, sr, &buf, &len); + ret =io_recvmsg_prep_multishot (kmsg, sr, &sel.addr, &len);if (ret) {io_kbuf_recycle (req, req->buf_list, issue_flags);return ret; } } -iov_iter_ubuf (&kmsg->msg.msg_iter,ITER_DEST , buf, len); +iov_iter_ubuf (&kmsg->msg.msg_iter,ITER_DEST , sel.addr, len); } kmsg->msg.msg_get_inq =1 ;
@@ -1155,13 +1155,13 @@ static int io_recv_buf_select(struct io_kiocb *req, struct io_async_msghdr *kmsgiov_iter_init (&kmsg->msg.msg_iter,ITER_DEST , arg.iovs, ret, arg.out_len); }else { -void __user *buf; +struct io_br_sel sel; *len = sr->len; - buf =io_buffer_select (req, len, sr->buf_group, issue_flags); -if (!buf) + sel =io_buffer_select (req, len, sr->buf_group, issue_flags); +if (!sel.addr)return -ENOBUFS ; - sr->buf = buf; + sr->buf = sel.addr; sr->len = *len; map_ubuf: ret = import_ubuf(ITER_DEST, sr->buf, sr->len,
io_uring/rw.c +20 -14
@@ -107,34 +107,35 @@ static int io_import_vec(int ddir, struct io_kiocb *req, }static int __io_import_rw_buffer (int ddir,struct io_kiocb *req, -struct io_async_rw *io, -unsigned int issue_flags) +struct io_async_rw *io,struct io_br_sel *sel, +unsigned int issue_flags) {const struct io_issue_def *def = &io_issue_defs[req->opcode]; struct io_rw *rw = io_kiocb_to_cmd(req, struct io_rw); -void __user *buf =u64_to_user_ptr (rw->addr);size_t sqe_len = rw->len; + sel->addr =u64_to_user_ptr (rw->addr);if (def->vectored && !(req->flags &REQ_F_BUFFER_SELECT )) -return io_import_vec (ddir, req, io, buf, sqe_len); +return io_import_vec (ddir, req, io, sel->addr, sqe_len);if (io_do_buffer_select (req)) { - buf =io_buffer_select (req, &sqe_len, io->buf_group, issue_flags); -if (!buf) + *sel =io_buffer_select (req, &sqe_len, io->buf_group, issue_flags); +if (!sel->addr)return -ENOBUFS ; - rw->addr = (unsigned long ) buf; + rw->addr = (unsigned long ) sel->addr; rw->len = sqe_len; } -return import_ubuf (ddir, buf, sqe_len, &io->iter); +return import_ubuf (ddir, sel->addr, sqe_len, &io->iter); }static inline int io_import_rw_buffer (int rw,struct io_kiocb *req,struct io_async_rw *io, +struct io_br_sel *sel,unsigned int issue_flags) {int ret; - ret =__io_import_rw_buffer (rw, req, io, issue_flags); + ret =__io_import_rw_buffer (rw, req, io, sel, issue_flags);if (unlikely (ret <0 ))return ret;
@@ -306,10 +307,12 @@ static int __io_prep_rw(struct io_kiocb *req, const struct io_uring_sqe *sqe,static int io_rw_do_import (struct io_kiocb *req,int ddir) { +struct io_br_sel sel = { }; +if (io_do_buffer_select (req))return 0 ; -return io_import_rw_buffer (ddir, req, req->async_data,0 ); +return io_import_rw_buffer (ddir, req, req->async_data, &sel,0 ); } static int io_prep_rw(struct io_kiocb *req, const struct io_uring_sqe *sqe,
@@ -899,7 +902,8 @@ static int io_rw_init_file(struct io_kiocb *req, fmode_t mode, int rw_type)return 0 ; } -static int __io_read (struct io_kiocb *req,unsigned int issue_flags) +static int __io_read (struct io_kiocb *req,struct io_br_sel *sel, +unsigned int issue_flags) {bool force_nonblock = issue_flags &IO_URING_F_NONBLOCK ; struct io_rw *rw = io_kiocb_to_cmd(req, struct io_rw);
@@ -913,7 +917,7 @@ static int __io_read(struct io_kiocb *req, unsigned int issue_flags)if (unlikely (ret))return ret; }else if (io_do_buffer_select (req)) { - ret =io_import_rw_buffer (ITER_DEST , req, io, issue_flags); + ret =io_import_rw_buffer (ITER_DEST , req, io, sel, issue_flags);if (unlikely (ret <0 ))return ret; }
@@ -1015,9 +1019,10 @@ static int __io_read(struct io_kiocb *req, unsigned int issue_flags)int io_read (struct io_kiocb *req,unsigned int issue_flags) { +struct io_br_sel sel = { };int ret; - ret =__io_read (req, issue_flags); + ret =__io_read (req, &sel, issue_flags);if (ret >=0 )return kiocb_done (req, ret, issue_flags);
@@ -1027,6 +1032,7 @@ int io_read(struct io_kiocb *req, unsigned int issue_flags)int io_read_mshot (struct io_kiocb *req,unsigned int issue_flags) { struct io_rw *rw = io_kiocb_to_cmd(req, struct io_rw); +struct io_br_sel sel = { };unsigned int cflags =0 ;int ret;
@@ -1038,7 +1044,7 @@ int io_read_mshot(struct io_kiocb *req, unsigned int issue_flags)/* make it sync, multishot doesn't support async execution */ rw->kiocb.ki_complete =NULL ; - ret =__io_read (req, issue_flags); + ret =__io_read (req, &sel, issue_flags);/* * If we get -EAGAIN, recycle our buffer and just let normal poll
Signature · 1 line
2.50.1
Jens Axboe
maintainerPATCH 5/9 io_uring/rw: recycle buffers manually for non-mshot reads
In reply to Jens AxboeThe mshot side of reads already does this, but the regular read path does not. This leads to needing recycling checks sprinkled in various spots in the "go async" path, like arming poll. In preparation for getting rid of those, ensure that read recycles appropriately.
--- io_uring/rw.c | 2 ++ 1 file changed, 2 insertions(+)
io_uring/rw.c +2 -0
@@ -1026,6 +1026,8 @@ int io_read(struct io_kiocb *req, unsigned int issue_flags)if (ret >=0 )return kiocb_done (req, ret, issue_flags); +if (req->flags &REQ_F_BUFFERS_COMMIT ) +io_kbuf_recycle (req, req->buf_list, issue_flags);return ret; }
Signature · 1 line
2.50.1
Jens Axboe
maintainerPATCH 6/9 io_uring/kbuf: use struct io_br_sel for multiple buffers picking
In reply to Jens AxboeThe networking side uses bundles, which is picking multiple buffers at the same time. Pass in struct io_br_sel to those helpers.
--- io_uring/kbuf.c | 5 +++-- io_uring/kbuf.h | 5 +++-- io_uring/net.c | 38 +++++++++++++++++++------------------- 3 files changed, 25 insertions(+), 23 deletions(-)
io_uring/kbuf.c +3 -2
@@ -299,7 +299,7 @@ static int io_ring_buffers_peek(struct io_kiocb *req, struct buf_sel_arg *arg, }int io_buffers_select (struct io_kiocb *req,struct buf_sel_arg *arg, -unsigned int issue_flags) +struct io_br_sel *sel,unsigned int issue_flags) {struct io_ring_ctx *ctx = req->ctx;struct io_buffer_list *bl;
@@ -331,7 +331,8 @@ int io_buffers_select(struct io_kiocb *req, struct buf_sel_arg *arg,return ret; } -int io_buffers_peek (struct io_kiocb *req,struct buf_sel_arg *arg) +int io_buffers_peek (struct io_kiocb *req,struct buf_sel_arg *arg, +struct io_br_sel *sel) {struct io_ring_ctx *ctx = req->ctx;struct io_buffer_list *bl;
io_uring/kbuf.h +3 -2
@@ -80,8 +80,9 @@ struct io_br_sel {struct io_br_sel io_buffer_select (struct io_kiocb *req,size_t *len,unsigned buf_group,unsigned int issue_flags);int io_buffers_select (struct io_kiocb *req,struct buf_sel_arg *arg, -unsigned int issue_flags); -int io_buffers_peek (struct io_kiocb *req,struct buf_sel_arg *arg); +struct io_br_sel *sel,unsigned int issue_flags); +int io_buffers_peek (struct io_kiocb *req,struct buf_sel_arg *arg, +struct io_br_sel *sel);void io_destroy_buffers (struct io_ring_ctx *ctx);int io_remove_buffers_prep (struct io_kiocb *req,const struct io_uring_sqe *sqe);
io_uring/net.c +19 -19
@@ -587,17 +587,16 @@ int io_sendmsg(struct io_kiocb *req, unsigned int issue_flags) }static int io_send_select_buffer (struct io_kiocb *req,unsigned int issue_flags, -struct io_async_msghdr *kmsg) +struct io_br_sel *sel,struct io_async_msghdr *kmsg) { struct io_sr_msg *sr = io_kiocb_to_cmd(req, struct io_sr_msg); - -int ret;struct buf_sel_arg arg = { .iovs = &kmsg->fast_iov, .max_len =min_not_zero (sr->len,INT_MAX ), .nr_iovs =1 , .buf_group = sr->buf_group, }; +int ret;if (kmsg->vec.iovec) { arg.nr_iovs = kmsg->vec.nr;
@@ -610,7 +609,7 @@ static int io_send_select_buffer(struct io_kiocb *req, unsigned int issue_flags,else arg.mode |=KBUF_MODE_EXPAND ; - ret =io_buffers_select (req, &arg, issue_flags); + ret =io_buffers_select (req, &arg, sel, issue_flags);if (unlikely (ret <0 ))return ret;
@@ -639,6 +638,7 @@ int io_send(struct io_kiocb *req, unsigned int issue_flags) { struct io_sr_msg *sr = io_kiocb_to_cmd(req, struct io_sr_msg);struct io_async_msghdr *kmsg = req->async_data; +struct io_br_sel sel = { };struct socket *sock;unsigned flags;int min_ret =0 ;
@@ -658,7 +658,7 @@ int io_send(struct io_kiocb *req, unsigned int issue_flags) retry_bundle:if (io_do_buffer_select (req)) { - ret =io_send_select_buffer (req, issue_flags, kmsg); + ret =io_send_select_buffer (req, issue_flags, &sel, kmsg);if (ret)return ret; }
@@ -1017,6 +1017,7 @@ int io_recvmsg(struct io_kiocb *req, unsigned int issue_flags) { struct io_sr_msg *sr = io_kiocb_to_cmd(req, struct io_sr_msg);struct io_async_msghdr *kmsg = req->async_data; +struct io_br_sel sel = { };struct socket *sock;unsigned flags;int ret, min_ret =0 ;
@@ -1037,7 +1038,6 @@ int io_recvmsg(struct io_kiocb *req, unsigned int issue_flags) retry_multishot:if (io_do_buffer_select (req)) { -struct io_br_sel sel;size_t len = sr->len; sel =io_buffer_select (req, &len, sr->buf_group, issue_flags);
@@ -1101,7 +1101,7 @@ int io_recvmsg(struct io_kiocb *req, unsigned int issue_flags) }static int io_recv_buf_select (struct io_kiocb *req,struct io_async_msghdr *kmsg, -size_t *len,unsigned int issue_flags) +struct io_br_sel *sel,unsigned int issue_flags) { struct io_sr_msg *sr = io_kiocb_to_cmd(req, struct io_sr_msg);int ret;
@@ -1126,15 +1126,15 @@ static int io_recv_buf_select(struct io_kiocb *req, struct io_async_msghdr *kmsg arg.mode |=KBUF_MODE_FREE ; } -if (*len) - arg.max_len = *len; +if (sel->val) + arg.max_len = sel->val;else if (kmsg->msg.msg_inq >1 ) - arg.max_len =min_not_zero (*len, (size_t ) kmsg->msg.msg_inq); + arg.max_len =min_not_zero (sel->val, (size_t ) kmsg->msg.msg_inq);/* if mshot limited, ensure we don't go over */ if (sr->flags &IORING_RECV_MSHOT_LIM ) arg.max_len =min_not_zero (arg.max_len, sr->mshot_total_len); - ret =io_buffers_peek (req, &arg); + ret =io_buffers_peek (req, &arg, sel);if (unlikely (ret <0 ))return ret;
@@ -1155,14 +1155,13 @@ static int io_recv_buf_select(struct io_kiocb *req, struct io_async_msghdr *kmsgiov_iter_init (&kmsg->msg.msg_iter,ITER_DEST , arg.iovs, ret, arg.out_len); }else { -struct io_br_sel sel; +size_t len = sel->val; - *len = sr->len; - sel =io_buffer_select (req, len, sr->buf_group, issue_flags); -if (!sel.addr) + *sel =io_buffer_select (req, &len, sr->buf_group, issue_flags); +if (!sel->addr)return -ENOBUFS ; - sr->buf = sel.addr; - sr->len = *len; + sr->buf = sel->addr; + sr->len = len; map_ubuf: ret =import_ubuf (ITER_DEST , sr->buf, sr->len, &kmsg->msg.msg_iter);
@@ -1177,11 +1176,11 @@ int io_recv(struct io_kiocb *req, unsigned int issue_flags) { struct io_sr_msg *sr = io_kiocb_to_cmd(req, struct io_sr_msg);struct io_async_msghdr *kmsg = req->async_data; +struct io_br_sel sel = { };struct socket *sock;unsigned flags;int ret, min_ret =0 ;bool force_nonblock = issue_flags &IO_URING_F_NONBLOCK ; -size_t len = sr->len;bool mshot_finished; if (!(req->flags & REQ_F_POLLED) &&
@@ -1198,7 +1197,8 @@ int io_recv(struct io_kiocb *req, unsigned int issue_flags) retry_multishot:if (io_do_buffer_select (req)) { - ret =io_recv_buf_select (req, kmsg, &len, issue_flags); + sel.val = sr->len; + ret =io_recv_buf_select (req, kmsg, &sel, issue_flags);if (unlikely (ret <0 )) { kmsg->msg.msg_inq =-1 ; goto out_free;
Signature · 1 line
2.50.1
Jens Axboe
maintainerPATCH 7/9 io_uring/net: use struct io_br_sel->val as the recv finish value
In reply to Jens AxboeCurrently a pointer is passed in to the 'ret' in the receive handlers, but since we already have a value field in io_br_sel, just use that. This is also in preparation for needing to pass in struct io_br_sel to io_recv_finish() anyway.
--- io_uring/net.c | 35 +++++++++++++++++++---------------- 1 file changed, 19 insertions(+), 16 deletions(-)
io_uring/net.c +19 -16
@@ -846,9 +846,10 @@ int io_recvmsg_prep(struct io_kiocb *req, const struct io_uring_sqe *sqe)* Returns true if it is actually finished, or false if it should run * again (for multishot). */ -static inline bool io_recv_finish (struct io_kiocb *req,int *ret, +static inline bool io_recv_finish (struct io_kiocb *req,struct io_async_msghdr *kmsg, -bool mshot_finished,unsigned issue_flags) +struct io_br_sel *sel,bool mshot_finished, +unsigned issue_flags) { struct io_sr_msg *sr = io_kiocb_to_cmd(req, struct io_sr_msg);unsigned int cflags =0 ;
@@ -856,13 +857,13 @@ static inline bool io_recv_finish(struct io_kiocb *req, int *ret,if (kmsg->msg.msg_inq >0 ) cflags |=IORING_CQE_F_SOCK_NONEMPTY ; -if (*ret >0 && sr->flags &IORING_RECV_MSHOT_LIM ) { +if (sel->val >0 && sr->flags &IORING_RECV_MSHOT_LIM ) {/* * If sr->len hits zero, the limit has been reached. Mark * mshot as finished, and flag MSHOT_DONE as well to prevent * a potential bundle from being retried. */ - sr->mshot_total_len -=min_t (int, *ret, sr->mshot_total_len); + sr->mshot_total_len -=min_t (int, sel->val, sr->mshot_total_len);if (!sr->mshot_total_len) { sr->flags |=IORING_RECV_MSHOT_DONE ; mshot_finished = true;
@@ -870,13 +871,13 @@ static inline bool io_recv_finish(struct io_kiocb *req, int *ret, }if (sr->flags &IORING_RECVSEND_BUNDLE ) { -size_t this_ret = *ret - sr->done_io; +size_t this_ret = sel->val - sr->done_io; cflags |=io_put_kbufs (req, this_ret,io_bundle_nbufs (kmsg, this_ret), req->buf_list, issue_flags);if (sr->flags &IORING_RECV_RETRY ) cflags = req->cqe.flags | (cflags &CQE_F_MASK ); -if (sr->mshot_len && *ret >= sr->mshot_len) +if (sr->mshot_len && sel->val >= sr->mshot_len) sr->flags |=IORING_RECV_MSHOT_CAP ;/* bundle with no more immediate buffers, we're done */ if (req->flags &REQ_F_BL_EMPTY )
@@ -895,7 +896,7 @@ static inline bool io_recv_finish(struct io_kiocb *req, int *ret,return false; } }else { - cflags |=io_put_kbuf (req, *ret, req->buf_list, issue_flags); + cflags |=io_put_kbuf (req, sel->val, req->buf_list, issue_flags); }/*
@@ -903,8 +904,8 @@ static inline bool io_recv_finish(struct io_kiocb *req, int *ret,* receive from this socket. */ if ((req->flags &REQ_F_APOLL_MULTISHOT ) && !mshot_finished && -io_req_post_cqe (req, *ret, cflags |IORING_CQE_F_MORE )) { - *ret =IOU_RETRY ; +io_req_post_cqe (req, sel->val, cflags |IORING_CQE_F_MORE )) { + sel->val =IOU_RETRY ;io_mshot_prep_retry (req, kmsg);/* Known not-empty or unknown state, retry */ if (cflags &IORING_CQE_F_SOCK_NONEMPTY || kmsg->msg.msg_inq <0 ) {
@@ -916,15 +917,15 @@ static inline bool io_recv_finish(struct io_kiocb *req, int *ret, sr->nr_multishot_loops =0 ; sr->flags &= ~IORING_RECV_MSHOT_CAP ;if (issue_flags &IO_URING_F_MULTISHOT ) - *ret =IOU_REQUEUE ; + sel->val =IOU_REQUEUE ; }return true; }/* Finish the request / stop multishot. */ finish: -io_req_set_res (req, *ret, cflags); - *ret =IOU_COMPLETE ; +io_req_set_res (req, sel->val, cflags); + sel->val =IOU_COMPLETE ;io_req_msg_cleanup (req, issue_flags);return true; }
@@ -1094,10 +1095,11 @@ int io_recvmsg(struct io_kiocb *req, unsigned int issue_flags)else io_kbuf_recycle (req, req->buf_list, issue_flags); -if (!io_recv_finish (req, &ret, kmsg, mshot_finished, issue_flags)) + sel.val = ret; +if (!io_recv_finish (req, kmsg, &sel, mshot_finished, issue_flags)) goto retry_multishot; -return ret; +return sel.val; } static int io_recv_buf_select(struct io_kiocb *req, struct io_async_msghdr *kmsg,
@@ -1242,10 +1244,11 @@ int io_recv(struct io_kiocb *req, unsigned int issue_flags)else io_kbuf_recycle (req, req->buf_list, issue_flags); -if (!io_recv_finish (req, &ret, kmsg, mshot_finished, issue_flags)) + sel.val = ret; +if (!io_recv_finish (req, kmsg, &sel, mshot_finished, issue_flags)) goto retry_multishot; -return ret; +return sel.val; }int io_recvzc_prep (struct io_kiocb *req,const struct io_uring_sqe *sqe)
Signature · 1 line
2.50.1
Jens Axboe
maintainerPATCH 8/9 io_uring/kbuf: switch to storing struct io_buffer_list locally
In reply to Jens AxboeCurrently the buffer list is stored in struct io_kiocb. The buffer list can be of two types:
1) Classic/legacy buffer list. These don't need to get referenced after a buffer pick, and hence storing them in struct io_kiocb is perfectly fine.
2) Ring provided buffer lists. These DO need to be referenced after the initial buffer pick, as they need to get consumed later on. This can be either just incrementing the head of the ring, or it can be consuming parts of a buffer if incremental buffer consumptions has been configured.
For case 2, io_uring needs to be careful not to access the buffer list after the initial pick-and-execute context. The core does recycling of these, but it's easy to make a mistake, because it's stored in the io_kiocb which does persist across multiple execution contexts. Either because it's a multishot request, or simply because it needed some kind of async trigger (eg poll) for retry purposes.
Add a struct io_buffer_list to struct io_br_sel, which is always on stack for the various users of it. This prevents the buffer list from leaking outside of that execution context, and additionally it enables kbuf to not even pass back the struct io_buffer_list if the given context isn't appropriately locked already.
This doesn't fix any bugs, it's simply a defensive measure to prevent any issues with reuse of a buffer list.
--- include/linux/io_uring_types.h | 6 ---- io_uring/io_uring.c | 6 ++-- io_uring/kbuf.c | 27 ++++++++++-------- io_uring/kbuf.h | 16 ++++------- io_uring/net.c | 50 +++++++++++++--------------------- io_uring/poll.c | 6 ++-- io_uring/rw.c | 22 +++++++-------- 7 files changed, 57 insertions(+), 76 deletions(-)
include/linux/io_uring_types.h +0 -6
@@ -674,12 +674,6 @@ struct io_kiocb {/* stores selected buf, valid IFF REQ_F_BUFFER_SELECTED is set */ struct io_buffer *kbuf; -/* -* stores buffer ID for ring provided buffers, valid IFF -* REQ_F_BUFFER_RING is set. -*/ -struct io_buffer_list *buf_list; -struct io_rsrc_node *buf_node; };
io_uring/io_uring.c +3 -3
@@ -1007,7 +1007,7 @@ void io_req_defer_failed(struct io_kiocb *req, s32 res)lockdep_assert_held (&req->ctx->uring_lock);req_set_fail (req); -io_req_set_res (req, res,io_put_kbuf (req, res, req->buf_list,IO_URING_F_UNLOCKED )); +io_req_set_res (req, res,io_put_kbuf (req, res,NULL ,IO_URING_F_UNLOCKED ));if (def->fail) def->fail (req);io_req_complete_defer (req);
@@ -2025,11 +2025,11 @@ static void io_queue_async(struct io_kiocb *req, unsigned int issue_flags, int rswitch (io_arm_poll_handler (req,0 )) {case IO_APOLL_READY : -io_kbuf_recycle (req, req->buf_list,0 ); +io_kbuf_recycle (req,NULL ,0 );io_req_task_queue (req);break ;case IO_APOLL_ABORTED : -io_kbuf_recycle (req, req->buf_list,0 ); +io_kbuf_recycle (req,NULL ,0 );io_queue_iowq (req);break ;case IO_APOLL_OK :
io_uring/kbuf.c +15 -12
@@ -171,8 +171,8 @@ static struct io_br_sel io_ring_buffer_select(struct io_kiocb *req, size_t *len,if (*len ==0 || *len > buf->len) *len = buf->len; req->flags |=REQ_F_BUFFER_RING |REQ_F_BUFFERS_COMMIT ; - req->buf_list = bl; req->buf_index = buf->bid; + sel.buf_list = bl; sel.addr =u64_to_user_ptr (buf->addr);if (issue_flags &IO_URING_F_UNLOCKED || !io_file_can_poll (req)) {
@@ -186,8 +186,8 @@ static struct io_br_sel io_ring_buffer_select(struct io_kiocb *req, size_t *len,* the transfer completes (or if we get -EAGAIN and must poll of * retry). */ -io_kbuf_commit (req, bl, *len,1 ); - req->buf_list =NULL ; +io_kbuf_commit (req, sel.buf_list, *len,1 ); + sel.buf_list =NULL ; }return sel; }
@@ -294,7 +294,6 @@ static int io_ring_buffers_peek(struct io_kiocb *req, struct buf_sel_arg *arg, req->flags |=REQ_F_BL_EMPTY ; req->flags |=REQ_F_BUFFER_RING ; - req->buf_list = bl;return iov - arg->iovs; }
@@ -302,16 +301,15 @@ int io_buffers_select(struct io_kiocb *req, struct buf_sel_arg *arg, struct io_br_sel *sel, unsigned int issue_flags) { struct io_ring_ctx *ctx = req->ctx; -struct io_buffer_list *bl;int ret = -ENOENT ;io_ring_submit_lock (ctx, issue_flags); - bl =io_buffer_get_list (ctx, arg->buf_group); -if (unlikely (!bl)) + sel->buf_list =io_buffer_get_list (ctx, arg->buf_group); +if (unlikely (!sel->buf_list)) goto out_unlock; -if (bl->flags &IOBL_BUF_RING ) { - ret =io_ring_buffers_peek (req, arg, bl); +if (sel->buf_list->flags &IOBL_BUF_RING ) { + ret =io_ring_buffers_peek (req, arg, sel->buf_list);/* * Don't recycle these buffers if we need to go through poll. * Nobody else can use them anyway, and holding on to provided
@@ -321,13 +319,16 @@ int io_buffers_select(struct io_kiocb *req, struct buf_sel_arg *arg,*/ if (ret >0 ) { req->flags |=REQ_F_BUFFERS_COMMIT |REQ_F_BL_NO_RECYCLE ; -io_kbuf_commit (req, bl, arg->out_len, ret); +io_kbuf_commit (req, sel->buf_list, arg->out_len, ret); } }else { - ret =io_provided_buffers_select (req, &arg->out_len, bl, arg->iovs); + ret =io_provided_buffers_select (req, &arg->out_len, sel->buf_list, arg->iovs); } out_unlock: -io_ring_submit_unlock (ctx, issue_flags); +if (issue_flags &IO_URING_F_UNLOCKED ) { + sel->buf_list =NULL ; +mutex_unlock (&ctx->uring_lock); + }return ret; }
@@ -348,10 +349,12 @@ int io_buffers_peek(struct io_kiocb *req, struct buf_sel_arg *arg, ret =io_ring_buffers_peek (req, arg, bl);if (ret >0 ) req->flags |=REQ_F_BUFFERS_COMMIT ; + sel->buf_list = bl;return ret; }/* don't support multiple buffer selections for legacy */ + sel->buf_list =NULL ;return io_provided_buffers_select (req, &arg->max_len, bl, arg->iovs); }
io_uring/kbuf.h +6 -10
@@ -63,11 +63,14 @@ struct buf_sel_arg { };/* -* Return value from io_buffer_list selection. Just returns the error or -* user address for now, will be extended to return the buffer list in the -* future. +* Return value from io_buffer_list selection, to avoid stashing it in +* struct io_kiocb. For legacy/classic provided buffers, keeping a reference +* across execution contexts are fine. But for ring provided buffers, the +* list may go away as soon as ->uring_lock is dropped. As the io_kiocb +* persists, it's better to just keep the buffer local for those cases. */ struct io_br_sel { +struct io_buffer_list *buf_list;/* * Some selection parts return the user address, others return an error. */
@@ -107,13 +110,6 @@ struct io_mapped_region *io_pbuf_get_region(struct io_ring_ctx *ctx,static inline bool io_kbuf_recycle_ring (struct io_kiocb *req,struct io_buffer_list *bl) { -/* -* We don't need to recycle for REQ_F_BUFFER_RING, we can just clear -* the flag and hence ensure that bl->head doesn't get incremented. -* If the tail has already been incremented, hang on to it. -* The exception is partial io, that case we should increment bl->head -* to monopolize the buffer. -*/ if (bl) { req->flags &= ~(REQ_F_BUFFER_RING |REQ_F_BUFFERS_COMMIT );return true;
io_uring/net.c +19 -31
166 lines not on this page · raw message
io_uring/poll.c +3 -3
@@ -316,10 +316,10 @@ void io_poll_task_func(struct io_kiocb *req, io_tw_token_t tw) ret =io_poll_check_events (req, tw);if (ret ==IOU_POLL_NO_ACTION ) { -io_kbuf_recycle (req, req->buf_list,0 ); +io_kbuf_recycle (req,NULL ,0 );return ; }else if (ret ==IOU_POLL_REQUEUE ) { -io_kbuf_recycle (req, req->buf_list,0 ); +io_kbuf_recycle (req,NULL ,0 );__io_poll_execute (req,0 );return ; }
@@ -686,7 +686,7 @@ int io_arm_apoll(struct io_kiocb *req, unsigned issue_flags, __poll_t mask) req->flags |=REQ_F_POLLED ; ipt.pt._qproc = io_async_queue_proc; -io_kbuf_recycle (req, req->buf_list, issue_flags); +io_kbuf_recycle (req,NULL , issue_flags); ret =__io_arm_poll_handler (req, &apoll->poll, &ipt, mask, issue_flags);if (ret)
io_uring/rw.c +11 -11
@@ -579,7 +579,7 @@ void io_req_rw_complete(struct io_kiocb *req, io_tw_token_t tw)io_req_io_end (req);if (req->flags & (REQ_F_BUFFER_SELECTED |REQ_F_BUFFER_RING )) - req->cqe.flags |=io_put_kbuf (req, req->cqe.res, req->buf_list,0 ); + req->cqe.flags |=io_put_kbuf (req, req->cqe.res,NULL ,0 );io_req_rw_cleanup (req,0 );io_req_task_complete (req, tw);
@@ -648,7 +648,7 @@ static inline void io_rw_done(struct io_kiocb *req, ssize_t ret) }static int kiocb_done (struct io_kiocb *req,ssize_t ret, -unsigned int issue_flags) +struct io_br_sel *sel,unsigned int issue_flags) { struct io_rw *rw = io_kiocb_to_cmd(req, struct io_rw);unsigned final_ret =io_fixup_rw_res (req, ret);
@@ -662,7 +662,7 @@ static int kiocb_done(struct io_kiocb *req, ssize_t ret,* from the submission path. */ io_req_io_end (req); -io_req_set_res (req, final_ret,io_put_kbuf (req, ret, req->buf_list, issue_flags)); +io_req_set_res (req, final_ret,io_put_kbuf (req, ret, sel->buf_list, issue_flags));io_req_rw_cleanup (req, issue_flags);return IOU_COMPLETE ; }else {
@@ -1024,10 +1024,10 @@ int io_read(struct io_kiocb *req, unsigned int issue_flags) ret =__io_read (req, &sel, issue_flags);if (ret >=0 ) -return kiocb_done (req, ret, issue_flags); +return kiocb_done (req, ret, &sel, issue_flags);if (req->flags &REQ_F_BUFFERS_COMMIT ) -io_kbuf_recycle (req, req->buf_list, issue_flags); +io_kbuf_recycle (req, sel.buf_list, issue_flags);return ret; }
@@ -1057,15 +1057,15 @@ int io_read_mshot(struct io_kiocb *req, unsigned int issue_flags)* Reset rw->len to 0 again to avoid clamping future mshot * reads, in case the buffer size varies. */ -if (io_kbuf_recycle (req, req->buf_list, issue_flags)) +if (io_kbuf_recycle (req, sel.buf_list, issue_flags)) rw->len =0 ;return IOU_RETRY ; }else if (ret <=0 ) { -io_kbuf_recycle (req, req->buf_list, issue_flags); +io_kbuf_recycle (req, sel.buf_list, issue_flags);if (ret <0 )req_set_fail (req); }else if (!(req->flags &REQ_F_APOLL_MULTISHOT )) { - cflags =io_put_kbuf (req, ret, req->buf_list, issue_flags); + cflags =io_put_kbuf (req, ret, sel.buf_list, issue_flags); }else {/* * Any successful return value will keep the multishot read
@@ -1073,7 +1073,7 @@ int io_read_mshot(struct io_kiocb *req, unsigned int issue_flags)* we fail to post a CQE, or multishot is no longer set, then * jump to the termination path. This request is then done. */ - cflags =io_put_kbuf (req, ret, req->buf_list, issue_flags); + cflags =io_put_kbuf (req, ret, sel.buf_list, issue_flags); rw->len =0 ;/* similarly to above, reset len to 0 */ if (io_req_post_cqe (req, ret, cflags |IORING_CQE_F_MORE )) {
@@ -1202,7 +1202,7 @@ int io_write(struct io_kiocb *req, unsigned int issue_flags)return -EAGAIN ; } done: -return kiocb_done (req, ret2, issue_flags); +return kiocb_done (req, ret2,NULL , issue_flags); }else { ret_eagain:iov_iter_restore (&io->iter, &io->iter_state);
@@ -1370,7 +1370,7 @@ int io_do_iopoll(struct io_ring_ctx *ctx, bool force_nonspin)if (!smp_load_acquire (&req->iopoll_completed))break ; nr_events++; - req->cqe.flags =io_put_kbuf (req, req->cqe.res, req->buf_list,0 ); + req->cqe.flags =io_put_kbuf (req, req->cqe.res,NULL ,0 );if (req->opcode !=IORING_OP_URING_CMD )io_req_rw_cleanup (req,0 ); }
Signature · 1 line
2.50.1
Jens Axboe
maintainerPATCH 9/9 io_uring: remove async/poll related provided buffer recycles
In reply to Jens AxboeThese aren't necessary anymore, get rid of them.
--- io_uring/io_uring.c | 2 -- io_uring/poll.c | 4 ---- 2 files changed, 6 deletions(-)
io_uring/io_uring.c +0 -2
@@ -2025,11 +2025,9 @@ static void io_queue_async(struct io_kiocb *req, unsigned int issue_flags, int rswitch (io_arm_poll_handler (req,0 )) {case IO_APOLL_READY : -io_kbuf_recycle (req,NULL ,0 );io_req_task_queue (req);break ;case IO_APOLL_ABORTED : -io_kbuf_recycle (req,NULL ,0 );io_queue_iowq (req);break ;case IO_APOLL_OK :
io_uring/poll.c +0 -4
@@ -316,10 +316,8 @@ void io_poll_task_func(struct io_kiocb *req, io_tw_token_t tw) ret =io_poll_check_events (req, tw);if (ret ==IOU_POLL_NO_ACTION ) { -io_kbuf_recycle (req,NULL ,0 );return ; }else if (ret ==IOU_POLL_REQUEUE ) { -io_kbuf_recycle (req,NULL ,0 );__io_poll_execute (req,0 );return ; }
@@ -686,8 +684,6 @@ int io_arm_apoll(struct io_kiocb *req, unsigned issue_flags, __poll_t mask) req->flags |=REQ_F_POLLED ; ipt.pt._qproc = io_async_queue_proc; -io_kbuf_recycle (req,NULL , issue_flags); - ret =__io_arm_poll_handler (req, &apoll->poll, &ipt, mask, issue_flags);if (ret)return ret >0 ?IO_APOLL_READY :IO_APOLL_ABORTED ;
Signature · 1 line
2.50.1