Misc cleanups
Jens Axboe
maintainercover Misc cleanups
Hi,
This is part of my recv/send bundle series, but they are ordered at the top and don't have dependencies. So sending these out separately.
Signature · 1 line
Jens Axboe
Jens Axboe
maintainerPATCH 1/4 io_uring/net: remove dependency on REQ_F_PARTIAL_IO for sr->done_io
Ensure that prep handlers always initialize sr->done_io before any potential failure conditions, and with that, we now it's always been set even for the failure case.
With that, we don't need to use the REQ_F_PARTIAL_IO flag to gate on that. Additionally, we should not overwrite req->cqe.res unless sr->done_io is actually positive.
--- io_uring/net.c | 12 +++++++----- 1 file changed, 7 insertions(+), 5 deletions(-)
io_uring/net.c +7 -5
@@ -387,6 +387,8 @@ int io_sendmsg_prep(struct io_kiocb *req, const struct io_uring_sqe *sqe) { struct io_sr_msg *sr = io_kiocb_to_cmd(req, struct io_sr_msg); + sr->done_io =0 ; +if (req->opcode ==IORING_OP_SEND ) {if (READ_ONCE (sqe->__pad3[0 ]))return -EINVAL ;
@@ -409,7 +411,6 @@ int io_sendmsg_prep(struct io_kiocb *req, const struct io_uring_sqe *sqe)if (req->ctx->compat) sr->msg_flags |=MSG_CMSG_COMPAT ;#endif - sr->done_io =0 ;return 0 ; }
@@ -631,6 +632,8 @@ int io_recvmsg_prep(struct io_kiocb *req, const struct io_uring_sqe *sqe) { struct io_sr_msg *sr = io_kiocb_to_cmd(req, struct io_sr_msg); + sr->done_io =0 ; +if (unlikely (sqe->file_index || sqe->addr2))return -EINVAL ;
@@ -667,7 +670,6 @@ int io_recvmsg_prep(struct io_kiocb *req, const struct io_uring_sqe *sqe)if (req->ctx->compat) sr->msg_flags |=MSG_CMSG_COMPAT ;#endif - sr->done_io =0 ; sr->nr_multishot_loops =0 ;return 0 ; }
@@ -1054,6 +1056,8 @@ int io_send_zc_prep(struct io_kiocb *req, const struct io_uring_sqe *sqe)struct io_ring_ctx *ctx = req->ctx;struct io_kiocb *notif; + zc->done_io =0 ; +if (unlikely (READ_ONCE (sqe->__pad2[0 ]) ||READ_ONCE (sqe->addr3)))return -EINVAL ;/* we don't support IOSQE_CQE_SKIP_SUCCESS just yet */
@@ -1106,8 +1110,6 @@ int io_send_zc_prep(struct io_kiocb *req, const struct io_uring_sqe *sqe)if (zc->msg_flags &MSG_DONTWAIT ) req->flags |=REQ_F_NOWAIT ; - zc->done_io =0 ; -#ifdef CONFIG_COMPAT if (req->ctx->compat) zc->msg_flags |=MSG_CMSG_COMPAT ;
@@ -1352,7 +1354,7 @@ void io_sendrecv_fail(struct io_kiocb *req) { struct io_sr_msg *sr = io_kiocb_to_cmd(req, struct io_sr_msg); -if (req->flags &REQ_F_PARTIAL_IO ) +if (sr->done_io) req->cqe.res = sr->done_io; if ((req->flags & REQ_F_NEED_CLEANUP) &&
Signature · 1 line
2.43.0
Jens Axboe
maintainerPATCH 2/4 io_uring/kbuf: rename REQ_F_PARTIAL_IO to REQ_F_BL_NO_RECYCLE
In reply to Jens AxboeWe only use the flag for this purpose, so rename it accordingly. This further prevents various other use cases of it, keeping it clean and consistent. Then we can also check it in one spot, when it's being attempted recycled, and remove some dead code in io_kbuf_recycle_ring().
--- include/linux/io_uring_types.h | 6 +++--- io_uring/kbuf.c | 9 --------- io_uring/kbuf.h | 20 +++++--------------- io_uring/net.c | 12 ++++++------ io_uring/rw.c | 4 ++-- 5 files changed, 16 insertions(+), 35 deletions(-)
include/linux/io_uring_types.h +3 -3
@@ -470,7 +470,6 @@ enum {REQ_F_SKIP_LINK_CQES_BIT ,REQ_F_SINGLE_POLL_BIT ,REQ_F_DOUBLE_POLL_BIT , -REQ_F_PARTIAL_IO_BIT ,REQ_F_APOLL_MULTISHOT_BIT ,REQ_F_CLEAR_POLLIN_BIT , REQ_F_HASH_LOCKED_BIT,
@@ -481,6 +480,7 @@ enum {REQ_F_CANCEL_SEQ_BIT ,REQ_F_CAN_POLL_BIT ,REQ_F_BL_EMPTY_BIT , +REQ_F_BL_NO_RECYCLE_BIT ,/* not a real bit, just to check we're not overflowing the space */ __REQ_F_LAST_BIT,
@@ -543,8 +543,6 @@ enum {REQ_F_SINGLE_POLL =IO_REQ_FLAG (REQ_F_SINGLE_POLL_BIT ),/* double poll may active */ REQ_F_DOUBLE_POLL =IO_REQ_FLAG (REQ_F_DOUBLE_POLL_BIT ), -/* request has already done partial IO */ -REQ_F_PARTIAL_IO =IO_REQ_FLAG (REQ_F_PARTIAL_IO_BIT ),/* fast poll multishot mode */ REQ_F_APOLL_MULTISHOT = IO_REQ_FLAG(REQ_F_APOLL_MULTISHOT_BIT),/* recvmsg special flag, clear EPOLLIN */
@@ -559,6 +557,8 @@ enum {REQ_F_CAN_POLL =IO_REQ_FLAG (REQ_F_CAN_POLL_BIT ),/* buffer list was empty after selection of buffer */ REQ_F_BL_EMPTY =IO_REQ_FLAG (REQ_F_BL_EMPTY_BIT ), +/* don't recycle provided buffers for this request */ + REQ_F_BL_NO_RECYCLE = IO_REQ_FLAG(REQ_F_BL_NO_RECYCLE_BIT), };typedef void (*io_req_tw_func_t )(struct io_kiocb *req,struct io_tw_state *ts);
io_uring/kbuf.c +0 -9
@@ -81,15 +81,6 @@ bool io_kbuf_recycle_legacy(struct io_kiocb *req, unsigned issue_flags)struct io_buffer_list *bl;struct io_buffer *buf; -/* -* For legacy provided buffer mode, don't recycle if we already did -* IO to this buffer. For ring-mapped provided buffer mode, we should -* increment ring->head to explicitly monopolize the buffer to avoid -* multiple use. -*/ -if (req->flags &REQ_F_PARTIAL_IO ) -return false; -io_ring_submit_lock (ctx, issue_flags); buf = req->kbuf;
io_uring/kbuf.h +5 -15
@@ -73,21 +73,9 @@ static inline bool io_kbuf_recycle_ring(struct io_kiocb *req)* to monopolize the buffer. */ if (req->buf_list) { -if (req->flags &REQ_F_PARTIAL_IO ) { -/* -* If we end up here, then the io_uring_lock has -* been kept held since we retrieved the buffer. -* For the io-wq case, we already cleared -* req->buf_list when the buffer was retrieved, -* hence it cannot be set here for that case. -*/ - req->buf_list->head++; - req->buf_list =NULL ; - }else { - req->buf_index = req->buf_list->bgid; - req->flags &= ~REQ_F_BUFFER_RING ; -return true; - } + req->buf_index = req->buf_list->bgid; + req->flags &= ~REQ_F_BUFFER_RING ; +return true; }return false; }
@@ -101,6 +89,8 @@ static inline bool io_do_buffer_select(struct io_kiocb *req)static inline bool io_kbuf_recycle (struct io_kiocb *req,unsigned issue_flags) { +if (req->flags &REQ_F_BL_NO_RECYCLE ) +return false;if (req->flags &REQ_F_BUFFER_SELECTED )return io_kbuf_recycle_legacy (req, issue_flags);if (req->flags &REQ_F_BUFFER_RING )
io_uring/net.c +6 -6
@@ -456,7 +456,7 @@ int io_sendmsg(struct io_kiocb *req, unsigned int issue_flags) kmsg->msg.msg_controllen =0 ; kmsg->msg.msg_control =NULL ; sr->done_io += ret; - req->flags |=REQ_F_PARTIAL_IO ; + req->flags |=REQ_F_BL_NO_RECYCLE ;return io_setup_async_msg (req, kmsg, issue_flags); }if (ret == -ERESTARTSYS )
@@ -535,7 +535,7 @@ int io_send(struct io_kiocb *req, unsigned int issue_flags) sr->len -= ret; sr->buf += ret; sr->done_io += ret; - req->flags |=REQ_F_PARTIAL_IO ; + req->flags |=REQ_F_BL_NO_RECYCLE ;return io_setup_async_addr (req, &__address, issue_flags); }if (ret == -ERESTARTSYS )
@@ -907,7 +907,7 @@ int io_recvmsg(struct io_kiocb *req, unsigned int issue_flags) }if (ret >0 &&io_net_retry (sock, flags)) { sr->done_io += ret; - req->flags |=REQ_F_PARTIAL_IO ; + req->flags |=REQ_F_BL_NO_RECYCLE ;return io_setup_async_msg (req, kmsg, issue_flags); }if (ret == -ERESTARTSYS )
@@ -1006,7 +1006,7 @@ int io_recv(struct io_kiocb *req, unsigned int issue_flags) sr->len -= ret; sr->buf += ret; sr->done_io += ret; - req->flags |=REQ_F_PARTIAL_IO ; + req->flags |=REQ_F_BL_NO_RECYCLE ;return -EAGAIN ; }if (ret == -ERESTARTSYS )
@@ -1249,7 +1249,7 @@ int io_send_zc(struct io_kiocb *req, unsigned int issue_flags) zc->len -= ret; zc->buf += ret; zc->done_io += ret; - req->flags |=REQ_F_PARTIAL_IO ; + req->flags |=REQ_F_BL_NO_RECYCLE ;return io_setup_async_addr (req, &__address, issue_flags); }if (ret == -ERESTARTSYS )
@@ -1319,7 +1319,7 @@ int io_sendmsg_zc(struct io_kiocb *req, unsigned int issue_flags)if (ret >0 &&io_net_retry (sock, flags)) { sr->done_io += ret; - req->flags |=REQ_F_PARTIAL_IO ; + req->flags |=REQ_F_BL_NO_RECYCLE ;return io_setup_async_msg (req, kmsg, issue_flags); }if (ret == -ERESTARTSYS )
io_uring/rw.c +2 -2
@@ -275,7 +275,7 @@ static bool __io_complete_rw_common(struct io_kiocb *req, long res)* current cycle. */ io_req_io_end (req); - req->flags |=REQ_F_REISSUE |REQ_F_PARTIAL_IO ; + req->flags |=REQ_F_REISSUE |REQ_F_BL_NO_RECYCLE ;return true; }req_set_fail (req);
@@ -342,7 +342,7 @@ static void io_complete_rw_iopoll(struct kiocb *kiocb, long res)io_req_end_write (req);if (unlikely (res != req->cqe.res)) {if (res == -EAGAIN &&io_rw_should_reissue (req)) { - req->flags |=REQ_F_REISSUE |REQ_F_PARTIAL_IO ; + req->flags |=REQ_F_REISSUE |REQ_F_BL_NO_RECYCLE ;return ; } req->cqe.res = res;
Signature · 1 line
2.43.0
Jens Axboe
maintainerPATCH 3/4 io_uring/net: simplify msghd->msg_inq checking
In reply to Jens AxboeJust check for larger than zero rather than check for non-zero and not -1. This is easier to read, and also protects against any errants < 0 values that aren't -1.
--- io_uring/net.c | 4 ++-- 1 file changed, 2 insertions(+), 2 deletions(-)
io_uring/net.c +2 -2
@@ -697,7 +697,7 @@ static inline bool io_recv_finish(struct io_kiocb *req, int *ret,unsigned int cflags; cflags =io_put_kbuf (req, issue_flags); -if (msg->msg_inq && msg->msg_inq !=-1 ) +if (msg->msg_inq >0 ) cflags |=IORING_CQE_F_SOCK_NONEMPTY ;if (!(req->flags &REQ_F_APOLL_MULTISHOT )) {
@@ -720,7 +720,7 @@ static inline bool io_recv_finish(struct io_kiocb *req, int *ret,io_recv_prep_retry (req);/* Known not-empty or unknown state, retry */ -if (cflags &IORING_CQE_F_SOCK_NONEMPTY || msg->msg_inq ==-1 ) { +if (cflags &IORING_CQE_F_SOCK_NONEMPTY || msg->msg_inq <0 ) {if (sr->nr_multishot_loops++ <MULTISHOT_MAX_RETRY )return false;/* mshot retries exceeded, force a requeue */
Signature · 1 line
2.43.0
Jens Axboe
maintainerPATCH 4/4 io_uring/net: add io_req_msg_cleanup() helper
In reply to Jens AxboeFor the fast inline path, we manually recycle the io_async_msghdr and free the iovec, and then clear the REQ_F_NEED_CLEANUP flag to avoid that needing doing in the slower path. We already do that in 2 spots, and in preparation for adding more, add a helper and use it.
--- io_uring/net.c | 26 ++++++++++++++------------ 1 file changed, 14 insertions(+), 12 deletions(-)
io_uring/net.c +14 -12
@@ -414,6 +414,17 @@ int io_sendmsg_prep(struct io_kiocb *req, const struct io_uring_sqe *sqe)return 0 ; } +static void io_req_msg_cleanup (struct io_kiocb *req, +struct io_async_msghdr *kmsg, +unsigned int issue_flags) +{ + req->flags &= ~REQ_F_NEED_CLEANUP ; +/* fast path, check for non-NULL to avoid function call */ +if (kmsg->free_iov) +kfree (kmsg->free_iov); +io_netmsg_recycle (req, issue_flags); +} +int io_sendmsg (struct io_kiocb *req,unsigned int issue_flags) { struct io_sr_msg *sr = io_kiocb_to_cmd(req, struct io_sr_msg);
@@ -463,11 +474,7 @@ int io_sendmsg(struct io_kiocb *req, unsigned int issue_flags) ret = -EINTR ;req_set_fail (req); } -/* fast path, check for non-NULL to avoid function call */ -if (kmsg->free_iov) -kfree (kmsg->free_iov); - req->flags &= ~REQ_F_NEED_CLEANUP ; -io_netmsg_recycle (req, issue_flags); +io_req_msg_cleanup (req, kmsg, issue_flags);if (ret >=0 ) ret += sr->done_io;else if (sr->done_io)
@@ -927,13 +934,8 @@ int io_recvmsg(struct io_kiocb *req, unsigned int issue_flags)if (!io_recv_finish (req, &ret, &kmsg->msg, mshot_finished, issue_flags)) goto retry_multishot; -if (mshot_finished) { -/* fast path, check for non-NULL to avoid function call */ -if (kmsg->free_iov) -kfree (kmsg->free_iov); -io_netmsg_recycle (req, issue_flags); - req->flags &= ~REQ_F_NEED_CLEANUP ; - } +if (mshot_finished) +io_req_msg_cleanup (req, kmsg, issue_flags);return ret; }
Signature · 1 line
2.43.0