Reduce overhead of multishot recv/recvmsg
Jens Axboe
maintainerPATCH 1/1 Reduce overhead of multishot recv/recvmsg
Hi,
With the way that the multishot variants of recv and recvmsg currently work, we always end up doing two sock_recvmsg() call for each subsequent receive outside of the initial receive. That is obviously extra overhead, and to make matters worse, that also involves picking and recycling an appropriate buffer for those operations.
For protocols that sanely pass back whether a socket has more data after a receive, we can rely on that hint to avoid iterating over recvmsg/buffers an extra time per receive.
Patches 1..3 are prep patches, patch 4 has the logic mentioned above.
Signature · 1 line
Jens Axboe
Jens Axboe
maintainerPATCH 1/4 io_uring/net: initialize struct msghdr more sanely for io_recv()
We only need to clear the input fields on the first invocation, not when potentially doing a retry.
--- io_uring/net.c | 15 ++++++++------- 1 file changed, 8 insertions(+), 7 deletions(-)
io_uring/net.c +8 -7
@@ -860,6 +860,14 @@ int io_recv(struct io_kiocb *req, unsigned int issue_flags)if (unlikely (!sock))return -ENOTSOCK ; + msg.msg_name =NULL ; + msg.msg_namelen =0 ; + msg.msg_control =NULL ; + msg.msg_get_inq =1 ; + msg.msg_controllen =0 ; + msg.msg_iocb =NULL ; + msg.msg_ubuf =NULL ; + retry_multishot:if (io_do_buffer_select (req)) {void __user *buf;
@@ -874,14 +882,7 @@ int io_recv(struct io_kiocb *req, unsigned int issue_flags)if (unlikely (ret)) goto out_free; - msg.msg_name =NULL ; - msg.msg_namelen =0 ; - msg.msg_control =NULL ; - msg.msg_get_inq =1 ; msg.msg_flags =0 ; - msg.msg_controllen =0 ; - msg.msg_iocb =NULL ; - msg.msg_ubuf =NULL ; flags = sr->msg_flags;if (force_nonblock)
Signature · 1 line
2.39.2
Jens Axboe
maintainerPATCH 2/4 io_uring/net: initalize msghdr->msg_inq to known value
In reply to Jens AxboeWe can't currently tell if ->msg_inq was set when we ask for msg_get_inq, initialize it to -1U so we can tell apart if it was set and there's no data left, or if it just wasn't set at all by the protocol.
--- io_uring/net.c | 6 ++++-- 1 file changed, 4 insertions(+), 2 deletions(-)
io_uring/net.c +4 -2
@@ -785,6 +785,7 @@ int io_recvmsg(struct io_kiocb *req, unsigned int issue_flags) min_ret =iov_iter_count (&kmsg->msg.msg_iter); kmsg->msg.msg_get_inq =1 ; + kmsg->msg.msg_inq =-1U ;if (req->flags &REQ_F_APOLL_MULTISHOT ) ret =io_recvmsg_multishot (sock, sr, kmsg, flags, &mshot_finished);
@@ -821,7 +822,7 @@ int io_recvmsg(struct io_kiocb *req, unsigned int issue_flags)io_kbuf_recycle (req, issue_flags); cflags =io_put_kbuf (req, issue_flags); -if (kmsg->msg.msg_inq) +if (kmsg->msg.msg_inq && kmsg->msg.msg_inq !=-1U ) cflags |=IORING_CQE_F_SOCK_NONEMPTY ;if (!io_recv_finish (req, &ret, cflags, mshot_finished, issue_flags))
@@ -882,6 +883,7 @@ int io_recv(struct io_kiocb *req, unsigned int issue_flags)if (unlikely (ret)) goto out_free; + msg.msg_inq =-1U ; msg.msg_flags =0 ; flags = sr->msg_flags;
@@ -923,7 +925,7 @@ int io_recv(struct io_kiocb *req, unsigned int issue_flags)io_kbuf_recycle (req, issue_flags); cflags =io_put_kbuf (req, issue_flags); -if (msg.msg_inq) +if (msg.msg_inq && msg.msg_inq !=-1U ) cflags |=IORING_CQE_F_SOCK_NONEMPTY ;if (!io_recv_finish (req, &ret, cflags, ret <=0 , issue_flags))
Signature · 1 line
2.39.2
Jens Axboe
maintainerPATCH 3/4 io_uring/net: push IORING_CQE_F_SOCK_NONEMPTY into io_recv_finish()
In reply to Jens AxboeRather than have this logic in both io_recv() and io_recvmsg_multishot(), push it into the handler they both call when finishing a receive operation.
--- io_uring/net.c | 22 +++++++++------------- 1 file changed, 9 insertions(+), 13 deletions(-)
io_uring/net.c +9 -13
@@ -616,9 +616,15 @@ static inline void io_recv_prep_retry(struct io_kiocb *req)* again (for multishot). */ static inline bool io_recv_finish (struct io_kiocb *req,int *ret, -unsigned int cflags,bool mshot_finished, +struct msghdr *msg,bool mshot_finished,unsigned issue_flags) { +unsigned int cflags; + + cflags =io_put_kbuf (req, issue_flags); +if (msg->msg_inq && msg->msg_inq !=-1U ) + cflags |=IORING_CQE_F_SOCK_NONEMPTY ; +if (!(req->flags &REQ_F_APOLL_MULTISHOT )) {io_req_set_res (req, *ret, cflags); *ret =IOU_OK ;
@@ -732,7 +738,6 @@ int io_recvmsg(struct io_kiocb *req, unsigned int issue_flags) struct io_sr_msg *sr = io_kiocb_to_cmd(req, struct io_sr_msg);struct io_async_msghdr iomsg, *kmsg;struct socket *sock; -unsigned int cflags;unsigned flags;int ret, min_ret =0 ;bool force_nonblock = issue_flags &IO_URING_F_NONBLOCK ;
@@ -821,11 +826,7 @@ int io_recvmsg(struct io_kiocb *req, unsigned int issue_flags)else io_kbuf_recycle (req, issue_flags); - cflags =io_put_kbuf (req, issue_flags); -if (kmsg->msg.msg_inq && kmsg->msg.msg_inq !=-1U ) - cflags |=IORING_CQE_F_SOCK_NONEMPTY ; - -if (!io_recv_finish (req, &ret, cflags, mshot_finished, issue_flags)) +if (!io_recv_finish (req, &ret, &kmsg->msg, mshot_finished, issue_flags)) goto retry_multishot;if (mshot_finished) {
@@ -844,7 +845,6 @@ int io_recv(struct io_kiocb *req, unsigned int issue_flags) struct io_sr_msg *sr = io_kiocb_to_cmd(req, struct io_sr_msg);struct msghdr msg;struct socket *sock; -unsigned int cflags;unsigned flags;int ret, min_ret =0 ;bool force_nonblock = issue_flags &IO_URING_F_NONBLOCK ;
@@ -924,11 +924,7 @@ int io_recv(struct io_kiocb *req, unsigned int issue_flags)else io_kbuf_recycle (req, issue_flags); - cflags =io_put_kbuf (req, issue_flags); -if (msg.msg_inq && msg.msg_inq !=-1U ) - cflags |=IORING_CQE_F_SOCK_NONEMPTY ; - -if (!io_recv_finish (req, &ret, cflags, ret <=0 , issue_flags)) +if (!io_recv_finish (req, &ret, &msg, ret <=0 , issue_flags)) goto retry_multishot;return ret;
Signature · 1 line
2.39.2
Jens Axboe
maintainerPATCH 4/4 io_uring/net: don't retry recvmsg() unnecessarily
In reply to Jens AxboeIf we're doing multishot receives, then we always end up doing two trips through sock_recvmsg(). For protocols that sanely set msghdr->msg_inq, then we don't need to waste time picking a new buffer and attemtping a new receive if there's nothing there.
--- io_uring/net.c | 10 +++++++++- 1 file changed, 9 insertions(+), 1 deletion(-)
io_uring/net.c +9 -1
@@ -635,7 +635,15 @@ static inline bool io_recv_finish(struct io_kiocb *req, int *ret,if (io_aux_cqe (req->ctx, issue_flags &IO_URING_F_COMPLETE_DEFER , req->cqe.user_data, *ret, cflags |IORING_CQE_F_MORE , true)) {io_recv_prep_retry (req); -return false; +/* Known not-empty or unknown state, retry */ +if (cflags &IORING_CQE_F_SOCK_NONEMPTY || + msg->msg_inq ==-1U ) +return false; +if (issue_flags &IO_URING_F_MULTISHOT ) + *ret =IOU_ISSUE_SKIP_COMPLETE ; +else + *ret = -EAGAIN ; +return true; }/* Otherwise stop multishot but use the current result. */ }
Signature · 1 line
2.39.2