send request cleanups
cover send request cleanups
The first batch of assorted cleanups for send path, which grew out of shape over time. It also prepare the code base for supporting registered buffers with non-zc send requests, which will be useful with leased/kernel buffers.
Pavel Begunkov (7):
io_uring/net: open code io_sendmsg_copy_hdr() io_uring/net: open code io_net_vec_assign() io_uring/net: combine sendzc flags writes io_uring/net: unify sendmsg setup with zc io_uring/net: clusterise send vs msghdr branches io_uring/net: set sg_from_iter in advance io_uring/net: import zc ubuf earlier
io_uring/net.c | 127 ++++++++++++++++--------------------------------- 1 file changed, 40 insertions(+), 87 deletions(-)
Signature · 1 line
2.48.1
PATCH 1/7 io_uring/net: open code io_sendmsg_copy_hdr()
io_sendmsg_setup() is trivial and io_sendmsg_copy_hdr() doesn't add any good abstraction, open code one into another.
--- io_uring/net.c | 30 ++++++++++-------------------- 1 file changed, 10 insertions(+), 20 deletions(-)
io_uring/net.c +10 -20
@@ -325,25 +325,6 @@ static int io_msg_copy_hdr(struct io_kiocb *req, struct io_async_msghdr *iomsg,return 0 ; } -static int io_sendmsg_copy_hdr (struct io_kiocb *req, -struct io_async_msghdr *iomsg) -{ - struct io_sr_msg *sr = io_kiocb_to_cmd(req, struct io_sr_msg); -struct user_msghdr msg; -int ret; - - ret =io_msg_copy_hdr (req, iomsg, &msg,ITER_SOURCE ,NULL ); -if (unlikely (ret)) -return ret; - -if (!(req->flags &REQ_F_BUFFER_SELECT )) - ret =io_net_import_vec (req, iomsg, msg.msg_iov, msg.msg_iovlen, -ITER_SOURCE ); -/* save msg_control as sys_sendmsg() overwrites it */ - sr->msg_control = iomsg->msg.msg_control_user; -return ret; -} -void io_sendmsg_recvmsg_cleanup (struct io_kiocb *req) {struct io_async_msghdr *io = req->async_data;
@@ -392,10 +373,19 @@ static int io_sendmsg_setup(struct io_kiocb *req, const struct io_uring_sqe *sqe { struct io_sr_msg *sr = io_kiocb_to_cmd(req, struct io_sr_msg);struct io_async_msghdr *kmsg = req->async_data; +struct user_msghdr msg; +int ret; sr->umsg =u64_to_user_ptr (READ_ONCE (sqe->addr)); + ret =io_msg_copy_hdr (req, kmsg, &msg,ITER_SOURCE ,NULL ); +if (unlikely (ret)) +return ret; +/* save msg_control as sys_sendmsg() overwrites it */ + sr->msg_control = kmsg->msg.msg_control_user; -return io_sendmsg_copy_hdr (req, kmsg); +if (req->flags &REQ_F_BUFFER_SELECT ) +return 0 ; +return io_net_import_vec (req, kmsg, msg.msg_iov, msg.msg_iovlen,ITER_SOURCE ); }static int io_sendmsg_zc_setup (struct io_kiocb *req,const struct io_uring_sqe *sqe)
Signature · 1 line
2.48.1
PATCH 2/7 io_uring/net: open code io_net_vec_assign()
In reply to Pavel BegunkovGet rid of io_net_vec_assign() by open coding it into its only caller.
--- io_uring/net.c | 16 +++++----------- 1 file changed, 5 insertions(+), 11 deletions(-)
io_uring/net.c +5 -11
@@ -176,16 +176,6 @@ static struct io_async_msghdr *io_msg_alloc_async(struct io_kiocb *req)return hdr; } -/* assign new iovec to kmsg, if we need to */ -static void io_net_vec_assign (struct io_kiocb *req,struct io_async_msghdr *kmsg, -struct iovec *iov) -{ -if (iov) { - req->flags |=REQ_F_NEED_CLEANUP ; -io_vec_reset_iovec (&kmsg->vec, iov, kmsg->msg.msg_iter.nr_segs); - } -} -static inline void io_mshot_prep_retry (struct io_kiocb *req,struct io_async_msghdr *kmsg) {
@@ -217,7 +207,11 @@ static int io_net_import_vec(struct io_kiocb *req, struct io_async_msghdr *iomsg &iomsg->msg.msg_iter, io_is_compat(req->ctx));if (unlikely (ret <0 ))return ret; -io_net_vec_assign (req, iomsg, iov); + +if (iov) { + req->flags |=REQ_F_NEED_CLEANUP ; +io_vec_reset_iovec (&iomsg->vec, iov, iomsg->msg.msg_iter.nr_segs); + }return 0 ; }
Signature · 1 line
2.48.1
PATCH 3/7 io_uring/net: combine sendzc flags writes
In reply to Pavel BegunkovSave an instruction / trip to the cache and assign some of sendzc flags together.
--- io_uring/net.c | 3 +-- 1 file changed, 1 insertion(+), 2 deletions(-)
io_uring/net.c +1 -2
@@ -1292,7 +1292,6 @@ int io_send_zc_prep(struct io_kiocb *req, const struct io_uring_sqe *sqe) zc->done_io =0 ; zc->retry = false; - req->flags |=REQ_F_POLL_NO_LAZY ;if (unlikely (READ_ONCE (sqe->__pad2[0 ]) ||READ_ONCE (sqe->addr3)))return -EINVAL ;
@@ -1306,7 +1305,7 @@ int io_send_zc_prep(struct io_kiocb *req, const struct io_uring_sqe *sqe) notif->cqe.user_data = req->cqe.user_data; notif->cqe.res =0 ; notif->cqe.flags =IORING_CQE_F_NOTIF ; - req->flags |=REQ_F_NEED_CLEANUP ; + req->flags |=REQ_F_NEED_CLEANUP |REQ_F_POLL_NO_LAZY ; zc->flags =READ_ONCE (sqe->ioprio);if (unlikely (zc->flags & ~IO_ZC_FLAGS_COMMON )) {
Signature · 1 line
2.48.1
PATCH 4/7 io_uring/net: unify sendmsg setup with zc
In reply to Pavel Begunkovio_sendmsg_zc_setup() duplicates parts of io_sendmsg_setup(), and the only difference between them is that the former support vectored registered buffers with nothing zerocopy specific. Merge them together, we want regular sendmsg to eventually support fixed buffers either way.
--- io_uring/net.c | 28 ++++++---------------------- 1 file changed, 6 insertions(+), 22 deletions(-)
io_uring/net.c +6 -22
@@ -377,32 +377,16 @@ static int io_sendmsg_setup(struct io_kiocb *req, const struct io_uring_sqe *sqe/* save msg_control as sys_sendmsg() overwrites it */ sr->msg_control = kmsg->msg.msg_control_user; +if (sr->flags &IORING_RECVSEND_FIXED_BUF ) { + kmsg->msg.msg_iter.nr_segs = msg.msg_iovlen; +return io_prep_reg_iovec (req, &kmsg->vec, msg.msg_iov, + msg.msg_iovlen); + }if (req->flags &REQ_F_BUFFER_SELECT )return 0 ;return io_net_import_vec (req, kmsg, msg.msg_iov, msg.msg_iovlen,ITER_SOURCE ); } -static int io_sendmsg_zc_setup (struct io_kiocb *req,const struct io_uring_sqe *sqe) -{ - struct io_sr_msg *sr = io_kiocb_to_cmd(req, struct io_sr_msg); -struct io_async_msghdr *kmsg = req->async_data; -struct user_msghdr msg; -int ret; - -if (!(sr->flags &IORING_RECVSEND_FIXED_BUF )) -return io_sendmsg_setup (req, sqe); - - sr->umsg =u64_to_user_ptr (READ_ONCE (sqe->addr)); - - ret =io_msg_copy_hdr (req, kmsg, &msg,ITER_SOURCE ,NULL ); -if (unlikely (ret)) -return ret; - sr->msg_control = kmsg->msg.msg_control_user; - kmsg->msg.msg_iter.nr_segs = msg.msg_iovlen; - -return io_prep_reg_iovec (req, &kmsg->vec, msg.msg_iov, msg.msg_iovlen); -} -#define SENDMSG_FLAGS (IORING_RECVSEND_POLL_FIRST | IORING_RECVSEND_BUNDLE)int io_sendmsg_prep (struct io_kiocb *req,const struct io_uring_sqe *sqe)
@@ -1340,7 +1324,7 @@ int io_send_zc_prep(struct io_kiocb *req, const struct io_uring_sqe *sqe) req->flags |=REQ_F_IMPORT_BUFFER ;return io_send_setup (req, sqe); } - ret =io_sendmsg_zc_setup (req, sqe); + ret =io_sendmsg_setup (req, sqe);if (unlikely (ret))return ret;
Signature · 1 line
2.48.1
PATCH 5/7 io_uring/net: clusterise send vs msghdr branches
In reply to Pavel BegunkovWe have multiple branches at prep for send vs sendmsg handling, put them together so that the variant handling is more localised.
--- io_uring/net.c | 15 ++++----------- 1 file changed, 4 insertions(+), 11 deletions(-)
io_uring/net.c +4 -11
@@ -395,12 +395,6 @@ int io_sendmsg_prep(struct io_kiocb *req, const struct io_uring_sqe *sqe) sr->done_io =0 ; sr->retry = false; - -if (req->opcode !=IORING_OP_SEND ) { -if (sqe->addr2 || sqe->file_index) -return -EINVAL ; - } - sr->len =READ_ONCE (sqe->len); sr->flags =READ_ONCE (sqe->ioprio);if (sr->flags & ~SENDMSG_FLAGS )
@@ -426,6 +420,8 @@ int io_sendmsg_prep(struct io_kiocb *req, const struct io_uring_sqe *sqe)return -ENOMEM ;if (req->opcode !=IORING_OP_SENDMSG )return io_send_setup (req, sqe); +if (unlikely (sqe->addr2 || sqe->file_index)) +return -EINVAL ;return io_sendmsg_setup (req, sqe); }
@@ -1304,11 +1300,6 @@ int io_send_zc_prep(struct io_kiocb *req, const struct io_uring_sqe *sqe) } } -if (req->opcode !=IORING_OP_SEND_ZC ) { -if (unlikely (sqe->addr2 || sqe->file_index)) -return -EINVAL ; - } - zc->len =READ_ONCE (sqe->len); zc->msg_flags =READ_ONCE (sqe->msg_flags) |MSG_NOSIGNAL |MSG_ZEROCOPY ; req->buf_index =READ_ONCE (sqe->buf_index);
@@ -1324,6 +1315,8 @@ int io_send_zc_prep(struct io_kiocb *req, const struct io_uring_sqe *sqe) req->flags |=REQ_F_IMPORT_BUFFER ;return io_send_setup (req, sqe); } +if (unlikely (sqe->addr2 || sqe->file_index)) +return -EINVAL ; ret =io_sendmsg_setup (req, sqe);if (unlikely (ret))return ret;
Signature · 1 line
2.48.1
PATCH 6/7 io_uring/net: set sg_from_iter in advance
In reply to Pavel BegunkovIn preparation to the next patch, set ->sg_from_iter callback at request prep time.
--- io_uring/net.c | 25 +++++++++++++++---------- 1 file changed, 15 insertions(+), 10 deletions(-)
io_uring/net.c +15 -10
@@ -97,6 +97,11 @@ struct io_recvzc {struct io_zcrx_ifq *ifq; }; +static int io_sg_from_iter_iovec (struct sk_buff *skb, +struct iov_iter *from,size_t length); +static int io_sg_from_iter (struct sk_buff *skb, +struct iov_iter *from,size_t length); +int io_shutdown_prep (struct io_kiocb *req,const struct io_uring_sqe *sqe) { struct io_shutdown *shutdown = io_kiocb_to_cmd(req, struct io_shutdown);
@@ -1267,6 +1272,7 @@ int io_send_zc_prep(struct io_kiocb *req, const struct io_uring_sqe *sqe) { struct io_sr_msg *zc = io_kiocb_to_cmd(req, struct io_sr_msg);struct io_ring_ctx *ctx = req->ctx; +struct io_async_msghdr *iomsg;struct io_kiocb *notif;int ret;
@@ -1309,8 +1315,15 @@ int io_send_zc_prep(struct io_kiocb *req, const struct io_uring_sqe *sqe)if (io_is_compat (req->ctx)) zc->msg_flags |=MSG_CMSG_COMPAT ; -if (unlikely (!io_msg_alloc_async (req))) + iomsg =io_msg_alloc_async (req); +if (unlikely (!iomsg))return -ENOMEM ; + +if (zc->flags &IORING_RECVSEND_FIXED_BUF ) + iomsg->msg.sg_from_iter = io_sg_from_iter; +else + iomsg->msg.sg_from_iter = io_sg_from_iter_iovec; +if (req->opcode ==IORING_OP_SEND_ZC ) { req->flags |=REQ_F_IMPORT_BUFFER ;return io_send_setup (req, sqe);
@@ -1321,11 +1334,8 @@ int io_send_zc_prep(struct io_kiocb *req, const struct io_uring_sqe *sqe)if (unlikely (ret))return ret; -if (!(zc->flags &IORING_RECVSEND_FIXED_BUF )) { -struct io_async_msghdr *iomsg = req->async_data; - +if (!(zc->flags &IORING_RECVSEND_FIXED_BUF ))return io_notif_account_mem (zc->notif, iomsg->msg.msg_iter.count); - }return 0 ; }
@@ -1392,7 +1402,6 @@ static int io_send_zc_import(struct io_kiocb *req, unsigned int issue_flags) ITER_SOURCE, issue_flags);if (unlikely (ret))return ret; - kmsg->msg.sg_from_iter = io_sg_from_iter; }else { ret =import_ubuf (ITER_SOURCE , sr->buf, sr->len, &kmsg->msg.msg_iter);if (unlikely (ret))
@@ -1400,7 +1409,6 @@ static int io_send_zc_import(struct io_kiocb *req, unsigned int issue_flags) ret =io_notif_account_mem (sr->notif, sr->len);if (unlikely (ret))return ret; - kmsg->msg.sg_from_iter = io_sg_from_iter_iovec; }return ret;
@@ -1483,8 +1491,6 @@ int io_sendmsg_zc(struct io_kiocb *req, unsigned int issue_flags)unsigned flags;int ret, min_ret =0 ; - kmsg->msg.sg_from_iter = io_sg_from_iter_iovec; -if (req->flags &REQ_F_IMPORT_BUFFER ) {unsigned uvec_segs = kmsg->msg.msg_iter.nr_segs;int ret;
@@ -1493,7 +1499,6 @@ int io_sendmsg_zc(struct io_kiocb *req, unsigned int issue_flags) &kmsg->vec, uvec_segs, issue_flags);if (unlikely (ret))return ret; - kmsg->msg.sg_from_iter = io_sg_from_iter; req->flags &= ~REQ_F_IMPORT_BUFFER ; }
Signature · 1 line
2.48.1
PATCH 7/7 io_uring/net: import zc ubuf earlier
In reply to Pavel Begunkovio_send_setup() already sets up the iterator for IORING_OP_SEND_ZC, we don't need repeating that at issue time. Move it all together with mem accounting at prep time, which is more consistent with how the non-zc version does that.
--- io_uring/net.c | 44 ++++++++++++++++---------------------------- 1 file changed, 16 insertions(+), 28 deletions(-)
io_uring/net.c +16 -28
@@ -1319,23 +1319,23 @@ int io_send_zc_prep(struct io_kiocb *req, const struct io_uring_sqe *sqe)if (unlikely (!iomsg))return -ENOMEM ; -if (zc->flags &IORING_RECVSEND_FIXED_BUF ) - iomsg->msg.sg_from_iter = io_sg_from_iter; -else - iomsg->msg.sg_from_iter = io_sg_from_iter_iovec; -if (req->opcode ==IORING_OP_SEND_ZC ) { - req->flags |=REQ_F_IMPORT_BUFFER ; -return io_send_setup (req, sqe); +if (zc->flags &IORING_RECVSEND_FIXED_BUF ) + req->flags |=REQ_F_IMPORT_BUFFER ; + ret =io_send_setup (req, sqe); + }else { +if (unlikely (sqe->addr2 || sqe->file_index)) +return -EINVAL ; + ret =io_sendmsg_setup (req, sqe); } -if (unlikely (sqe->addr2 || sqe->file_index)) -return -EINVAL ; - ret =io_sendmsg_setup (req, sqe);if (unlikely (ret))return ret; -if (!(zc->flags &IORING_RECVSEND_FIXED_BUF )) +if (!(zc->flags &IORING_RECVSEND_FIXED_BUF )) { + iomsg->msg.sg_from_iter = io_sg_from_iter_iovec;return io_notif_account_mem (zc->notif, iomsg->msg.msg_iter.count); + } + iomsg->msg.sg_from_iter = io_sg_from_iter;return 0 ; }
@@ -1393,25 +1393,13 @@ static int io_send_zc_import(struct io_kiocb *req, unsigned int issue_flags) { struct io_sr_msg *sr = io_kiocb_to_cmd(req, struct io_sr_msg);struct io_async_msghdr *kmsg = req->async_data; -int ret; -if (sr->flags &IORING_RECVSEND_FIXED_BUF ) { - sr->notif->buf_index = req->buf_index; - ret =io_import_reg_buf (sr->notif, &kmsg->msg.msg_iter, - (u64 )(uintptr_t )sr->buf, sr->len, -ITER_SOURCE , issue_flags); -if (unlikely (ret)) -return ret; - }else { - ret =import_ubuf (ITER_SOURCE , sr->buf, sr->len, &kmsg->msg.msg_iter); -if (unlikely (ret)) -return ret; - ret =io_notif_account_mem (sr->notif, sr->len); -if (unlikely (ret)) -return ret; - } +WARN_ON_ONCE (!(sr->flags &IORING_RECVSEND_FIXED_BUF )); -return ret; + sr->notif->buf_index = req->buf_index; +return io_import_reg_buf (sr->notif, &kmsg->msg.msg_iter, + (u64 )(uintptr_t )sr->buf, sr->len, +ITER_SOURCE , issue_flags); }int io_send_zc (struct io_kiocb *req,unsigned int issue_flags)
Signature · 1 line
2.48.1
Quoted 14 lines · Pavel Begunkov
The first batch of assorted cleanups for send path, which grew out of shape over time. It also prepare the code base for supporting registered buffers with non-zc send requests, which will be useful with leased/kernel buffers.
Pavel Begunkov (7):
io_uring/net: open codeio_sendmsg_copy_hdr()io_uring/net: open codeio_net_vec_assign()io_uring/net: combine sendzc flags writesio_uring/net: unify sendmsg setup with zcio_uring/net: clusterise send vs msghdr branchesio_uring/net: setsg_from_iterin advanceio_uring/net: import zc ubuf earlier
Quoted 1 line
[...]
Applied, thanks!
[1/7] io_uring/net: open code io_sendmsg_copy_hdr()
commit: a20b8631c8885cda45a331a151d29a83dfbfdefb
[2/7] io_uring/net: open code io_net_vec_assign()
commit: 5f364117db942c15980111f2e8ff6025c7e5893a
[3/7] io_uring/net: combine sendzc flags writes
commit: c55e2845dfa72e647ed8d9a7b4c6e11a8ed0fc1e
[4/7] io_uring/net: unify sendmsg setup with zc
commit: 63b16e4f0b90abad500ecb7bc7a625278febdc2c
[5/7] io_uring/net: clusterise send vs msghdr branches
commit: 49dbce5602dc50343c9794d0ddf05d1f6c9cb592
[6/7] io_uring/net: set sg_from_iter in advance
commit: ad3f6cc40084f9adb1a53bf386d966073dc6a4e9
[7/7] io_uring/net: import zc ubuf earlier
commit: fbe1a30c5d3e6f184ddd63deded6f30c3ecc4c3fBest regards,
Signature · 1 line
Jens Axboe