send[msg] refactoring
cover send[msg] refactoring
Clean up the send[msg] setup path. The pathes should be good enough by themselves, but more can be done on top. It's also needed as a dependency for supporting vectored fixed buffers for sendmsg zc.
Pavel Begunkov (4):
io_uring/net: split send and sendmsg prep helpers io_uring/net: don't store send address ptr io_uring/net: don't alias send user pointer reads io_uring/net: clean up io_msg_copy_hdr
io_uring/net.c | 75 ++++++++++++++++++++++++++------------------------ 1 file changed, 39 insertions(+), 36 deletions(-)
Signature · 1 line
2.46.0
PATCH 1/4 io_uring/net: split send and sendmsg prep helpers
A preparation patch splitting io_sendmsg_prep_setup into two separate helpers for send and sendmsg variants.
--- io_uring/net.c | 21 ++++++++++++--------- 1 file changed, 12 insertions(+), 9 deletions(-)
io_uring/net.c +12 -9
@@ -384,16 +384,11 @@ static int io_send_setup(struct io_kiocb *req)return 0 ; } -static int io_sendmsg_prep_setup (struct io_kiocb *req,int is_msg) +static int io_sendmsg_setup (struct io_kiocb *req) { -struct io_async_msghdr *kmsg; +struct io_async_msghdr *kmsg = req->async_data;int ret; - kmsg =io_msg_alloc_async (req); -if (unlikely (!kmsg)) -return -ENOMEM ; -if (!is_msg) -return io_send_setup (req); ret =io_sendmsg_copy_hdr (req, kmsg);if (!ret) req->flags |=REQ_F_NEED_CLEANUP ;
@@ -439,7 +434,11 @@ int io_sendmsg_prep(struct io_kiocb *req, const struct io_uring_sqe *sqe)if (req->ctx->compat) sr->msg_flags |=MSG_CMSG_COMPAT ;#endif -return io_sendmsg_prep_setup (req, req->opcode ==IORING_OP_SENDMSG ); +if (unlikely (!io_msg_alloc_async (req))) +return -ENOMEM ; +if (req->opcode !=IORING_OP_SENDMSG ) +return io_send_setup (req); +return io_sendmsg_setup (req); } static void io_req_msg_cleanup(struct io_kiocb *req,
@@ -1286,7 +1285,11 @@ int io_send_zc_prep(struct io_kiocb *req, const struct io_uring_sqe *sqe)if (req->ctx->compat) zc->msg_flags |=MSG_CMSG_COMPAT ;#endif -return io_sendmsg_prep_setup (req, req->opcode ==IORING_OP_SENDMSG_ZC ); +if (unlikely (!io_msg_alloc_async (req))) +return -ENOMEM ; +if (req->opcode !=IORING_OP_SENDMSG_ZC ) +return io_send_setup (req); +return io_sendmsg_setup (req); } static int io_sg_from_iter_iovec(struct sk_buff *skb,
Signature · 1 line
2.46.0
PATCH 2/4 io_uring/net: don't store send address ptr
In reply to Pavel BegunkovFor non "msg" requests we copy the address at the prep stage and there is no need to store the address user pointer long term. Pass the SQE into io_send_setup(), let it parse it, and remove struct io_sr_msg addr addr_len fields. It saves some space and also less confusing.
--- io_uring/net.c | 36 ++++++++++++++++-------------------- 1 file changed, 16 insertions(+), 20 deletions(-)
io_uring/net.c +16 -20
@@ -74,9 +74,7 @@ struct io_sr_msg {unsigned nr_multishot_loops;u16 flags;/* initialised and used only by !msg send variants */ -u16 addr_len;u16 buf_group; -void __user *addr;void __user *msg_control;/* used only for send zerocopy */ struct io_kiocb *notif;
@@ -356,24 +354,31 @@ void io_sendmsg_recvmsg_cleanup(struct io_kiocb *req)io_netmsg_iovec_free (io); } -static int io_send_setup (struct io_kiocb *req) +static int io_send_setup (struct io_kiocb *req,const struct io_uring_sqe *sqe) { struct io_sr_msg *sr = io_kiocb_to_cmd(req, struct io_sr_msg);struct io_async_msghdr *kmsg = req->async_data; +void __user *addr; +u16 addr_len;int ret; +if (READ_ONCE (sqe->__pad3[0 ])) +return -EINVAL ; + kmsg->msg.msg_name =NULL ; kmsg->msg.msg_namelen =0 ; kmsg->msg.msg_control =NULL ; kmsg->msg.msg_controllen =0 ; kmsg->msg.msg_ubuf =NULL ; -if (sr->addr) { - ret =move_addr_to_kernel (sr->addr, sr->addr_len, &kmsg->addr); + addr =u64_to_user_ptr (READ_ONCE (sqe->addr2)); + addr_len =READ_ONCE (sqe->addr_len); +if (addr) { + ret =move_addr_to_kernel (addr, addr_len, &kmsg->addr);if (unlikely (ret <0 ))return ret; kmsg->msg.msg_name = &kmsg->addr; - kmsg->msg.msg_namelen = sr->addr_len; + kmsg->msg.msg_namelen = addr_len; }if (!io_do_buffer_select (req)) { ret = import_ubuf(ITER_SOURCE, sr->buf, sr->len,
@@ -403,13 +408,9 @@ int io_sendmsg_prep(struct io_kiocb *req, const struct io_uring_sqe *sqe) sr->done_io =0 ; -if (req->opcode ==IORING_OP_SEND ) { -if (READ_ONCE (sqe->__pad3[0 ])) +if (req->opcode !=IORING_OP_SEND ) { +if (sqe->addr2 || sqe->file_index)return -EINVAL ; - sr->addr =u64_to_user_ptr (READ_ONCE (sqe->addr2)); - sr->addr_len =READ_ONCE (sqe->addr_len); - }else if (sqe->addr2 || sqe->file_index) { -return -EINVAL ; } sr->umsg =u64_to_user_ptr (READ_ONCE (sqe->addr));
@@ -437,7 +438,7 @@ int io_sendmsg_prep(struct io_kiocb *req, const struct io_uring_sqe *sqe)if (unlikely (!io_msg_alloc_async (req)))return -ENOMEM ;if (req->opcode !=IORING_OP_SENDMSG ) -return io_send_setup (req); +return io_send_setup (req, sqe);return io_sendmsg_setup (req); }
@@ -1263,12 +1264,7 @@ int io_send_zc_prep(struct io_kiocb *req, const struct io_uring_sqe *sqe)io_req_set_rsrc_node (notif, ctx,0 ); } -if (req->opcode ==IORING_OP_SEND_ZC ) { -if (READ_ONCE (sqe->__pad3[0 ])) -return -EINVAL ; - zc->addr =u64_to_user_ptr (READ_ONCE (sqe->addr2)); - zc->addr_len =READ_ONCE (sqe->addr_len); - }else { +if (req->opcode !=IORING_OP_SEND_ZC ) {if (unlikely (sqe->addr2 || sqe->file_index))return -EINVAL ;if (unlikely (zc->flags &IORING_RECVSEND_FIXED_BUF ))
@@ -1288,7 +1284,7 @@ int io_send_zc_prep(struct io_kiocb *req, const struct io_uring_sqe *sqe)if (unlikely (!io_msg_alloc_async (req)))return -ENOMEM ;if (req->opcode !=IORING_OP_SENDMSG_ZC ) -return io_send_setup (req); +return io_send_setup (req, sqe);return io_sendmsg_setup (req); }
Signature · 1 line
2.46.0
PATCH 3/4 io_uring/net: don't alias send user pointer reads
In reply to Pavel BegunkovWe keep user pointers in an union, which could be a user buffer or a user pointer to msghdr. What is confusing is that it potenitally reads and assigns sqe->addr as one type but then uses it as another via the union. Even more, it's not even consistent across copy and zerocopy versions.
Make send and sendmsg setup helpers read sqe->addr and treat it as the right type from the beginning. The end goal would be to get rid of the use of struct io_sr_msg::umsg for send requests as we only need it at the prep side.
--- io_uring/net.c | 13 ++++++++----- 1 file changed, 8 insertions(+), 5 deletions(-)
io_uring/net.c +8 -5
@@ -362,6 +362,8 @@ static int io_send_setup(struct io_kiocb *req, const struct io_uring_sqe *sqe)u16 addr_len;int ret; + sr->buf =u64_to_user_ptr (READ_ONCE (sqe->addr)); +if (READ_ONCE (sqe->__pad3[0 ]))return -EINVAL ;
@@ -389,11 +391,14 @@ static int io_send_setup(struct io_kiocb *req, const struct io_uring_sqe *sqe)return 0 ; } -static int io_sendmsg_setup (struct io_kiocb *req) +static int io_sendmsg_setup (struct io_kiocb *req,const struct io_uring_sqe *sqe) { + struct io_sr_msg *sr = io_kiocb_to_cmd(req, struct io_sr_msg);struct io_async_msghdr *kmsg = req->async_data;int ret; + sr->umsg =u64_to_user_ptr (READ_ONCE (sqe->addr)); + ret =io_sendmsg_copy_hdr (req, kmsg);if (!ret) req->flags |=REQ_F_NEED_CLEANUP ;
@@ -413,7 +418,6 @@ int io_sendmsg_prep(struct io_kiocb *req, const struct io_uring_sqe *sqe)return -EINVAL ; } - sr->umsg =u64_to_user_ptr (READ_ONCE (sqe->addr)); sr->len =READ_ONCE (sqe->len); sr->flags =READ_ONCE (sqe->ioprio);if (sr->flags & ~SENDMSG_FLAGS )
@@ -439,7 +443,7 @@ int io_sendmsg_prep(struct io_kiocb *req, const struct io_uring_sqe *sqe)return -ENOMEM ;if (req->opcode !=IORING_OP_SENDMSG )return io_send_setup (req, sqe); -return io_sendmsg_setup (req); +return io_sendmsg_setup (req, sqe); } static void io_req_msg_cleanup(struct io_kiocb *req,
@@ -1271,7 +1275,6 @@ int io_send_zc_prep(struct io_kiocb *req, const struct io_uring_sqe *sqe)return -EINVAL ; } - zc->buf =u64_to_user_ptr (READ_ONCE (sqe->addr)); zc->len =READ_ONCE (sqe->len); zc->msg_flags =READ_ONCE (sqe->msg_flags) |MSG_NOSIGNAL |MSG_ZEROCOPY ;if (zc->msg_flags &MSG_DONTWAIT )
@@ -1285,7 +1288,7 @@ int io_send_zc_prep(struct io_kiocb *req, const struct io_uring_sqe *sqe)return -ENOMEM ;if (req->opcode !=IORING_OP_SENDMSG_ZC )return io_send_setup (req, sqe); -return io_sendmsg_setup (req); +return io_sendmsg_setup (req, sqe); } static int io_sg_from_iter_iovec(struct sk_buff *skb,
Signature · 1 line
2.46.0
PATCH 4/4 io_uring/net: clean up io_msg_copy_hdr
In reply to Pavel BegunkovPut sr->umsg into a local variable, so it doesn't repeat "sr->umsg->" for every field. It looks nicer, and likely without the patch it compiles into a bunch of umsg memory reads.
--- io_uring/net.c | 15 ++++++++------- 1 file changed, 8 insertions(+), 7 deletions(-)
io_uring/net.c +8 -7
@@ -261,6 +261,7 @@ static int io_msg_copy_hdr(struct io_kiocb *req, struct io_async_msghdr *iomsg, struct user_msghdr *msg, int ddir) { struct io_sr_msg *sr = io_kiocb_to_cmd(req, struct io_sr_msg); +struct user_msghdr __user *umsg = sr->umsg;struct iovec *iov;int ret, nr_segs;
@@ -272,16 +273,16 @@ static int io_msg_copy_hdr(struct io_kiocb *req, struct io_async_msghdr *iomsg, nr_segs =1 ; } -if (!user_access_begin (sr->umsg,sizeof (*sr->umsg))) +if (!user_access_begin (umsg,sizeof (*umsg)))return -EFAULT ; ret = -EFAULT ; -unsafe_get_user (msg->msg_name, &sr->umsg->msg_name, ua_end); -unsafe_get_user (msg->msg_namelen, &sr->umsg->msg_namelen, ua_end); -unsafe_get_user (msg->msg_iov, &sr->umsg->msg_iov, ua_end); -unsafe_get_user (msg->msg_iovlen, &sr->umsg->msg_iovlen, ua_end); -unsafe_get_user (msg->msg_control, &sr->umsg->msg_control, ua_end); -unsafe_get_user (msg->msg_controllen, &sr->umsg->msg_controllen, ua_end); +unsafe_get_user (msg->msg_name, &umsg->msg_name, ua_end); +unsafe_get_user (msg->msg_namelen, &umsg->msg_namelen, ua_end); +unsafe_get_user (msg->msg_iov, &umsg->msg_iov, ua_end); +unsafe_get_user (msg->msg_iovlen, &umsg->msg_iovlen, ua_end); +unsafe_get_user (msg->msg_control, &umsg->msg_control, ua_end); +unsafe_get_user (msg->msg_controllen, &umsg->msg_controllen, ua_end); msg->msg_flags =0 ;if (req->flags &REQ_F_BUFFER_SELECT ) {
Signature · 1 line
2.46.0
Quoted 11 lines · Pavel Begunkov
Clean up the send[msg] setup path. The pathes should be good enough by themselves, but more can be done on top. It's also needed as a dependency for supporting vectored fixed buffers for sendmsg zc.
Pavel Begunkov (4):
io_uring/net: split send and sendmsg prep helpersio_uring/net: don't store send address ptrio_uring/net: don't alias send user pointer readsio_uring/net: clean upio_msg_copy_hdr[...]
Applied, thanks!
[1/4] io_uring/net: split send and sendmsg prep helpers
commit: e16d4fa2e642258237f1899e8569db7dff83fd54
[2/4] io_uring/net: don't store send address ptr
commit: 1aa83a963550d0bdd83dfc0f1ab4fcfdec1a060e
[3/4] io_uring/net: don't alias send user pointer reads
commit: f2320d008abab8ed654437febb2f6395cdbec131
[4/4] io_uring/net: clean up io_msg_copy_hdr
commit: bfd23424a581cefee8e13d0433bcf321a12b2d95Best regards,
Signature · 1 line
Jens Axboe