improve net msghdr / iovec handlng
cover improve net msghdr / iovec handlng
Note: depends on ("io_uring/net: save msg_control for compat")
Continuing refactoring how iovecs are treated, this series adds some more sanity to handling struct msghdr in the networking code. We can do some more cleaning on top, but it should be in a good shape, and it'll be easier to do new stuff with that in.
Pavel Begunkov (7):
io_uring/net: remove unnecessary REQ_F_NEED_CLEANUP io_uring/net: simplify compat selbuf iov parsing io_uring/net: isolate msghdr copying code io_uring/net: verify msghdr before copying iovec io_uring/net: derive iovec storage later io_uring/net: unify *mshot_prep calls with compat io_uring/net: extract iovec import into a helper
io_uring/net.c | 180 ++++++++++++++++++++++--------------------------- 1 file changed, 81 insertions(+), 99 deletions(-)
Signature · 1 line
2.48.1
PATCH 1/7 io_uring/net: remove unnecessary REQ_F_NEED_CLEANUP
REQ_F_NEED_CLEANUP in io_recvmsg_prep_setup() and in io_sendmsg_setup() are relics of the past and don't do anything useful, the flag should be and are set earlier on iovec and async_data allocation.
--- io_uring/net.c | 11 ++--------- 1 file changed, 2 insertions(+), 9 deletions(-)
io_uring/net.c +2 -9
@@ -388,14 +388,10 @@ static int io_sendmsg_setup(struct io_kiocb *req, const struct io_uring_sqe *sqe { struct io_sr_msg *sr = io_kiocb_to_cmd(req, struct io_sr_msg);struct io_async_msghdr *kmsg = req->async_data; -int ret; sr->umsg =u64_to_user_ptr (READ_ONCE (sqe->addr)); - ret =io_sendmsg_copy_hdr (req, kmsg); -if (!ret) - req->flags |=REQ_F_NEED_CLEANUP ; -return ret; +return io_sendmsg_copy_hdr (req, kmsg); }#define SENDMSG_FLAGS (IORING_RECVSEND_POLL_FIRST | IORING_RECVSEND_BUNDLE)
@@ -774,10 +770,7 @@ static int io_recvmsg_prep_setup(struct io_kiocb *req)return 0 ; } - ret =io_recvmsg_copy_hdr (req, kmsg); -if (!ret) - req->flags |=REQ_F_NEED_CLEANUP ; -return ret; +return io_recvmsg_copy_hdr (req, kmsg); }#define RECVMSG_FLAGS (IORING_RECVSEND_POLL_FIRST | IORING_RECV_MULTISHOT | \
Signature · 1 line
2.48.1
PATCH 2/7 io_uring/net: simplify compat selbuf iov parsing
In reply to Pavel BegunkovUse copy_from_user() instead of open coded access_ok() + get_user(), that's simpler and we don't care about compat that much.
--- io_uring/net.c | 12 +++++------- 1 file changed, 5 insertions(+), 7 deletions(-)
io_uring/net.c +5 -7
@@ -215,21 +215,19 @@ static int io_compat_msg_copy_hdr(struct io_kiocb *req, uiov =compat_ptr (msg->msg_iov);if (req->flags &REQ_F_BUFFER_SELECT ) { -compat_ssize_t clen; -if (msg->msg_iovlen ==0 ) { sr->len = iov->iov_len =0 ; iov->iov_base =NULL ; }else if (msg->msg_iovlen >1 ) {return -EINVAL ; }else { -if (!access_ok (uiov,sizeof (*uiov))) -return -EFAULT ; -if (__get_user (clen, &uiov->iov_len)) +struct compat_iovec tmp_iov; + +if (copy_from_user (&tmp_iov, uiov,sizeof (tmp_iov)))return -EFAULT ; -if (clen <0 ) +if (tmp_iov.iov_len <0 )return -EINVAL ; - sr->len = clen; + sr->len = tmp_iov.iov_len; }return 0 ;
Signature · 1 line
2.48.1
PATCH 3/7 io_uring/net: isolate msghdr copying code
In reply to Pavel BegunkovThe user access section in io_msg_copy_hdr() is overextended by covering selected buffers. It's hard to work with and prone to errors. Limit the section to msghdr import only, selected buffers will do a separate copy_from_user() call, and then move it into its own function. This should be fine, selected buffer single shots are not important, for multishots the overhead should be non-existent, and it's not that expensive overall.
--- io_uring/net.c | 45 +++++++++++++++++++++++++-------------------- 1 file changed, 25 insertions(+), 20 deletions(-)
io_uring/net.c +25 -20
@@ -243,6 +243,24 @@ static int io_compat_msg_copy_hdr(struct io_kiocb *req, }#endif +static int io_copy_msghdr_from_user (struct user_msghdr *msg, +struct user_msghdr __user *umsg) +{ +if (!user_access_begin (umsg,sizeof (*umsg))) +return -EFAULT ; +unsafe_get_user (msg->msg_name, &umsg->msg_name, ua_end); +unsafe_get_user (msg->msg_namelen, &umsg->msg_namelen, ua_end); +unsafe_get_user (msg->msg_iov, &umsg->msg_iov, ua_end); +unsafe_get_user (msg->msg_iovlen, &umsg->msg_iovlen, ua_end); +unsafe_get_user (msg->msg_control, &umsg->msg_control, ua_end); +unsafe_get_user (msg->msg_controllen, &umsg->msg_controllen, ua_end); +user_access_end (); +return 0 ; +ua_end: +user_access_end (); +return -EFAULT ; +} +static int io_msg_copy_hdr (struct io_kiocb *req,struct io_async_msghdr *iomsg,struct user_msghdr *msg,int ddir) {
@@ -259,16 +277,10 @@ static int io_msg_copy_hdr(struct io_kiocb *req, struct io_async_msghdr *iomsg, nr_segs =1 ; } -if (!user_access_begin (umsg,sizeof (*umsg))) -return -EFAULT ; + ret =io_copy_msghdr_from_user (msg, umsg); +if (unlikely (ret)) +return ret; - ret = -EFAULT ; -unsafe_get_user (msg->msg_name, &umsg->msg_name, ua_end); -unsafe_get_user (msg->msg_namelen, &umsg->msg_namelen, ua_end); -unsafe_get_user (msg->msg_iov, &umsg->msg_iov, ua_end); -unsafe_get_user (msg->msg_iovlen, &umsg->msg_iovlen, ua_end); -unsafe_get_user (msg->msg_control, &umsg->msg_control, ua_end); -unsafe_get_user (msg->msg_controllen, &umsg->msg_controllen, ua_end); msg->msg_flags =0 ;if (req->flags &REQ_F_BUFFER_SELECT ) {
@@ -276,24 +288,17 @@ static int io_msg_copy_hdr(struct io_kiocb *req, struct io_async_msghdr *iomsg, sr->len = iov->iov_len =0 ; iov->iov_base =NULL ; }else if (msg->msg_iovlen >1 ) { - ret = -EINVAL ; - goto ua_end; +return -EINVAL ; }else {struct iovec __user *uiov = msg->msg_iov; -/* we only need the length for provided buffers */ -if (!access_ok (&uiov->iov_len,sizeof (uiov->iov_len))) - goto ua_end; -unsafe_get_user (iov->iov_len, &uiov->iov_len, ua_end); +if (copy_from_user (iov, uiov,sizeof (*iov))) +return -EFAULT ; sr->len = iov->iov_len; } - ret =0 ; -ua_end: -user_access_end (); -return ret; +return 0 ; } -user_access_end (); ret =__import_iovec (ddir, msg->msg_iov, msg->msg_iovlen, nr_segs, &iov, &iomsg->msg.msg_iter, false);if (unlikely (ret <0 ))
Signature · 1 line
2.48.1
PATCH 4/7 io_uring/net: verify msghdr before copying iovec
In reply to Pavel BegunkovNormally, net/ would verify msghdr before importing iovec, for example see copy_msghdr_from_user(), which further assumed by __copy_msghdr() validating msg->msg_iovlen.
io_uring does it in reverse order, which is fine, but it'll be more convenient for flip it so that the iovec business is done at the end and eventually can be nicely pulled out of msghdr parsing section and thought as a sepaarate step. That also makes structure accesses more localised, which should be better for caches.
--- io_uring/net.c | 43 ++++++++++++++++++------------------------- 1 file changed, 18 insertions(+), 25 deletions(-)
io_uring/net.c +18 -25
@@ -195,7 +195,8 @@ static inline void io_mshot_prep_retry(struct io_kiocb *req,#ifdef CONFIG_COMPAT static int io_compat_msg_copy_hdr (struct io_kiocb *req,struct io_async_msghdr *iomsg, -struct compat_msghdr *msg,int ddir) +struct compat_msghdr *msg,int ddir, +struct sockaddr __user **save_addr) { struct io_sr_msg *sr = io_kiocb_to_cmd(req, struct io_sr_msg);struct compat_iovec __user *uiov;
@@ -213,6 +214,10 @@ static int io_compat_msg_copy_hdr(struct io_kiocb *req,if (copy_from_user (msg, sr->umsg_compat,sizeof (*msg)))return -EFAULT ; + ret =__get_compat_msghdr (&iomsg->msg, msg, save_addr); +if (ret) +return ret; + uiov =compat_ptr (msg->msg_iov);if (req->flags &REQ_F_BUFFER_SELECT ) {if (msg->msg_iovlen ==0 ) {
@@ -262,7 +267,8 @@ static int io_copy_msghdr_from_user(struct user_msghdr *msg, }static int io_msg_copy_hdr (struct io_kiocb *req,struct io_async_msghdr *iomsg, -struct user_msghdr *msg,int ddir) +struct user_msghdr *msg,int ddir, +struct sockaddr __user **save_addr) { struct io_sr_msg *sr = io_kiocb_to_cmd(req, struct io_sr_msg);struct user_msghdr __user *umsg = sr->umsg;
@@ -283,6 +289,10 @@ static int io_msg_copy_hdr(struct io_kiocb *req, struct io_async_msghdr *iomsg, msg->msg_flags =0 ; + ret =__copy_msghdr (&iomsg->msg, msg, save_addr); +if (ret) +return ret; +if (req->flags &REQ_F_BUFFER_SELECT ) {if (msg->msg_iovlen ==0 ) { sr->len = iov->iov_len =0 ;
@@ -322,22 +332,14 @@ static int io_sendmsg_copy_hdr(struct io_kiocb *req,if (io_is_compat (req->ctx)) {struct compat_msghdr cmsg; - ret =io_compat_msg_copy_hdr (req, iomsg, &cmsg,ITER_SOURCE ); -if (unlikely (ret)) -return ret; - - ret =__get_compat_msghdr (&iomsg->msg, &cmsg,NULL ); + ret =io_compat_msg_copy_hdr (req, iomsg, &cmsg,ITER_SOURCE , +NULL ); sr->msg_control = iomsg->msg.msg_control_user;return ret; }#endif - ret =io_msg_copy_hdr (req, iomsg, &msg,ITER_SOURCE ); -if (unlikely (ret)) -return ret; - - ret =__copy_msghdr (&iomsg->msg, &msg,NULL ); - + ret =io_msg_copy_hdr (req, iomsg, &msg,ITER_SOURCE ,NULL );/* save msg_control as sys_sendmsg() overwrites it */ sr->msg_control = iomsg->msg.msg_control_user;return ret;
@@ -719,27 +721,18 @@ static int io_recvmsg_copy_hdr(struct io_kiocb *req,if (io_is_compat (req->ctx)) {struct compat_msghdr cmsg; - ret =io_compat_msg_copy_hdr (req, iomsg, &cmsg,ITER_DEST ); + ret =io_compat_msg_copy_hdr (req, iomsg, &cmsg,ITER_DEST , + &iomsg->uaddr);if (unlikely (ret))return ret; - - ret =__get_compat_msghdr (&iomsg->msg, &cmsg, &iomsg->uaddr); -if (unlikely (ret)) -return ret; -return io_recvmsg_mshot_prep (req, iomsg, cmsg.msg_namelen, cmsg.msg_controllen); }#endif - ret =io_msg_copy_hdr (req, iomsg, &msg,ITER_DEST ); + ret =io_msg_copy_hdr (req, iomsg, &msg,ITER_DEST , &iomsg->uaddr);if (unlikely (ret))return ret; - - ret =__copy_msghdr (&iomsg->msg, &msg, &iomsg->uaddr); -if (unlikely (ret)) -return ret; -return io_recvmsg_mshot_prep (req, iomsg, msg.msg_namelen, msg.msg_controllen); }
Signature · 1 line
2.48.1
PATCH 5/7 io_uring/net: derive iovec storage later
In reply to Pavel BegunkovDon't read free_iov until right before we need it to import the iovec. The only place that uses it before that is provided buffer selection, but it only serves as temporary storage and iovec content is not reused afterwards, so use a local variable for that.
--- io_uring/net.c | 43 +++++++++++++++++++++---------------------- 1 file changed, 21 insertions(+), 22 deletions(-)
io_uring/net.c +21 -22
@@ -203,14 +203,6 @@ static int io_compat_msg_copy_hdr(struct io_kiocb *req,struct iovec *iov;int ret, nr_segs; -if (iomsg->free_iov) { - nr_segs = iomsg->free_iov_nr; - iov = iomsg->free_iov; - }else { - iov = &iomsg->fast_iov; - nr_segs =1 ; - } -if (copy_from_user (msg, sr->umsg_compat,sizeof (*msg)))return -EFAULT ;
@@ -221,8 +213,7 @@ static int io_compat_msg_copy_hdr(struct io_kiocb *req, uiov =compat_ptr (msg->msg_iov);if (req->flags &REQ_F_BUFFER_SELECT ) {if (msg->msg_iovlen ==0 ) { - sr->len = iov->iov_len =0 ; - iov->iov_base =NULL ; + sr->len =0 ; }else if (msg->msg_iovlen >1 ) {return -EINVAL ; }else {
@@ -238,6 +229,14 @@ static int io_compat_msg_copy_hdr(struct io_kiocb *req,return 0 ; } +if (iomsg->free_iov) { + nr_segs = iomsg->free_iov_nr; + iov = iomsg->free_iov; + }else { + iov = &iomsg->fast_iov; + nr_segs =1 ; + } + ret =__import_iovec (ddir, (struct iovec __user *)uiov, msg->msg_iovlen, nr_segs, &iov, &iomsg->msg.msg_iter, true);if (unlikely (ret <0 ))
@@ -275,14 +274,6 @@ static int io_msg_copy_hdr(struct io_kiocb *req, struct io_async_msghdr *iomsg,struct iovec *iov;int ret, nr_segs; -if (iomsg->free_iov) { - nr_segs = iomsg->free_iov_nr; - iov = iomsg->free_iov; - }else { - iov = &iomsg->fast_iov; - nr_segs =1 ; - } - ret =io_copy_msghdr_from_user (msg, umsg);if (unlikely (ret))return ret;
@@ -295,20 +286,28 @@ static int io_msg_copy_hdr(struct io_kiocb *req, struct io_async_msghdr *iomsg,if (req->flags &REQ_F_BUFFER_SELECT ) {if (msg->msg_iovlen ==0 ) { - sr->len = iov->iov_len =0 ; - iov->iov_base =NULL ; + sr->len =0 ; }else if (msg->msg_iovlen >1 ) {return -EINVAL ; }else {struct iovec __user *uiov = msg->msg_iov; +struct iovec tmp_iov; -if (copy_from_user (iov, uiov,sizeof (*iov))) +if (copy_from_user (&tmp_iov, uiov,sizeof (tmp_iov)))return -EFAULT ; - sr->len = iov->iov_len; + sr->len = tmp_iov.iov_len; }return 0 ; } +if (iomsg->free_iov) { + nr_segs = iomsg->free_iov_nr; + iov = iomsg->free_iov; + }else { + iov = &iomsg->fast_iov; + nr_segs =1 ; + } + ret =__import_iovec (ddir, msg->msg_iov, msg->msg_iovlen, nr_segs, &iov, &iomsg->msg.msg_iter, false);if (unlikely (ret <0 ))
Signature · 1 line
2.48.1
PATCH 6/7 io_uring/net: unify *mshot_prep calls with compat
In reply to Pavel BegunkovInstead of duplicating a io_recvmsg_mshot_prep() call in the compat path, let the common code handle it. For that, copy necessary compat fields into struct user_msghdr. Note, it zeroes user_msghdr to be on the safe side as compat is not that interesting and overhead shouldn't be high.
--- io_uring/net.c | 14 +++++++------- 1 file changed, 7 insertions(+), 7 deletions(-)
io_uring/net.c +7 -7
@@ -716,20 +716,20 @@ static int io_recvmsg_copy_hdr(struct io_kiocb *req, iomsg->msg.msg_name = &iomsg->addr; iomsg->msg.msg_iter.nr_segs =0 ; -#ifdef CONFIG_COMPAT if (io_is_compat (req->ctx)) { +#ifdef CONFIG_COMPAT struct compat_msghdr cmsg; ret =io_compat_msg_copy_hdr (req, iomsg, &cmsg,ITER_DEST , &iomsg->uaddr); -if (unlikely (ret)) -return ret; -return io_recvmsg_mshot_prep (req, iomsg, cmsg.msg_namelen, - cmsg.msg_controllen); - } +memset (&msg,0 ,sizeof (msg)); + msg.msg_namelen = cmsg.msg_namelen; + msg.msg_controllen = cmsg.msg_controllen;#endif + }else { + ret =io_msg_copy_hdr (req, iomsg, &msg,ITER_DEST , &iomsg->uaddr); + } - ret =io_msg_copy_hdr (req, iomsg, &msg,ITER_DEST , &iomsg->uaddr);if (unlikely (ret))return ret; return io_recvmsg_mshot_prep(req, iomsg, msg.msg_namelen,
Signature · 1 line
2.48.1
PATCH 7/7 io_uring/net: extract iovec import into a helper
In reply to Pavel BegunkovDeduplicate iovec imports between compat and !compat by introducing a helper function.
--- io_uring/net.c | 62 +++++++++++++++++++++++--------------------------- 1 file changed, 28 insertions(+), 34 deletions(-)
io_uring/net.c +28 -34
@@ -192,6 +192,29 @@ static inline void io_mshot_prep_retry(struct io_kiocb *req, req->buf_index = sr->buf_group; } +static int io_net_import_vec (struct io_kiocb *req,struct io_async_msghdr *iomsg, +const struct iovec __user *uiov,unsigned uvec_seg, +int ddir) +{ +struct iovec *iov; +int ret, nr_segs; + +if (iomsg->free_iov) { + nr_segs = iomsg->free_iov_nr; + iov = iomsg->free_iov; + }else { + nr_segs =1 ; + iov = &iomsg->fast_iov; + } + + ret =__import_iovec (ddir, uiov, uvec_seg, nr_segs, &iov, + &iomsg->msg.msg_iter,io_is_compat (req->ctx)); +if (unlikely (ret <0 )) +return ret; +io_net_vec_assign (req, iomsg, iov); +return 0 ; +} +#ifdef CONFIG_COMPAT static int io_compat_msg_copy_hdr(struct io_kiocb *req, struct io_async_msghdr *iomsg,
@@ -200,8 +223,7 @@ static int io_compat_msg_copy_hdr(struct io_kiocb *req, { struct io_sr_msg *sr = io_kiocb_to_cmd(req, struct io_sr_msg);struct compat_iovec __user *uiov; -struct iovec *iov; -int ret, nr_segs; +int ret;if (copy_from_user (msg, sr->umsg_compat,sizeof (*msg)))return -EFAULT ;
@@ -229,21 +251,8 @@ static int io_compat_msg_copy_hdr(struct io_kiocb *req, return 0; } -if (iomsg->free_iov) { - nr_segs = iomsg->free_iov_nr; - iov = iomsg->free_iov; - }else { - iov = &iomsg->fast_iov; - nr_segs =1 ; - } - - ret =__import_iovec (ddir, (struct iovec __user *)uiov, msg->msg_iovlen, - nr_segs, &iov, &iomsg->msg.msg_iter, true); -if (unlikely (ret <0 )) -return ret; - -io_net_vec_assign (req, iomsg, iov); -return 0 ; +return io_net_import_vec (req, iomsg, (struct iovec __user *)uiov, + msg->msg_iovlen, ddir); }#endif
@@ -271,8 +280,7 @@ static int io_msg_copy_hdr(struct io_kiocb *req, struct io_async_msghdr *iomsg, { struct io_sr_msg *sr = io_kiocb_to_cmd(req, struct io_sr_msg);struct user_msghdr __user *umsg = sr->umsg; -struct iovec *iov; -int ret, nr_segs; +int ret; ret =io_copy_msghdr_from_user (msg, umsg);if (unlikely (ret))
@@ -300,21 +308,7 @@ static int io_msg_copy_hdr(struct io_kiocb *req, struct io_async_msghdr *iomsg,return 0 ; } -if (iomsg->free_iov) { - nr_segs = iomsg->free_iov_nr; - iov = iomsg->free_iov; - }else { - iov = &iomsg->fast_iov; - nr_segs =1 ; - } - - ret =__import_iovec (ddir, msg->msg_iov, msg->msg_iovlen, nr_segs, - &iov, &iomsg->msg.msg_iter, false); -if (unlikely (ret <0 )) -return ret; - -io_net_vec_assign (req, iomsg, iov); -return 0 ; +return io_net_import_vec (req, iomsg, msg->msg_iov, msg->msg_iovlen, ddir); } static int io_sendmsg_copy_hdr(struct io_kiocb *req,
Signature · 1 line
2.48.1
Quoted 8 lines · Pavel Begunkov
Note: depends on ("
io_uring/net: savemsg_controlfor compat")Continuing refactoring how iovecs are treated, this series adds some more sanity to handling struct msghdr in the networking code. We can do some more cleaning on top, but it should be in a good shape, and it'll be easier to do new stuff with that in.
[...]
Applied, thanks!
[1/7] io_uring/net: remove unnecessary REQ_F_NEED_CLEANUP
commit: 398421b7d776edecbd89a7ca6cdaaac0d965762d
[2/7] io_uring/net: simplify compat selbuf iov parsing
commit: c30f89f1d08b205ab96aa09ce06549ba34bbba67
[3/7] io_uring/net: isolate msghdr copying code
commit: 6b69dd00e98a6f9041415351f9894539da0e73c4
[4/7] io_uring/net: verify msghdr before copying iovec
commit: 820c215726a57ffd766376d23feed2bebe27a18f
[5/7] io_uring/net: derive iovec storage later
commit: 67f9fbe72a2d2abb25aafd1af6d9f6373cbc5024
[6/7] io_uring/net: unify *mshot_prep calls with compat
commit: 68525267875757520752ff1abbda0af58fc172b9
[7/7] io_uring/net: extract iovec import into a helper
commit: 7eb5bb749ed945ead6a089e18e1d1da74523ac9aBest regards,
Signature · 1 line
Jens Axboe