support for {send,recv}[msg] with registered bufs
cover support for {send,recv}[msg] with registered bufs
By passing IO_MSG_FIXED with msg_flags teaches send(), recv(), sendmsg() and recvmsg() to work with registered buffers. In that case sqe->buf_index should contain a valid registered buffer index, and an iov or an immediate {ptr,len} pair for recv/send have to point to a buffer as it's with rw.
As unused bits in msg_flags are never checked by the net stack, I'd probably need to flag it somehow else.
That's mainly for interested to toy with zerocopy, thus RFC, though may be useful regardless. Based on 5.11 + 2 my send/recvmsg cleanup patches of the same day.
Pavel Begunkov (5):
io_uring: move io_recvmsg_copy_hdr()
io_uring: copy hdr consistently for send and recv
io_uring: opcode independent import_fixed
io_uring: send/recv with registered buffer
io_uring: sendmsg/recvmsg with registered buffers
fs/io_uring.c | 321 +++++++++++++++++++++++++++----------------------- 1 file changed, 171 insertions(+), 150 deletions(-)
Signature · 1 line
2.24.0
PATCH 1/5 io_uring: move io_recvmsg_copy_hdr()
Move io_recvmsg_copy_hdr() for it to be reused in later patches.
--- fs/io_uring.c | 238 +++++++++++++++++++++++++------------------------- 1 file changed, 119 insertions(+), 119 deletions(-)
fs/io_uring.c +119 -119
@@ -4494,125 +4494,6 @@ static int io_setup_async_msg(struct io_kiocb *req,return -EAGAIN ; } -static int io_sendmsg_copy_hdr (struct io_kiocb *req, -struct io_async_msghdr *iomsg) -{ - iomsg->iov = iomsg->fast_iov; - iomsg->msg.msg_name = &iomsg->addr; -return sendmsg_copy_msghdr (&iomsg->msg, req->sr_msg.umsg, - req->sr_msg.msg_flags, &iomsg->iov); -} - -static int io_sendmsg_prep (struct io_kiocb *req,const struct io_uring_sqe *sqe) -{ -struct io_async_msghdr *async_msg = req->async_data; -struct io_sr_msg *sr = &req->sr_msg; -int ret; - -if (unlikely (req->ctx->flags &IORING_SETUP_IOPOLL )) -return -EINVAL ; - - sr->msg_flags =READ_ONCE (sqe->msg_flags); - sr->umsg =u64_to_user_ptr (READ_ONCE (sqe->addr)); - sr->len =READ_ONCE (sqe->len); - -#ifdef CONFIG_COMPAT -if (req->ctx->compat) - sr->msg_flags |=MSG_CMSG_COMPAT ; -#endif - -if (!async_msg || !io_op_defs[req->opcode].needs_async_data) -return 0 ; - ret =io_sendmsg_copy_hdr (req, async_msg); -if (!ret) - req->flags |=REQ_F_NEED_CLEANUP ; -return ret; -} - -static int io_sendmsg (struct io_kiocb *req,bool force_nonblock, -struct io_comp_state *cs) -{ -struct io_async_msghdr iomsg, *kmsg; -struct socket *sock; -unsigned flags; -int ret; - - sock =sock_from_file (req->file, &ret); -if (unlikely (!sock)) -return ret; - - kmsg = req->async_data; -if (!kmsg) { - ret =io_sendmsg_copy_hdr (req, &iomsg); -if (ret) -return ret; - kmsg = &iomsg; - } - - flags = req->sr_msg.msg_flags; -if (flags &MSG_DONTWAIT ) - req->flags |=REQ_F_NOWAIT ; -else if (force_nonblock) - flags |=MSG_DONTWAIT ; - - ret =__sys_sendmsg_sock (sock, &kmsg->msg, flags); -if (force_nonblock && ret == -EAGAIN ) -return io_setup_async_msg (req, kmsg); -if (ret == -ERESTARTSYS ) - ret = -EINTR ; - -/* it's reportedly faster to check for null here */ -if (kmsg->iov) -kfree (kmsg->iov); - req->flags &= ~REQ_F_NEED_CLEANUP ; -if (ret <0 ) -req_set_fail_links (req); -__io_req_complete (req, ret,0 , cs); -return 0 ; -} - -static int io_send (struct io_kiocb *req,bool force_nonblock, -struct io_comp_state *cs) -{ -struct io_sr_msg *sr = &req->sr_msg; -struct msghdr msg; -struct iovec iov; -struct socket *sock; -unsigned flags; -int ret; - - sock =sock_from_file (req->file, &ret); -if (unlikely (!sock)) -return ret; - - ret =import_single_range (WRITE , sr->buf, sr->len, &iov, &msg.msg_iter); -if (unlikely (ret)) -return ret; - - msg.msg_name =NULL ; - msg.msg_control =NULL ; - msg.msg_controllen =0 ; - msg.msg_namelen =0 ; - - flags = req->sr_msg.msg_flags; -if (flags &MSG_DONTWAIT ) - req->flags |=REQ_F_NOWAIT ; -else if (force_nonblock) - flags |=MSG_DONTWAIT ; - - msg.msg_flags = flags; - ret =sock_sendmsg (sock, &msg); -if (force_nonblock && ret == -EAGAIN ) -return -EAGAIN ; -if (ret == -ERESTARTSYS ) - ret = -EINTR ; - -if (ret <0 ) -req_set_fail_links (req); -__io_req_complete (req, ret,0 , cs); -return 0 ; -} -static int __io_recvmsg_copy_hdr (struct io_kiocb *req,struct io_async_msghdr *iomsg) {
@@ -4733,6 +4614,125 @@ static int io_recvmsg_copy_hdr(struct io_kiocb *req,return 0 ; } +static int io_sendmsg_copy_hdr (struct io_kiocb *req, +struct io_async_msghdr *iomsg) +{ + iomsg->iov = iomsg->fast_iov; + iomsg->msg.msg_name = &iomsg->addr; +return sendmsg_copy_msghdr (&iomsg->msg, req->sr_msg.umsg, + req->sr_msg.msg_flags, &iomsg->iov); +} + +static int io_sendmsg_prep (struct io_kiocb *req,const struct io_uring_sqe *sqe) +{ +struct io_async_msghdr *async_msg = req->async_data; +struct io_sr_msg *sr = &req->sr_msg; +int ret; + +if (unlikely (req->ctx->flags &IORING_SETUP_IOPOLL )) +return -EINVAL ; + + sr->msg_flags =READ_ONCE (sqe->msg_flags); + sr->umsg =u64_to_user_ptr (READ_ONCE (sqe->addr)); + sr->len =READ_ONCE (sqe->len); + +#ifdef CONFIG_COMPAT +if (req->ctx->compat) + sr->msg_flags |=MSG_CMSG_COMPAT ; +#endif + +if (!async_msg || !io_op_defs[req->opcode].needs_async_data) +return 0 ; + ret =io_sendmsg_copy_hdr (req, async_msg); +if (!ret) + req->flags |=REQ_F_NEED_CLEANUP ; +return ret; +} + +static int io_sendmsg (struct io_kiocb *req,bool force_nonblock, +struct io_comp_state *cs) +{ +struct io_async_msghdr iomsg, *kmsg; +struct socket *sock; +unsigned flags; +int ret; + + sock =sock_from_file (req->file, &ret); +if (unlikely (!sock)) +return ret; + + kmsg = req->async_data; +if (!kmsg) { + ret =io_sendmsg_copy_hdr (req, &iomsg); +if (ret) +return ret; + kmsg = &iomsg; + } + + flags = req->sr_msg.msg_flags; +if (flags &MSG_DONTWAIT ) + req->flags |=REQ_F_NOWAIT ; +else if (force_nonblock) + flags |=MSG_DONTWAIT ; + + ret =__sys_sendmsg_sock (sock, &kmsg->msg, flags); +if (force_nonblock && ret == -EAGAIN ) +return io_setup_async_msg (req, kmsg); +if (ret == -ERESTARTSYS ) + ret = -EINTR ; + +/* it's reportedly faster to check for null here */ +if (kmsg->iov) +kfree (kmsg->iov); + req->flags &= ~REQ_F_NEED_CLEANUP ; +if (ret <0 ) +req_set_fail_links (req); +__io_req_complete (req, ret,0 , cs); +return 0 ; +} + +static int io_send (struct io_kiocb *req,bool force_nonblock, +struct io_comp_state *cs) +{ +struct io_sr_msg *sr = &req->sr_msg; +struct msghdr msg; +struct iovec iov; +struct socket *sock; +unsigned flags; +int ret; + + sock =sock_from_file (req->file, &ret); +if (unlikely (!sock)) +return ret; + + ret =import_single_range (WRITE , sr->buf, sr->len, &iov, &msg.msg_iter); +if (unlikely (ret)) +return ret; + + msg.msg_name =NULL ; + msg.msg_control =NULL ; + msg.msg_controllen =0 ; + msg.msg_namelen =0 ; + + flags = req->sr_msg.msg_flags; +if (flags &MSG_DONTWAIT ) + req->flags |=REQ_F_NOWAIT ; +else if (force_nonblock) + flags |=MSG_DONTWAIT ; + + msg.msg_flags = flags; + ret =sock_sendmsg (sock, &msg); +if (force_nonblock && ret == -EAGAIN ) +return -EAGAIN ; +if (ret == -ERESTARTSYS ) + ret = -EINTR ; + +if (ret <0 ) +req_set_fail_links (req); +__io_req_complete (req, ret,0 , cs); +return 0 ; +} +static inline unsigned int io_put_recv_kbuf (struct io_kiocb *req) {return io_put_kbuf (req, req->sr_msg.kbuf);
Signature · 1 line
2.24.0
PATCH 4/5 io_uring: send/recv with registered buffer
In reply to Pavel BegunkovAdd support of registered buffers to send() and recv(). Done by exploiting last bit of send/recv flags, IO_MSG_FIXED, which is cleared before going into net stack.
--- fs/io_uring.c | 39 +++++++++++++++++++++++++++------------ 1 file changed, 27 insertions(+), 12 deletions(-)
fs/io_uring.c +27 -12
@@ -104,6 +104,8 @@#define IORING_MAX_RESTRICTIONS (IORING_RESTRICTION_LAST + \ IORING_REGISTER_LAST + IORING_OP_LAST) +#define IO_MSG_FIXED (1U << 31) +struct io_uring {u32 head____cacheline_aligned_in_smp ;u32 tail____cacheline_aligned_in_smp ;
@@ -4689,18 +4691,25 @@ static int io_send(struct io_kiocb *req, bool force_nonblock, struct io_comp_state *cs) {struct io_sr_msg *sr = &req->sr_msg; +unsigned int flags = sr->msg_flags;struct msghdr msg;struct iovec iov;struct socket *sock; -unsigned flags;int ret; sock =sock_from_file (req->file, &ret);if (unlikely (!sock))return ret; - ret =import_single_range (WRITE , sr->buf, sr->len, &iov, &msg.msg_iter); -if (unlikely (ret)) +if (flags &IO_MSG_FIXED ) { + ret =io_import_fixed (req,WRITE , (u64 )sr->buf, sr->len, + &msg.msg_iter); + flags &= ~IO_MSG_FIXED ; + }else { + ret =import_single_range (WRITE , sr->buf, sr->len, &iov, + &msg.msg_iter); + } +if (unlikely (ret <0 ))return ret; msg.msg_name =NULL ;
@@ -4708,7 +4717,6 @@ static int io_send(struct io_kiocb *req, bool force_nonblock, msg.msg_controllen =0 ; msg.msg_namelen =0 ; - flags = req->sr_msg.msg_flags;if (flags &MSG_DONTWAIT ) req->flags |=REQ_F_NOWAIT ;else if (force_nonblock)
@@ -4821,15 +4829,22 @@ static int io_recv(struct io_kiocb *req, bool force_nonblock,if (unlikely (!sock))return ret; -if (req->flags &REQ_F_BUFFER_SELECT ) { - kbuf =io_recv_buffer_select (req, !force_nonblock); -if (IS_ERR (kbuf)) -return PTR_ERR (kbuf); - buf =u64_to_user_ptr (kbuf->addr); - } +if (flags &IO_MSG_FIXED ) { + ret =io_import_fixed (req,READ , (u64 )buf, sr->len, + &msg.msg_iter); + flags &= ~IO_MSG_FIXED ; + }else { +if (req->flags &REQ_F_BUFFER_SELECT ) { + kbuf =io_recv_buffer_select (req, !force_nonblock); +if (IS_ERR (kbuf)) +return PTR_ERR (kbuf); + buf =u64_to_user_ptr (kbuf->addr); + } - ret =import_single_range (READ , buf, sr->len, &iov, &msg.msg_iter); -if (unlikely (ret)) + ret =import_single_range (READ , buf, sr->len, &iov, + &msg.msg_iter); + } +if (unlikely (ret <0 )) goto out_free; msg.msg_name =NULL ;
Signature · 1 line
2.24.0
PATCH 5/5 io_uring: sendmsg/recvmsg with registered buffers
In reply to Pavel BegunkovAdd support of registered buffers to sendmsg() and recvmsg(). As with previous one, it uses IO_MSG_FIXED, last bit of flags, which is cleared before going into net stack.
--- fs/io_uring.c | 26 +++++++++++++++++++------- 1 file changed, 19 insertions(+), 7 deletions(-)
fs/io_uring.c +19 -7
@@ -4488,7 +4488,7 @@ static int io_setup_async_msg(struct io_kiocb *req,memcpy (async_msg, kmsg,sizeof (*kmsg)); async_msg->msg.msg_name = &async_msg->addr;/* if iov is not set, it uses fast_iov */ -if (!async_msg->iov) +if (!async_msg->iov && !(req->sr_msg.msg_flags &IO_MSG_FIXED )) async_msg->msg.msg_iter.iov = async_msg->fast_iov;return -EAGAIN ; }
@@ -4508,7 +4508,8 @@ static int __io_msg_copy_hdr(struct io_kiocb *req,if (ret)return ret; -if (req->flags &REQ_F_BUFFER_SELECT ) { +if ((req->flags &REQ_F_BUFFER_SELECT ) || + (sr->msg_flags &IO_MSG_FIXED )) {if (iov_len >1 )return -EINVAL ;if (copy_from_user (iomsg->iov, uiov,sizeof (*uiov)))
@@ -4548,7 +4549,8 @@ static int __io_compat_msg_copy_hdr(struct io_kiocb *req,return ret; uiov =compat_ptr (ptr); -if (req->flags &REQ_F_BUFFER_SELECT ) { +if ((req->flags &REQ_F_BUFFER_SELECT ) || + (sr->msg_flags &IO_MSG_FIXED )) {compat_ssize_t clen;if (len >1 )
@@ -4591,6 +4593,7 @@ static struct io_buffer *io_recv_buffer_select(struct io_kiocb *req,static int io_import_msg (struct io_kiocb *req,struct io_async_msghdr *iomsg,int rw) { +struct io_sr_msg *sr = &req->sr_msg;struct io_buffer *kbuf;int ret;
@@ -4606,7 +4609,7 @@ static int io_import_msg(struct io_kiocb *req, struct io_async_msghdr *iomsg,return ret;if (req->flags &REQ_F_BUFFER_SELECT ) { -if (rw !=READ ) +if (rw !=READ || (sr->msg_flags &IO_MSG_FIXED ))return -EINVAL ;/* init is always done with uring_lock held */ kbuf =io_recv_buffer_select (req, false);
@@ -4614,7 +4617,14 @@ static int io_import_msg(struct io_kiocb *req, struct io_async_msghdr *iomsg,return PTR_ERR (kbuf); iomsg->fast_iov[0 ].iov_base =u64_to_user_ptr (kbuf->addr);iov_iter_init (&iomsg->msg.msg_iter,READ , iomsg->fast_iov,1 , - req->sr_msg.len); + sr->len); + }else if (sr->msg_flags &IO_MSG_FIXED ) { +struct iovec *iov = &iomsg->fast_iov[0 ]; + + ret =io_import_fixed (req, rw, (u64 )iov->iov_base, iov->iov_len, + &iomsg->msg.msg_iter); +if (ret <0 ) +return ret; }return 0 ; }
@@ -4631,6 +4641,7 @@ static int io_sendmsg_prep(struct io_kiocb *req, const struct io_uring_sqe *sqe) sr->msg_flags =READ_ONCE (sqe->msg_flags); sr->umsg =u64_to_user_ptr (READ_ONCE (sqe->addr)); sr->len =READ_ONCE (sqe->len); + req->buf_index =READ_ONCE (sqe->buf_index);#ifdef CONFIG_COMPAT if (req->ctx->compat)
@@ -4665,7 +4676,7 @@ static int io_sendmsg(struct io_kiocb *req, bool force_nonblock, kmsg = &iomsg; } - flags = req->sr_msg.msg_flags; + flags = req->sr_msg.msg_flags & ~IO_MSG_FIXED ;if (flags &MSG_DONTWAIT ) req->flags |=REQ_F_NOWAIT ;else if (force_nonblock)
@@ -4754,6 +4765,7 @@ static int io_recvmsg_prep(struct io_kiocb *req, sr->umsg =u64_to_user_ptr (READ_ONCE (sqe->addr)); sr->len =READ_ONCE (sqe->len); sr->bgid =READ_ONCE (sqe->buf_group); + req->buf_index =READ_ONCE (sqe->buf_index);#ifdef CONFIG_COMPAT if (req->ctx->compat)
@@ -4788,7 +4800,7 @@ static int io_recvmsg(struct io_kiocb *req, bool force_nonblock, kmsg = &iomsg; } - flags = req->sr_msg.msg_flags; + flags = req->sr_msg.msg_flags & ~IO_MSG_FIXED ;if (flags &MSG_DONTWAIT ) req->flags |=REQ_F_NOWAIT ;else if (force_nonblock)
Signature · 1 line
2.24.0
PATCH 3/5 io_uring: opcode independent import_fixed
In reply to Pavel BegunkovPass a buf explicitly into io_import_fixed(), so it can be used not only for rw requests.
--- fs/io_uring.c | 9 +++------ 1 file changed, 3 insertions(+), 6 deletions(-)
fs/io_uring.c +3 -6
@@ -2933,21 +2933,18 @@ static void kiocb_done(struct kiocb *kiocb, ssize_t ret,io_rw_done (kiocb, ret); } -static ssize_t io_import_fixed (struct io_kiocb *req,int rw, -struct iov_iter *iter) +static ssize_t io_import_fixed (struct io_kiocb *req,int rw,u64 buf_addr, +size_t len,struct iov_iter *iter) {struct io_ring_ctx *ctx = req->ctx; -size_t len = req->rw.len;struct io_mapped_ubuf *imu;u16 index, buf_index = req->buf_index;size_t offset; -u64 buf_addr;if (unlikely (buf_index >= ctx->nr_user_bufs))return -EFAULT ; index =array_index_nospec (buf_index, ctx->nr_user_bufs); imu = &ctx->user_bufs[index]; - buf_addr = req->rw.addr;/* overflow */ if (buf_addr + len < buf_addr)
@@ -3153,7 +3150,7 @@ static ssize_t io_import_iovec(int rw, struct io_kiocb *req, opcode = req->opcode;if (opcode ==IORING_OP_READ_FIXED || opcode ==IORING_OP_WRITE_FIXED ) { *iovec =NULL ; -return io_import_fixed (req, rw, iter); +return io_import_fixed (req, rw, req->rw.addr, sqe_len, iter); }/* buffer index only valid with fixed read/write, or buffer select */
Signature · 1 line
2.24.0
PATCH 2/5 io_uring: copy hdr consistently for send and recv
In reply to Pavel Begunkovrecvmsg() goes through a custom written msg headers/iovec copying helper, do that for sendmsg() as well. Apart from being more consistent in general, it allows to extend it (e.g. for registered buffers) without duplication.
--- fs/io_uring.c | 51 ++++++++++++++++++++++++--------------------------- 1 file changed, 24 insertions(+), 27 deletions(-)
fs/io_uring.c +24 -27
@@ -4494,16 +4494,18 @@ static int io_setup_async_msg(struct io_kiocb *req,return -EAGAIN ; } -static int __io_recvmsg_copy_hdr (struct io_kiocb *req, -struct io_async_msghdr *iomsg) +static int __io_msg_copy_hdr (struct io_kiocb *req, +struct io_async_msghdr *iomsg,int rw) { +struct sockaddr __user **save_addr;struct io_sr_msg *sr = &req->sr_msg;struct iovec __user *uiov;size_t iov_len;int ret; - ret =__copy_msghdr_from_user (&iomsg->msg, sr->umsg, - &iomsg->uaddr, &uiov, &iov_len); + save_addr = (rw ==READ ) ? &iomsg->uaddr :NULL ; + ret =__copy_msghdr_from_user (&iomsg->msg, sr->umsg, save_addr, + &uiov, &iov_len);if (ret)return ret;
@@ -4517,7 +4519,7 @@ static int __io_recvmsg_copy_hdr(struct io_kiocb *req, sr->len); iomsg->iov =NULL ; }else { - ret =__import_iovec (READ , uiov, iov_len,UIO_FASTIOV , + ret =__import_iovec (rw, uiov, iov_len,UIO_FASTIOV , &iomsg->iov, &iomsg->msg.msg_iter, false);if (ret >0 )
@@ -4528,9 +4530,10 @@ static int __io_recvmsg_copy_hdr(struct io_kiocb *req, }#ifdef CONFIG_COMPAT -static int __io_compat_recvmsg_copy_hdr (struct io_kiocb *req, -struct io_async_msghdr *iomsg) +static int __io_compat_msg_copy_hdr (struct io_kiocb *req, +struct io_async_msghdr *iomsg,int rw) { +struct sockaddr __user **save_addr;struct compat_msghdr __user *msg_compat;struct io_sr_msg *sr = &req->sr_msg;struct compat_iovec __user *uiov;
@@ -4539,8 +4542,9 @@ static int __io_compat_recvmsg_copy_hdr(struct io_kiocb *req,int ret; msg_compat = (struct compat_msghdr __user *) sr->umsg; - ret =__get_compat_msghdr (&iomsg->msg, msg_compat, &iomsg->uaddr, - &ptr, &len); + save_addr = (rw ==READ ) ? &iomsg->uaddr :NULL ; + ret =__get_compat_msghdr (&iomsg->msg, msg_compat, save_addr, + &ptr, &len);if (ret)return ret;
@@ -4559,7 +4563,7 @@ static int __io_compat_recvmsg_copy_hdr(struct io_kiocb *req, sr->len = iomsg->iov[0 ].iov_len; iomsg->iov =NULL ; }else { - ret =__import_iovec (READ , (struct iovec __user *)uiov, len, + ret =__import_iovec (rw, (struct iovec __user *)uiov, len,UIO_FASTIOV , &iomsg->iov, &iomsg->msg.msg_iter, true);if (ret <0 )
@@ -4585,8 +4589,8 @@ static struct io_buffer *io_recv_buffer_select(struct io_kiocb *req,return kbuf; } -static int io_recvmsg_copy_hdr (struct io_kiocb *req, -struct io_async_msghdr *iomsg) +static int io_import_msg (struct io_kiocb *req,struct io_async_msghdr *iomsg, +int rw) {struct io_buffer *kbuf;int ret;
@@ -4595,14 +4599,16 @@ static int io_recvmsg_copy_hdr(struct io_kiocb *req, iomsg->iov = iomsg->fast_iov;#ifdef CONFIG_COMPAT if (req->ctx->compat) - ret =__io_compat_recvmsg_copy_hdr (req, iomsg); + ret =__io_compat_msg_copy_hdr (req, iomsg, rw); else#endif - ret =__io_recvmsg_copy_hdr (req, iomsg); + ret =__io_msg_copy_hdr (req, iomsg, rw);if (ret <0 )return ret;if (req->flags &REQ_F_BUFFER_SELECT ) { +if (rw !=READ ) +return -EINVAL ;/* init is always done with uring_lock held */ kbuf =io_recv_buffer_select (req, false);if (IS_ERR (kbuf))
@@ -4614,15 +4620,6 @@ static int io_recvmsg_copy_hdr(struct io_kiocb *req,return 0 ; } -static int io_sendmsg_copy_hdr (struct io_kiocb *req, -struct io_async_msghdr *iomsg) -{ - iomsg->iov = iomsg->fast_iov; - iomsg->msg.msg_name = &iomsg->addr; -return sendmsg_copy_msghdr (&iomsg->msg, req->sr_msg.umsg, - req->sr_msg.msg_flags, &iomsg->iov); -} -static int io_sendmsg_prep (struct io_kiocb *req,const struct io_uring_sqe *sqe) {struct io_async_msghdr *async_msg = req->async_data;
@@ -4643,7 +4640,7 @@ static int io_sendmsg_prep(struct io_kiocb *req, const struct io_uring_sqe *sqe)if (!async_msg || !io_op_defs[req->opcode].needs_async_data)return 0 ; - ret =io_sendmsg_copy_hdr (req, async_msg); + ret =io_import_msg (req, async_msg,WRITE );if (!ret) req->flags |=REQ_F_NEED_CLEANUP ;return ret;
@@ -4663,7 +4660,7 @@ static int io_sendmsg(struct io_kiocb *req, bool force_nonblock, kmsg = req->async_data;if (!kmsg) { - ret =io_sendmsg_copy_hdr (req, &iomsg); + ret =io_import_msg (req, &iomsg,WRITE );if (ret)return ret; kmsg = &iomsg;
@@ -4760,7 +4757,7 @@ static int io_recvmsg_prep(struct io_kiocb *req,if (!async_msg || !io_op_defs[req->opcode].needs_async_data)return 0 ; - ret =io_recvmsg_copy_hdr (req, async_msg); + ret =io_import_msg (req, async_msg,READ );if (!ret) req->flags |=REQ_F_NEED_CLEANUP ;return ret;
@@ -4780,7 +4777,7 @@ static int io_recvmsg(struct io_kiocb *req, bool force_nonblock, kmsg = req->async_data;if (!kmsg) { - ret =io_recvmsg_copy_hdr (req, &iomsg); + ret =io_import_msg (req, &iomsg,READ );if (ret)return ret; kmsg = &iomsg;
Signature · 1 line
2.24.0