mix of 6.1 net patches
cover mix of 6.1 net patches
0-4 fix a rare scenario when a partial I/O is completed with an error code rather than the number of bytes processed, which is especially bad with sockets and other non-idempotent files.
5-9 add some net features, specifically 6/9 optionally transforms normal sends into sendto, and 9/9 adds zerocopy sendmsg.
Pavel Begunkov (9):
io_uring: add custom opcode hooks on fail io_uring/rw: don't lose partial IO result on fail io_uring/net: don't lose partial send/recv on fail io_uring/net: don't lose partial send_zc on fail io_uring/net: refactor io_setup_async_addr io_uring/net: support non-zerocopy sendto io_uring/net: rename io_sendzc() io_uring/net: combine fail handlers io_uring/net: zerocopy sendmsg
include/uapi/linux/io_uring.h | 1 + io_uring/io_uring.c | 4 + io_uring/net.c | 176 +++++++++++++++++++++++++++++----- io_uring/net.h | 12 ++- io_uring/opdef.c | 41 +++++++- io_uring/opdef.h | 1 + io_uring/rw.c | 8 ++ io_uring/rw.h | 1 + 8 files changed, 214 insertions(+), 30 deletions(-)
Signature · 1 line
2.37.2
PATCH 1/9 io_uring: add custom opcode hooks on fail
Sometimes we have to do a little bit of a fixup on a request failuer in io_req_complete_failed(). Add a callback in opdef for that.
--- io_uring/io_uring.c | 4 ++++ io_uring/opdef.h | 1 + 2 files changed, 5 insertions(+)
io_uring/io_uring.c +4 -0
@@ -865,8 +865,12 @@ inline void __io_req_complete(struct io_kiocb *req, unsigned issue_flags)void io_req_complete_failed (struct io_kiocb *req,s32 res) { +const struct io_op_def *def = &io_op_defs[req->opcode]; +req_set_fail (req);io_req_set_res (req, res,io_put_kbuf (req,IO_URING_F_UNLOCKED )); +if (def->fail) + def->fail (req);io_req_complete_post (req); }
io_uring/opdef.h +1 -0
@@ -36,6 +36,7 @@ struct io_op_def {int (*issue)(struct io_kiocb *,unsigned int );int (*prep_async)(struct io_kiocb *);void (*cleanup)(struct io_kiocb *); +void (*fail)(struct io_kiocb *); };extern const struct io_op_def io_op_defs[];
Signature · 1 line
2.37.2
PATCH 2/9 io_uring/rw: don't lose partial IO result on fail
In reply to Pavel BegunkovA partially done read/write may end up in io_req_complete_failed() and loose the result, make sure we return the number of bytes processed.
--- io_uring/opdef.c | 6 ++++++ io_uring/rw.c | 8 ++++++++ io_uring/rw.h | 1 + 3 files changed, 15 insertions(+)
io_uring/opdef.c +6 -0
@@ -69,6 +69,7 @@ const struct io_op_def io_op_defs[] = { .issue = io_read, .prep_async = io_readv_prep_async, .cleanup = io_readv_writev_cleanup, + .fail = io_rw_fail, }, [IORING_OP_WRITEV] = { .needs_file = 1,
@@ -85,6 +86,7 @@ const struct io_op_def io_op_defs[] = { .issue = io_write, .prep_async = io_writev_prep_async, .cleanup = io_readv_writev_cleanup, + .fail = io_rw_fail, }, [IORING_OP_FSYNC] = { .needs_file = 1,
@@ -105,6 +107,7 @@ const struct io_op_def io_op_defs[] = { .name = "READ_FIXED", .prep = io_prep_rw, .issue = io_read, + .fail = io_rw_fail, }, [IORING_OP_WRITE_FIXED] = { .needs_file = 1,
@@ -119,6 +122,7 @@ const struct io_op_def io_op_defs[] = { .name = "WRITE_FIXED", .prep = io_prep_rw, .issue = io_write, + .fail = io_rw_fail, }, [IORING_OP_POLL_ADD] = { .needs_file = 1,
@@ -275,6 +279,7 @@ const struct io_op_def io_op_defs[] = { .name = "READ", .prep = io_prep_rw, .issue = io_read, + .fail = io_rw_fail, }, [IORING_OP_WRITE] = { .needs_file = 1,
@@ -289,6 +294,7 @@ const struct io_op_def io_op_defs[] = { .name = "WRITE", .prep = io_prep_rw, .issue = io_write, + .fail = io_rw_fail, }, [IORING_OP_FADVISE] = { .needs_file = 1,
io_uring/rw.c +8 -0
@@ -954,6 +954,14 @@ static void io_cqring_ev_posted_iopoll(struct io_ring_ctx *ctx)io_cqring_wake (ctx); } +void io_rw_fail (struct io_kiocb *req) +{ +int res; + + res =io_fixup_rw_res (req, req->cqe.res); +io_req_set_res (req, res, req->cqe.flags); +} +int io_do_iopoll (struct io_ring_ctx *ctx,bool force_nonspin) {struct io_wq_work_node *pos, *start, *prev;
io_uring/rw.h +1 -0
@@ -21,3 +21,4 @@ int io_readv_prep_async(struct io_kiocb *req);int io_write (struct io_kiocb *req,unsigned int issue_flags);int io_writev_prep_async (struct io_kiocb *req);void io_readv_writev_cleanup (struct io_kiocb *req); +void io_rw_fail (struct io_kiocb *req);
Signature · 1 line
2.37.2
PATCH 3/9 io_uring/net: don't lose partial send/recv on fail
In reply to Pavel BegunkovJust as with rw, partial send/recv may end up in io_req_complete_failed() and loose the result, make sure we return the number of bytes processed.
--- io_uring/net.c | 10 ++++++++++ io_uring/net.h | 2 ++ io_uring/opdef.c | 4 ++++ 3 files changed, 16 insertions(+)
io_uring/net.c +10 -0
@@ -1093,6 +1093,16 @@ int io_sendzc(struct io_kiocb *req, unsigned int issue_flags)return IOU_OK ; } +void io_sendrecv_fail (struct io_kiocb *req) +{ + struct io_sr_msg *sr = io_kiocb_to_cmd(req, struct io_sr_msg); +int res = req->cqe.res; + +if (req->flags &REQ_F_PARTIAL_IO ) + res = sr->done_io; +io_req_set_res (req, res, req->cqe.flags); +} +int io_accept_prep (struct io_kiocb *req,const struct io_uring_sqe *sqe) { struct io_accept *accept = io_kiocb_to_cmd(req, struct io_accept);
io_uring/net.h +2 -0
@@ -43,6 +43,8 @@ int io_recvmsg_prep(struct io_kiocb *req, const struct io_uring_sqe *sqe);int io_recvmsg (struct io_kiocb *req,unsigned int issue_flags);int io_recv (struct io_kiocb *req,unsigned int issue_flags); +void io_sendrecv_fail (struct io_kiocb *req); +int io_accept_prep (struct io_kiocb *req,const struct io_uring_sqe *sqe);int io_accept (struct io_kiocb *req,unsigned int issue_flags);
io_uring/opdef.c +4 -0
@@ -158,6 +158,7 @@ const struct io_op_def io_op_defs[] = { .issue = io_sendmsg, .prep_async = io_sendmsg_prep_async, .cleanup = io_sendmsg_recvmsg_cleanup, + .fail = io_sendrecv_fail, #else .prep = io_eopnotsupp_prep,#endif
@@ -176,6 +177,7 @@ const struct io_op_def io_op_defs[] = { .issue = io_recvmsg, .prep_async = io_recvmsg_prep_async, .cleanup = io_sendmsg_recvmsg_cleanup, + .fail = io_sendrecv_fail, #else .prep = io_eopnotsupp_prep,#endif
@@ -318,6 +320,7 @@ const struct io_op_def io_op_defs[] = {#if defined(CONFIG_NET ) .prep = io_sendmsg_prep, .issue = io_send, + .fail = io_sendrecv_fail, #else .prep = io_eopnotsupp_prep,#endif
@@ -333,6 +336,7 @@ const struct io_op_def io_op_defs[] = {#if defined(CONFIG_NET ) .prep = io_recvmsg_prep, .issue = io_recv, + .fail = io_sendrecv_fail, #else .prep = io_eopnotsupp_prep,#endif
Signature · 1 line
2.37.2
PATCH 4/9 io_uring/net: don't lose partial send_zc on fail
In reply to Pavel BegunkovPartial zc send may end up in io_req_complete_failed(), which not only would return invalid result but also mask out the notification leading to lifetime issues.
--- io_uring/net.c | 16 ++++++++++++++++ io_uring/net.h | 1 + io_uring/opdef.c | 1 + 3 files changed, 18 insertions(+)
io_uring/net.c +16 -0
@@ -1103,6 +1103,22 @@ void io_sendrecv_fail(struct io_kiocb *req)io_req_set_res (req, res, req->cqe.flags); } +void io_send_zc_fail (struct io_kiocb *req) +{ + struct io_sr_msg *sr = io_kiocb_to_cmd(req, struct io_sr_msg); +int res = req->cqe.res; + +if (req->flags &REQ_F_PARTIAL_IO ) { +if (req->flags &REQ_F_NEED_CLEANUP ) { +io_notif_flush (sr->notif); + sr->notif =NULL ; + req->flags &= ~REQ_F_NEED_CLEANUP ; + } + res = sr->done_io; + } +io_req_set_res (req, res, req->cqe.flags); +} +int io_accept_prep (struct io_kiocb *req,const struct io_uring_sqe *sqe) { struct io_accept *accept = io_kiocb_to_cmd(req, struct io_accept);
io_uring/net.h +1 -0
@@ -58,6 +58,7 @@ int io_connect(struct io_kiocb *req, unsigned int issue_flags);int io_sendzc (struct io_kiocb *req,unsigned int issue_flags);int io_sendzc_prep (struct io_kiocb *req,const struct io_uring_sqe *sqe);void io_sendzc_cleanup (struct io_kiocb *req); +void io_send_zc_fail (struct io_kiocb *req);void io_netmsg_cache_free (struct io_cache_entry *entry);#else
io_uring/opdef.c +1 -0
@@ -497,6 +497,7 @@ const struct io_op_def io_op_defs[] = { .issue = io_sendzc, .prep_async = io_sendzc_prep_async, .cleanup = io_sendzc_cleanup, + .fail = io_send_zc_fail, #else .prep = io_eopnotsupp_prep,#endif
Signature · 1 line
2.37.2
PATCH 5/9 io_uring/net: refactor io_setup_async_addr
In reply to Pavel BegunkovInstead of passing the right address into io_setup_async_addr() only specify local on-stack storage and let the function infer where to grab it from. It optimises out one local variable we have to deal with.
--- io_uring/net.c | 18 +++++++++--------- 1 file changed, 9 insertions(+), 9 deletions(-)
io_uring/net.c +9 -9
@@ -196,17 +196,18 @@ int io_sendzc_prep_async(struct io_kiocb *req) }static int io_setup_async_addr (struct io_kiocb *req, -struct sockaddr_storage *addr, +struct sockaddr_storage *addr_storage,unsigned int issue_flags) { + struct io_sr_msg *sr = io_kiocb_to_cmd(req, struct io_sr_msg);struct io_async_msghdr *io; -if (!addr ||req_has_async_data (req)) +if (!sr->addr ||req_has_async_data (req))return -EAGAIN ; io =io_msg_alloc_async (req, issue_flags);if (!io)return -ENOMEM ; -memcpy (&io->addr, addr,sizeof (io->addr)); +memcpy (&io->addr, addr_storage,sizeof (io->addr));return -EAGAIN ; }
@@ -1000,7 +1001,7 @@ static int io_sg_from_iter(struct sock *sk, struct sk_buff *skb,int io_sendzc (struct io_kiocb *req,unsigned int issue_flags) { -struct sockaddr_storage __address, *addr =NULL ; +struct sockaddr_storage __address; struct io_sr_msg *zc = io_kiocb_to_cmd(req, struct io_sr_msg);struct msghdr msg;struct iovec iov;
@@ -1021,20 +1022,19 @@ int io_sendzc(struct io_kiocb *req, unsigned int issue_flags)if (req_has_async_data (req)) {struct io_async_msghdr *io = req->async_data; - msg.msg_name = addr = &io->addr; + msg.msg_name = &io->addr; }else { ret =move_addr_to_kernel (zc->addr, zc->addr_len, &__address);if (unlikely (ret <0 ))return ret; msg.msg_name = (struct sockaddr *)&__address; - addr = &__address; } msg.msg_namelen = zc->addr_len; }if (!(req->flags &REQ_F_POLLED ) && (zc->flags &IORING_RECVSEND_POLL_FIRST )) -return io_setup_async_addr (req, addr, issue_flags); +return io_setup_async_addr (req, &__address, issue_flags);if (zc->flags &IORING_RECVSEND_FIXED_BUF ) { ret = io_import_fixed(WRITE, &msg.msg_iter, req->imu,
@@ -1065,14 +1065,14 @@ int io_sendzc(struct io_kiocb *req, unsigned int issue_flags)if (unlikely (ret < min_ret)) {if (ret == -EAGAIN && (issue_flags &IO_URING_F_NONBLOCK )) -return io_setup_async_addr (req, addr, issue_flags); +return io_setup_async_addr (req, &__address, issue_flags);if (ret >0 &&io_net_retry (sock, msg.msg_flags)) { zc->len -= ret; zc->buf += ret; zc->done_io += ret; req->flags |=REQ_F_PARTIAL_IO ; -return io_setup_async_addr (req, addr, issue_flags); +return io_setup_async_addr (req, &__address, issue_flags); }if (ret <0 && !zc->done_io) zc->notif->flags |=REQ_F_CQE_SKIP ;
Signature · 1 line
2.37.2
PATCH 6/9 io_uring/net: support non-zerocopy sendto
In reply to Pavel BegunkovWe have normal sends, but what is missing is sendto-like requests. Add sendto() capabilities to IORING_OP_SEND by passing in addr just as we do for IORING_OP_SEND_ZC.
--- io_uring/net.c | 35 +++++++++++++++++++++++++++++------ io_uring/net.h | 3 ++- io_uring/opdef.c | 5 ++++- 3 files changed, 35 insertions(+), 8 deletions(-)
io_uring/net.c +29 -6
@@ -59,9 +59,10 @@ struct io_sr_msg {unsigned done_io;unsigned msg_flags;u16 flags; -/* used only for sendzc */ +/* initialised and used only by !msg send variants */ u16 addr_len;void __user *addr; +/* used only for send zerocopy */ struct io_kiocb *notif; };
@@ -180,7 +181,7 @@ static int io_sendmsg_copy_hdr(struct io_kiocb *req, &iomsg->free_iov); } -int io_sendzc_prep_async (struct io_kiocb *req) +int io_send_prep_async (struct io_kiocb *req) { struct io_sr_msg *zc = io_kiocb_to_cmd(req, struct io_sr_msg);struct io_async_msghdr *io;
@@ -234,8 +235,14 @@ int io_sendmsg_prep(struct io_kiocb *req, const struct io_uring_sqe *sqe) { struct io_sr_msg *sr = io_kiocb_to_cmd(req, struct io_sr_msg); -if (unlikely (sqe->file_index || sqe->addr2)) +if (req->opcode ==IORING_OP_SEND ) { +if (READ_ONCE (sqe->__pad3[0 ])) +return -EINVAL ; + sr->addr =u64_to_user_ptr (READ_ONCE (sqe->addr2)); + sr->addr_len =READ_ONCE (sqe->addr_len); + }else if (sqe->addr2 || sqe->file_index) {return -EINVAL ; + } sr->umsg =u64_to_user_ptr (READ_ONCE (sqe->addr)); sr->len =READ_ONCE (sqe->len);
@@ -315,6 +322,7 @@ int io_sendmsg(struct io_kiocb *req, unsigned int issue_flags)int io_send (struct io_kiocb *req,unsigned int issue_flags) { +struct sockaddr_storage __address; struct io_sr_msg *sr = io_kiocb_to_cmd(req, struct io_sr_msg);struct msghdr msg;struct iovec iov;
@@ -323,9 +331,23 @@ int io_send(struct io_kiocb *req, unsigned int issue_flags)int min_ret =0 ;int ret; +if (sr->addr) { +if (req_has_async_data (req)) { +struct io_async_msghdr *io = req->async_data; + + msg.msg_name = &io->addr; + }else { + ret =move_addr_to_kernel (sr->addr, sr->addr_len, &__address); +if (unlikely (ret <0 )) +return ret; + msg.msg_name = (struct sockaddr *)&__address; + } + msg.msg_namelen = sr->addr_len; + } +if (!(req->flags &REQ_F_POLLED ) && (sr->flags &IORING_RECVSEND_POLL_FIRST )) -return -EAGAIN ; +return io_setup_async_addr (req, &__address, issue_flags); sock =sock_from_file (req->file);if (unlikely (!sock))
@@ -351,13 +373,14 @@ int io_send(struct io_kiocb *req, unsigned int issue_flags) ret =sock_sendmsg (sock, &msg);if (ret < min_ret) {if (ret == -EAGAIN && (issue_flags &IO_URING_F_NONBLOCK )) -return -EAGAIN ; +return io_setup_async_addr (req, &__address, issue_flags); +if (ret >0 &&io_net_retry (sock, flags)) { sr->len -= ret; sr->buf += ret; sr->done_io += ret; req->flags |=REQ_F_PARTIAL_IO ; -return -EAGAIN ; +return io_setup_async_addr (req, &__address, issue_flags); }if (ret == -ERESTARTSYS ) ret = -EINTR ;
io_uring/net.h +2 -1
@@ -31,12 +31,13 @@ struct io_async_connect {int io_shutdown_prep (struct io_kiocb *req,const struct io_uring_sqe *sqe);int io_shutdown (struct io_kiocb *req,unsigned int issue_flags); -int io_sendzc_prep_async (struct io_kiocb *req);int io_sendmsg_prep_async (struct io_kiocb *req);void io_sendmsg_recvmsg_cleanup (struct io_kiocb *req);int io_sendmsg_prep (struct io_kiocb *req,const struct io_uring_sqe *sqe);int io_sendmsg (struct io_kiocb *req,unsigned int issue_flags); +int io_send (struct io_kiocb *req,unsigned int issue_flags); +int io_send_prep_async (struct io_kiocb *req);int io_recvmsg_prep_async (struct io_kiocb *req);int io_recvmsg_prep (struct io_kiocb *req,const struct io_uring_sqe *sqe);
io_uring/opdef.c +4 -1
@@ -316,11 +316,14 @@ const struct io_op_def io_op_defs[] = { .pollout = 1, .audit_skip = 1, .ioprio = 1, + .manual_alloc = 1, .name = "SEND",#if defined(CONFIG_NET ) + .async_size = sizeof(struct io_async_msghdr), .prep = io_sendmsg_prep, .issue = io_send, .fail = io_sendrecv_fail, + .prep_async = io_send_prep_async, #else .prep = io_eopnotsupp_prep,#endif
@@ -495,7 +498,7 @@ const struct io_op_def io_op_defs[] = { .async_size = sizeof(struct io_async_msghdr), .prep = io_sendzc_prep, .issue = io_sendzc, - .prep_async = io_sendzc_prep_async, + .prep_async = io_send_prep_async, .cleanup = io_sendzc_cleanup, .fail = io_send_zc_fail,#else
Signature · 1 line
2.37.2
PATCH 8/9 io_uring/net: combine fail handlers
In reply to Pavel BegunkovMerge io_send_zc_fail() into io_sendrecv_fail(), saves a few lines of code and some headache for following patch.
--- io_uring/net.c | 31 ++++++++++++++++--------------- io_uring/net.h | 1 - io_uring/opdef.c | 2 +- 3 files changed, 17 insertions(+), 17 deletions(-)
io_uring/net.c +16 -15
@@ -192,6 +192,7 @@ int io_send_prep_async(struct io_kiocb *req) io =io_msg_alloc_async_prep (req);if (!io)return -ENOMEM ; + io->free_iov =NULL ; ret =move_addr_to_kernel (zc->addr, zc->addr_len, &io->addr);return ret; }
@@ -208,6 +209,7 @@ static int io_setup_async_addr(struct io_kiocb *req, io =io_msg_alloc_async (req, issue_flags);if (!io)return -ENOMEM ; + io->free_iov =NULL ;memcpy (&io->addr, addr_storage,sizeof (io->addr));return -EAGAIN ; }
@@ -1119,26 +1121,25 @@ int io_send_zc(struct io_kiocb *req, unsigned int issue_flags)void io_sendrecv_fail (struct io_kiocb *req) { struct io_sr_msg *sr = io_kiocb_to_cmd(req, struct io_sr_msg); +struct io_async_msghdr *io;int res = req->cqe.res;if (req->flags &REQ_F_PARTIAL_IO ) res = sr->done_io; -io_req_set_res (req, res, req->cqe.flags); -} - -void io_send_zc_fail (struct io_kiocb *req) -{ - struct io_sr_msg *sr = io_kiocb_to_cmd(req, struct io_sr_msg); -int res = req->cqe.res; - -if (req->flags &REQ_F_PARTIAL_IO ) { -if (req->flags &REQ_F_NEED_CLEANUP ) { -io_notif_flush (sr->notif); - sr->notif =NULL ; - req->flags &= ~REQ_F_NEED_CLEANUP ; - } - res = sr->done_io; +if ((req->flags &REQ_F_NEED_CLEANUP ) && + req->opcode ==IORING_OP_SEND_ZC ) { +/* preserve notification for partial I/O */ +if (res <0 ) + sr->notif->flags |=REQ_F_CQE_SKIP ; +io_notif_flush (sr->notif); + sr->notif =NULL ; } +if (req_has_async_data (req)) { + io = req->async_data; +kfree (io->free_iov); + io->free_iov =NULL ; + } + req->flags &= ~REQ_F_NEED_CLEANUP ;io_req_set_res (req, res, req->cqe.flags); }
io_uring/net.h +0 -1
@@ -59,7 +59,6 @@ int io_connect(struct io_kiocb *req, unsigned int issue_flags);int io_send_zc (struct io_kiocb *req,unsigned int issue_flags);int io_send_zc_prep (struct io_kiocb *req,const struct io_uring_sqe *sqe);void io_send_zc_cleanup (struct io_kiocb *req); -void io_send_zc_fail (struct io_kiocb *req);void io_netmsg_cache_free (struct io_cache_entry *entry);#else
io_uring/opdef.c +1 -1
@@ -500,7 +500,7 @@ const struct io_op_def io_op_defs[] = { .issue = io_send_zc, .prep_async = io_send_prep_async, .cleanup = io_send_zc_cleanup, - .fail = io_send_zc_fail, + .fail = io_sendrecv_fail, #else .prep = io_eopnotsupp_prep,#endif
Signature · 1 line
2.37.2
PATCH 7/9 io_uring/net: rename io_sendzc()
In reply to Pavel BegunkovSimple renaming of io_sendzc*() functions in preparatio to adding a zerocopy sendmsg variant.
--- io_uring/net.c | 6 +++--- io_uring/net.h | 6 +++--- io_uring/opdef.c | 6 +++--- 3 files changed, 9 insertions(+), 9 deletions(-)
io_uring/net.c +3 -3
@@ -904,7 +904,7 @@ int io_recv(struct io_kiocb *req, unsigned int issue_flags)return ret; } -void io_sendzc_cleanup (struct io_kiocb *req) +void io_send_zc_cleanup (struct io_kiocb *req) { struct io_sr_msg *zc = io_kiocb_to_cmd(req, struct io_sr_msg);
@@ -913,7 +913,7 @@ void io_sendzc_cleanup(struct io_kiocb *req) zc->notif =NULL ; } -int io_sendzc_prep (struct io_kiocb *req,const struct io_uring_sqe *sqe) +int io_send_zc_prep (struct io_kiocb *req,const struct io_uring_sqe *sqe) { struct io_sr_msg *zc = io_kiocb_to_cmd(req, struct io_sr_msg);struct io_ring_ctx *ctx = req->ctx;
@@ -1022,7 +1022,7 @@ static int io_sg_from_iter(struct sock *sk, struct sk_buff *skb,return ret; } -int io_sendzc (struct io_kiocb *req,unsigned int issue_flags) +int io_send_zc (struct io_kiocb *req,unsigned int issue_flags) {struct sockaddr_storage __address; struct io_sr_msg *zc = io_kiocb_to_cmd(req, struct io_sr_msg);
io_uring/net.h +3 -3
@@ -56,9 +56,9 @@ int io_connect_prep_async(struct io_kiocb *req);int io_connect_prep (struct io_kiocb *req,const struct io_uring_sqe *sqe);int io_connect (struct io_kiocb *req,unsigned int issue_flags); -int io_sendzc (struct io_kiocb *req,unsigned int issue_flags); -int io_sendzc_prep (struct io_kiocb *req,const struct io_uring_sqe *sqe); -void io_sendzc_cleanup (struct io_kiocb *req); +int io_send_zc (struct io_kiocb *req,unsigned int issue_flags); +int io_send_zc_prep (struct io_kiocb *req,const struct io_uring_sqe *sqe); +void io_send_zc_cleanup (struct io_kiocb *req);void io_send_zc_fail (struct io_kiocb *req);void io_netmsg_cache_free (struct io_cache_entry *entry);
io_uring/opdef.c +3 -3
@@ -496,10 +496,10 @@ const struct io_op_def io_op_defs[] = { .manual_alloc = 1,#if defined(CONFIG_NET ) .async_size = sizeof(struct io_async_msghdr), - .prep = io_sendzc_prep, - .issue = io_sendzc, + .prep = io_send_zc_prep, + .issue = io_send_zc, .prep_async = io_send_prep_async, - .cleanup = io_sendzc_cleanup, + .cleanup = io_send_zc_cleanup, .fail = io_send_zc_fail, #else .prep = io_eopnotsupp_prep,
Signature · 1 line
2.37.2
PATCH 9/9 io_uring/net: zerocopy sendmsg
In reply to Pavel BegunkovAdd a zerocopy version of sendmsg.
--- include/uapi/linux/io_uring.h | 1 + io_uring/net.c | 92 +++++++++++++++++++++++++++++++++-- io_uring/net.h | 1 + io_uring/opdef.c | 19 ++++++++ 4 files changed, 108 insertions(+), 5 deletions(-)
include/uapi/linux/io_uring.h +1 -0
@@ -213,6 +213,7 @@ enum io_uring_op {IORING_OP_SOCKET ,IORING_OP_URING_CMD ,IORING_OP_SEND_ZC , +IORING_OP_SENDMSG_ZC ,/* this goes last, obviously */ IORING_OP_LAST,
io_uring/net.c +87 -5
129 lines not on this page · raw message
io_uring/net.h +1 -0
@@ -57,6 +57,7 @@ int io_connect_prep(struct io_kiocb *req, const struct io_uring_sqe *sqe);int io_connect (struct io_kiocb *req,unsigned int issue_flags);int io_send_zc (struct io_kiocb *req,unsigned int issue_flags); +int io_sendmsg_zc (struct io_kiocb *req,unsigned int issue_flags);int io_send_zc_prep (struct io_kiocb *req,const struct io_uring_sqe *sqe);void io_send_zc_cleanup (struct io_kiocb *req);
io_uring/opdef.c +19 -0
@@ -503,6 +503,25 @@ const struct io_op_def io_op_defs[] = { .fail = io_sendrecv_fail, #else .prep = io_eopnotsupp_prep, +#endif + }, + [IORING_OP_SENDMSG_ZC ] = { + .name ="SENDMSG_ZC" , + .needs_file =1 , + .unbound_nonreg_file =1 , + .pollout =1 , + .audit_skip =1 , + .ioprio =1 , + .manual_alloc =1 , +#if defined(CONFIG_NET) + .async_size =sizeof (struct io_async_msghdr ), + .prep = io_send_zc_prep, + .issue = io_sendmsg_zc, + .prep_async = io_sendmsg_prep_async, + .cleanup = io_send_zc_cleanup, + .fail = io_sendrecv_fail, +#else + .prep = io_eopnotsupp_prep, #endif }, };
Signature · 1 line
2.37.2
Quoted 8 lines · Pavel Begunkov
0-4 fix a rare scenario when a partial I/O is completed with an error code rather than the number of bytes processed, which is especially bad with sockets and other non-idempotent files.
5-9 add some net features, specifically 6/9 optionally transforms normal sends into sendto, and 9/9 adds zerocopy sendmsg.
[...]
Applied, thanks!
[1/9] io_uring: add custom opcode hooks on fail
commit: 18b24300d6d3223192e69216d76ea273fa4b4f25
[2/9] io_uring/rw: don't lose partial IO result on fail
commit: 7f4149434dbd8a9c356a959263b9e327bc5ed65b
[3/9] io_uring/net: don't lose partial send/recv on fail
commit: 9a837772f2745351610e3554e193caf82e315bb3
[4/9] io_uring/net: don't lose partial send_zc on fail
commit: 98e7a0e1c8102354ab28b46be737258739f7c92c
[5/9] io_uring/net: refactor io_setup_async_addr
commit: 9f60a56760986bf3f2be3f906d2cd65215d0c065
[6/9] io_uring/net: support non-zerocopy sendto
commit: ce6a8fdbb23f05cb3f6ddc6d0d76274eadf56a10
[7/9] io_uring/net: rename io_sendzc()
commit: a7093eb542f723cc8f8e9d122d0aa82061f59ab4
[8/9] io_uring/net: combine fail handlers
commit: 8cf8464184f4141294a9e86820b74a49e18a2436
[9/9] io_uring/net: zerocopy sendmsg
commit: 2ec37c3b63d326147d7b70913b17c4ea3295c47bBest regards,
Signature · 1 line
Jens Axboe