io_uring/net: use REQ_F_IMPORT_BUFFER for send_zc
PATCH 1/1 io_uring/net: use REQ_F_IMPORT_BUFFER for send_zc
Instead of a bool field in struct io_sr_msg, use REQ_F_IMPORT_BUFFER to track whether io_send_zc() has already imported the buffer. This flag already serves a similar purpose for sendmsg_zc and {read,write}v_fixed.
--- include/linux/io_uring_types.h | 5 ++++- io_uring/net.c | 8 +++----- 2 files changed, 7 insertions(+), 6 deletions(-)
include/linux/io_uring_types.h +4 -1
@@ -583,11 +583,14 @@ enum {REQ_F_BUFFERS_COMMIT =IO_REQ_FLAG (REQ_F_BUFFERS_COMMIT_BIT ),/* buf node is valid */ REQ_F_BUF_NODE =IO_REQ_FLAG (REQ_F_BUF_NODE_BIT ),/* request has read/write metadata assigned */ REQ_F_HAS_METADATA =IO_REQ_FLAG (REQ_F_HAS_METADATA_BIT ), -/* resolve padded iovec to registered buffers */ +/* +* For vectored fixed buffers, resolve iovec to registered buffers. +* For SEND_ZC, whether to import buffers (i.e. the first issue). +*/ REQ_F_IMPORT_BUFFER = IO_REQ_FLAG(REQ_F_IMPORT_BUFFER_BIT), };typedef void (*io_req_tw_func_t )(struct io_kiocb *req,io_tw_token_t tw);
io_uring/net.c +3 -5
@@ -75,11 +75,10 @@ struct io_sr_msg {unsigned nr_multishot_loops;u16 flags;/* initialised and used only by !msg send variants */ u16 buf_group;bool retry; -bool imported;/* only for io_send_zc */ void __user *msg_control;/* used only for send zerocopy */ struct io_kiocb *notif; };
@@ -1305,12 +1304,11 @@ int io_send_zc_prep(struct io_kiocb *req, const struct io_uring_sqe *sqe)struct io_ring_ctx *ctx = req->ctx;struct io_kiocb *notif; zc->done_io =0 ; zc->retry = false; - zc->imported = false; - req->flags |=REQ_F_POLL_NO_LAZY ; + req->flags |=REQ_F_POLL_NO_LAZY |REQ_F_IMPORT_BUFFER ;if (unlikely (READ_ONCE (sqe->__pad2[0 ]) ||READ_ONCE (sqe->addr3)))return -EINVAL ;/* we don't support IOSQE_CQE_SKIP_SUCCESS just yet */ if (req->flags &REQ_F_CQE_SKIP )
@@ -1447,12 +1445,12 @@ int io_send_zc(struct io_kiocb *req, unsigned int issue_flags)if (unlikely (!sock))return -ENOTSOCK ;if (!test_bit (SOCK_SUPPORT_ZC , &sock->flags))return -EOPNOTSUPP ; -if (!zc->imported) { - zc->imported = true; +if (req->flags &REQ_F_IMPORT_BUFFER ) { + req->flags &= ~REQ_F_IMPORT_BUFFER ; ret =io_send_zc_import (req, issue_flags);if (unlikely (ret))return ret; }
Signature · 1 line
2.45.2
Quoted 48 lines · Caleb Sander Mateos
Instead of a bool field in struct
io_sr_msg, useREQ_F_IMPORT_BUFFERto track whetherio_send_zc()has already imported the buffer. This flag already serves a similar purpose forsendmsg_zcand {read,write}v_fixed.Signed-off-by Caleb Sander Mateos <[email protected]>Suggested-by Pavel Begunkov <[email protected]>--- include/linux/io_uring_types.h | 5 ++++- io_uring/net.c | 8 +++----- 2 files changed, 7 insertions(+), 6 deletions(-)include/linux/io_uring_types.h +4
-1@@ -583,11 +583,14 @@ enum {REQ_F_BUFFERS_COMMIT =IO_REQ_FLAG (REQ_F_BUFFERS_COMMIT_BIT ),/* buf node is valid */ REQ_F_BUF_NODE =IO_REQ_FLAG (REQ_F_BUF_NODE_BIT ),/* request has read/write metadata assigned */ REQ_F_HAS_METADATA =IO_REQ_FLAG (REQ_F_HAS_METADATA_BIT ), -/* resolve padded iovec to registered buffers */ +/* +* For vectored fixed buffers, resolve iovec to registered buffers. +* For SEND_ZC, whether to import buffers (i.e. the first issue). +*/ REQ_F_IMPORT_BUFFER = IO_REQ_FLAG(REQ_F_IMPORT_BUFFER_BIT), };typedef void (*io_req_tw_func_t )(struct io_kiocb *req,io_tw_token_t tw);io_uring/net.c +0
-1@@ -75,11 +75,10 @@ struct io_sr_msg {unsigned nr_multishot_loops;u16 flags;/* initialised and used only by !msg send variants */ u16 buf_group;bool retry; -bool imported;/* only for io_send_zc */ void __user *msg_control;/* used only for send zerocopy */ struct io_kiocb *notif; };
io_uring/net.c int io_send_zc_prep(struct io_kiocb *req, const struct io_uring_sqe *sqe) · line 1309
Quoted 8 lines
io_uring/net.c +1
-2@@ -1305,12 +1304,11 @@ int io_send_zc_prep(struct io_kiocb *req, const struct io_uring_sqe *sqe)struct io_ring_ctx *ctx = req->ctx;struct io_kiocb *notif; zc->done_io =0 ; zc->retry = false; - zc->imported = false; - req->flags |=REQ_F_POLL_NO_LAZY ; + req->flags |=REQ_F_POLL_NO_LAZY |REQ_F_IMPORT_BUFFER ;
This function is shared with sendmsg_zc, so if we set it here, it'll trigger io_import_reg_vec() in io_sendmsg_zc() even for non register buffer request.
Quoted 20 lines
io_uring/net.c +2
-2if (unlikely (READ_ONCE (sqe->__pad2[0 ]) ||READ_ONCE (sqe->addr3)))return -EINVAL ;/* we don't support IOSQE_CQE_SKIP_SUCCESS just yet */ if (req->flags &REQ_F_CQE_SKIP )@@ -1447,12 +1445,12 @@ int io_send_zc(struct io_kiocb *req, unsigned int issue_flags)if (unlikely (!sock))return -ENOTSOCK ;if (!test_bit (SOCK_SUPPORT_ZC , &sock->flags))return -EOPNOTSUPP ; -if (!zc->imported) { - zc->imported = true; +if (req->flags &REQ_F_IMPORT_BUFFER ) { + req->flags &= ~REQ_F_IMPORT_BUFFER ; ret =io_send_zc_import (req, issue_flags);if (unlikely (ret))return ret; }
Signature · 1 line
Pavel Begunkov
Quoted 53 lines · Pavel Begunkov
Quoted 51 lines · Caleb Sander Mateos
Instead of a bool field in struct
io_sr_msg, useREQ_F_IMPORT_BUFFERto track whetherio_send_zc()has already imported the buffer. This flag already serves a similar purpose forsendmsg_zcand {read,write}v_fixed.Signed-off-by Caleb Sander Mateos <[email protected]>Suggested-by Pavel Begunkov <[email protected]>--- include/linux/io_uring_types.h | 5 ++++- io_uring/net.c | 8 +++----- 2 files changed, 7 insertions(+), 6 deletions(-)include/linux/io_uring_types.h +4
-1@@ -583,11 +583,14 @@ enum {REQ_F_BUFFERS_COMMIT =IO_REQ_FLAG (REQ_F_BUFFERS_COMMIT_BIT ),/* buf node is valid */ REQ_F_BUF_NODE =IO_REQ_FLAG (REQ_F_BUF_NODE_BIT ),/* request has read/write metadata assigned */ REQ_F_HAS_METADATA =IO_REQ_FLAG (REQ_F_HAS_METADATA_BIT ), -/* resolve padded iovec to registered buffers */ +/* +* For vectored fixed buffers, resolve iovec to registered buffers. +* For SEND_ZC, whether to import buffers (i.e. the first issue). +*/ REQ_F_IMPORT_BUFFER = IO_REQ_FLAG(REQ_F_IMPORT_BUFFER_BIT), };typedef void (*io_req_tw_func_t )(struct io_kiocb *req,io_tw_token_t tw);io_uring/net.c +0
-1@@ -75,11 +75,10 @@ struct io_sr_msg {unsigned nr_multishot_loops;u16 flags;/* initialised and used only by !msg send variants */ u16 buf_group;bool retry; -bool imported;/* only for io_send_zc */ void __user *msg_control;/* used only for send zerocopy */ struct io_kiocb *notif; };@@ -1305,12 +1304,11 @@ int io_send_zc_prep(struct io_kiocb *req, const struct io_uring_sqe *sqe)struct io_ring_ctx *ctx = req->ctx;struct io_kiocb *notif;
Quoted 9 lines
io_uring/net.c int io_send_zc_prep(struct io_kiocb *req, const struct io_uring_sqe *sqe) · line 1309
Quoted 5 lines
io_uring/net.c +1
-2zc->done_io =0 ; zc->retry = false; - zc->imported = false; - req->flags |=REQ_F_POLL_NO_LAZY ; + req->flags |=REQ_F_POLL_NO_LAZY |REQ_F_IMPORT_BUFFER ;This function is shared with
sendmsg_zc, so if we set it here, it'll triggerio_import_reg_vec()inio_sendmsg_zc()even for non register buffer request.
Good catch. I keep forgetting which prep and issue functions are shared between which opcodes.
Thanks,
Caleb
Quoted 13 lines · Caleb Sander Mateos
Quoted 12 lines · Pavel Begunkov
Quoted 10 lines · Caleb Sander Mateos
Instead of a bool field in struct
io_sr_msg, useREQ_F_IMPORT_BUFFERto track whetherio_send_zc()has already imported the buffer. This flag already serves a similar purpose forsendmsg_zcand {read,write}v_fixed.Signed-off-by Caleb Sander Mateos <[email protected]>Suggested-by Pavel Begunkov <[email protected]>--- include/linux/io_uring_types.h | 5 ++++- io_uring/net.c | 8 +++----- 2 files changed, 7 insertions(+), 6 deletions(-)
...
Quoted 14 lines
Quoted 4 lines
Quoted 4 lines
io_uring/net.c +0
-0@@ -1305,12 +1304,11 @@ int io_send_zc_prep(struct io_kiocb *req, const struct io_uring_sqe *sqe)struct io_ring_ctx *ctx = req->ctx;struct io_kiocb *notif;Quoted 9 lines
io_uring/net.c int io_send_zc_prep(struct io_kiocb *req, const struct io_uring_sqe *sqe) · line 1309
Quoted 5 lines
io_uring/net.c +1
-2zc->done_io =0 ; zc->retry = false; - zc->imported = false; - req->flags |=REQ_F_POLL_NO_LAZY ; + req->flags |=REQ_F_POLL_NO_LAZY |REQ_F_IMPORT_BUFFER ;This function is shared with
sendmsg_zc, so if we set it here, it'll triggerio_import_reg_vec()inio_sendmsg_zc()even for non register buffer request.
Quoted 2 lines
Good catch. I keep forgetting which prep and issue functions are shared between which opcodes.
No worries, can happen. I'd recommend to run liburing tests, they're very useful for catching such things. I run it out of curiosity, it crashes on send-zerocopy.c pretty fast.
Signature · 1 line
Pavel Begunkov