sendzc dedup / simplification
cover sendzc dedup / simplification
Consolidate two pairs of duplicated callbacks for zerocopy send, which was a premature optimisation, and it's cleaner and simpler to keep them unified.
Pavel Begunkov (3):
io_uring/net: merge ubuf sendzc callbacks io_uring/net: get rid of io_notif_complete_tw_ext io_uring/net: set MSG_ZEROCOPY for sendzc in advance
io_uring/net.c | 16 ++++++++-------- io_uring/notif.c | 32 +++++++------------------------- io_uring/notif.h | 6 ++++-- 3 files changed, 19 insertions(+), 35 deletions(-)
Signature · 1 line
2.44.0
PATCH 1/3 io_uring/net: merge ubuf sendzc callbacks
Splitting io_tx_ubuf_callback_ext from io_tx_ubuf_callback is a pre mature optimisation that doesn't give us much. Merge the functions into one and reclaim some simplicity back.
--- io_uring/notif.c | 26 ++++++++------------------ 1 file changed, 8 insertions(+), 18 deletions(-)
io_uring/notif.c +8 -18
@@ -30,36 +30,24 @@ static void io_tx_ubuf_callback(struct sk_buff *skb, struct ubuf_info *uarg, struct io_notif_data *nd = container_of(uarg, struct io_notif_data, uarg);struct io_kiocb *notif =cmd_to_io_kiocb (nd); -if (refcount_dec_and_test (&uarg->refcnt)) -__io_req_task_work_add (notif,IOU_F_TWQ_LAZY_WAKE ); -} - -static void io_tx_ubuf_callback_ext (struct sk_buff *skb,struct ubuf_info *uarg, -bool success) -{ - struct io_notif_data *nd = container_of(uarg, struct io_notif_data, uarg); -if (nd->zc_report) {if (success && !nd->zc_used && skb)WRITE_ONCE (nd->zc_used, true);else if (!success && !nd->zc_copied)WRITE_ONCE (nd->zc_copied, true); } -io_tx_ubuf_callback (skb, uarg, success); + +if (refcount_dec_and_test (&uarg->refcnt)) +__io_req_task_work_add (notif,IOU_F_TWQ_LAZY_WAKE ); }void io_notif_set_extended (struct io_kiocb *notif) {struct io_notif_data *nd =io_notif_to_data (notif); -if (nd->uarg.callback != io_tx_ubuf_callback_ext) { - nd->account_pages =0 ; - nd->zc_report = false; - nd->zc_used = false; - nd->zc_copied = false; - nd->uarg.callback = io_tx_ubuf_callback_ext; - notif->io_task_work.func = io_notif_complete_tw_ext; - } + nd->zc_used = false; + nd->zc_copied = false; + notif->io_task_work.func = io_notif_complete_tw_ext; }struct io_kiocb *io_alloc_notif (struct io_ring_ctx *ctx)
@@ -79,6 +67,8 @@ struct io_kiocb *io_alloc_notif(struct io_ring_ctx *ctx) notif->io_task_work.func = io_req_task_complete; nd =io_notif_to_data (notif); + nd->zc_report = false; + nd->account_pages =0 ; nd->uarg.flags =IO_NOTIF_UBUF_FLAGS ; nd->uarg.callback = io_tx_ubuf_callback;refcount_set (&nd->uarg.refcnt,1 );
Signature · 1 line
2.44.0
PATCH 2/3 io_uring/net: get rid of io_notif_complete_tw_ext
In reply to Pavel Begunkovio_notif_complete_tw_ext() can be removed and combined with io_notif_complete_tw to make it simpler without sacrificing anything.
--- io_uring/net.c | 10 +++++----- io_uring/notif.c | 18 +++++------------- io_uring/notif.h | 6 ++++-- 3 files changed, 14 insertions(+), 20 deletions(-)
io_uring/net.c +5 -5
@@ -1019,8 +1019,11 @@ int io_send_zc_prep(struct io_kiocb *req, const struct io_uring_sqe *sqe)if (zc->flags & ~IO_ZC_FLAGS_VALID )return -EINVAL ;if (zc->flags &IORING_SEND_ZC_REPORT_USAGE ) { -io_notif_set_extended (notif); -io_notif_to_data (notif)->zc_report = true; +struct io_notif_data *nd =io_notif_to_data (notif); + + nd->zc_report = true; + nd->zc_used = false; + nd->zc_copied = false; } }
@@ -1129,7 +1132,6 @@ static int io_send_zc_import(struct io_kiocb *req, struct io_async_msghdr *kmsg)return ret; kmsg->msg.sg_from_iter = io_sg_from_iter; }else { -io_notif_set_extended (sr->notif); ret =import_ubuf (ITER_SOURCE , sr->buf, sr->len, &kmsg->msg.msg_iter);if (unlikely (ret))return ret;
@@ -1218,8 +1220,6 @@ int io_sendmsg_zc(struct io_kiocb *req, unsigned int issue_flags)unsigned flags;int ret, min_ret =0 ; -io_notif_set_extended (sr->notif); - sock =sock_from_file (req->file);if (unlikely (!sock))return -ENOTSOCK ;
io_uring/notif.c +5 -13
@@ -9,12 +9,12 @@#include "notif.h" #include "rsrc.h" -static void io_notif_complete_tw_ext (struct io_kiocb *notif,struct io_tw_state *ts) +void io_notif_tw_complete (struct io_kiocb *notif,struct io_tw_state *ts) {struct io_notif_data *nd =io_notif_to_data (notif);struct io_ring_ctx *ctx = notif->ctx; -if (nd->zc_report && (nd->zc_copied || !nd->zc_used)) +if (unlikely (nd->zc_report) && (nd->zc_copied || !nd->zc_used)) notif->cqe.res |=IORING_NOTIF_USAGE_ZC_COPIED ;if (nd->account_pages && ctx->user) {
@@ -37,17 +37,10 @@ static void io_tx_ubuf_callback(struct sk_buff *skb, struct ubuf_info *uarg,WRITE_ONCE (nd->zc_copied, true); } -if (refcount_dec_and_test (&uarg->refcnt)) +if (refcount_dec_and_test (&uarg->refcnt)) { + notif->io_task_work.func = io_notif_tw_complete;__io_req_task_work_add (notif,IOU_F_TWQ_LAZY_WAKE ); -} - -void io_notif_set_extended (struct io_kiocb *notif) -{ -struct io_notif_data *nd =io_notif_to_data (notif); - - nd->zc_used = false; - nd->zc_copied = false; - notif->io_task_work.func = io_notif_complete_tw_ext; + } }struct io_kiocb *io_alloc_notif (struct io_ring_ctx *ctx)
@@ -64,7 +57,6 @@ struct io_kiocb *io_alloc_notif(struct io_ring_ctx *ctx) notif->task = current;io_get_task_refs (1 ); notif->rsrc_node =NULL ; - notif->io_task_work.func = io_req_task_complete; nd =io_notif_to_data (notif); nd->zc_report = false;
io_uring/notif.h +4 -2
@@ -20,7 +20,7 @@ struct io_notif_data { };struct io_kiocb *io_alloc_notif (struct io_ring_ctx *ctx); -void io_notif_set_extended (struct io_kiocb *notif); +void io_notif_tw_complete (struct io_kiocb *notif,struct io_tw_state *ts);static inline struct io_notif_data *io_notif_to_data (struct io_kiocb *notif) {
@@ -33,8 +33,10 @@ static inline void io_notif_flush(struct io_kiocb *notif)struct io_notif_data *nd =io_notif_to_data (notif);/* drop slot's master ref */ -if (refcount_dec_and_test (&nd->uarg.refcnt)) +if (refcount_dec_and_test (&nd->uarg.refcnt)) { + notif->io_task_work.func = io_notif_tw_complete;__io_req_task_work_add (notif,IOU_F_TWQ_LAZY_WAKE ); + } }static inline int io_notif_account_mem (struct io_kiocb *notif,unsigned len)
Signature · 1 line
2.44.0
PATCH 3/3 io_uring/net: set MSG_ZEROCOPY for sendzc in advance
In reply to Pavel BegunkovWe can set MSG_ZEROCOPY at the preparation step, do it so we don't have to care about it later in the issue callback.
--- io_uring/net.c | 6 +++--- 1 file changed, 3 insertions(+), 3 deletions(-)
io_uring/net.c +3 -3
@@ -1051,7 +1051,7 @@ int io_send_zc_prep(struct io_kiocb *req, const struct io_uring_sqe *sqe) zc->buf =u64_to_user_ptr (READ_ONCE (sqe->addr)); zc->len =READ_ONCE (sqe->len); - zc->msg_flags =READ_ONCE (sqe->msg_flags) |MSG_NOSIGNAL ; + zc->msg_flags =READ_ONCE (sqe->msg_flags) |MSG_NOSIGNAL |MSG_ZEROCOPY ;if (zc->msg_flags &MSG_DONTWAIT ) req->flags |=REQ_F_NOWAIT ;
@@ -1168,7 +1168,7 @@ int io_send_zc(struct io_kiocb *req, unsigned int issue_flags)return ret; } - msg_flags = zc->msg_flags |MSG_ZEROCOPY ; + msg_flags = zc->msg_flags;if (issue_flags &IO_URING_F_NONBLOCK ) msg_flags |=MSG_DONTWAIT ;if (msg_flags &MSG_WAITALL )
@@ -1230,7 +1230,7 @@ int io_sendmsg_zc(struct io_kiocb *req, unsigned int issue_flags) (sr->flags & IORING_RECVSEND_POLL_FIRST))return -EAGAIN ; - flags = sr->msg_flags |MSG_ZEROCOPY ; + flags = sr->msg_flags;if (issue_flags &IO_URING_F_NONBLOCK ) flags |=MSG_DONTWAIT ;if (flags &MSG_WAITALL )
Signature · 1 line
2.44.0
Quoted 10 lines · Pavel Begunkov
Consolidate two pairs of duplicated callbacks for zerocopy send, which was a premature optimisation, and it's cleaner and simpler to keep them unified.
Pavel Begunkov (3):
io_uring/net: merge ubuf sendzc callbacksio_uring/net: get rid ofio_notif_complete_tw_extio_uring/net: setMSG_ZEROCOPYfor sendzc in advance[...]
Applied, thanks!
[1/3] io_uring/net: merge ubuf sendzc callbacks
(no commit info)
[2/3] io_uring/net: get rid of io_notif_complete_tw_ext
(no commit info)
[3/3] io_uring/net: set MSG_ZEROCOPY for sendzc in advance
(no commit info)
Best regards,
Signature · 1 line
Jens Axboe