io_uring/net: introduce IORING_SEND_ZC_REPORT_USAGE flag
PATCH 1/1 io_uring/net: introduce IORING_SEND_ZC_REPORT_USAGE flag
In reply to a message not in the archiveIt might be useful for applications to detect if a zero copy transfer with SEND[MSG]_ZC was actually possible or not. The application can fallback to plain SEND[MSG] in order to avoid the overhead of two cqes per request.
Or it can generate a log message that could indicate to an administrator that no zero copy was possible and could explain degraded performance.
--- include/uapi/linux/io_uring.h | 18 ++++++++++++++++++ io_uring/net.c | 6 +++++- io_uring/notif.c | 12 ++++++++++++ io_uring/notif.h | 3 +++ 4 files changed, 38 insertions(+), 1 deletion(-)
include/uapi/linux/io_uring.h +18 -0
@@ -296,10 +296,28 @@ enum io_uring_op {* * IORING_RECVSEND_FIXED_BUF Use registered buffers, the index is stored in * the buf_index field. +* +* IORING_SEND_ZC_REPORT_USAGE +* If set, SEND[MSG]_ZC should report +* the zerocopy usage in cqe.res +* for the IORING_CQE_F_NOTIF cqe. +* 0 is reported if zerocopy was actually possible. +* IORING_NOTIF_USAGE_ZC_COPIED if data was copied +* (at least partially). */ #define IORING_RECVSEND_POLL_FIRST (1U << 0)#define IORING_RECV_MULTISHOT (1U << 1)#define IORING_RECVSEND_FIXED_BUF (1U << 2) +#define IORING_SEND_ZC_REPORT_USAGE (1U << 3) + +/* +* cqe.res for IORING_CQE_F_NOTIF if +* IORING_SEND_ZC_REPORT_USAGE was requested +* +* It should be treated as a flag, all other +* bits of cqe.res should be treated as reserved! +*/ +#define IORING_NOTIF_USAGE_ZC_COPIED (1U << 31)/* * accept flags stored in sqe->ioprio
io_uring/net.c +5 -1
@@ -939,7 +939,8 @@ int io_send_zc_prep(struct io_kiocb *req, const struct io_uring_sqe *sqe) zc->flags =READ_ONCE (sqe->ioprio);if (zc->flags & ~(IORING_RECVSEND_POLL_FIRST | -IORING_RECVSEND_FIXED_BUF )) +IORING_RECVSEND_FIXED_BUF | +IORING_SEND_ZC_REPORT_USAGE ))return -EINVAL ; notif = zc->notif =io_alloc_notif (ctx);if (!notif)
@@ -957,6 +958,9 @@ int io_send_zc_prep(struct io_kiocb *req, const struct io_uring_sqe *sqe) req->imu =READ_ONCE (ctx->user_bufs[idx]);io_req_set_rsrc_node (notif, ctx,0 ); } +if (zc->flags &IORING_SEND_ZC_REPORT_USAGE ) { +io_notif_to_data (notif)->zc_report = true; + }if (req->opcode ==IORING_OP_SEND_ZC ) {if (READ_ONCE (sqe->__pad3[0 ]))
io_uring/notif.c +12 -0
@@ -18,6 +18,10 @@ static void __io_notif_complete_tw(struct io_kiocb *notif, bool *locked)__io_unaccount_mem (ctx->user, nd->account_pages); nd->account_pages =0 ; } + +if (nd->zc_report && (nd->zc_copied || !nd->zc_used)) + notif->cqe.res |=IORING_NOTIF_USAGE_ZC_COPIED ; +io_req_task_complete (notif, locked); }
@@ -28,6 +32,13 @@ static void io_uring_tx_zerocopy_callback(struct sk_buff *skb, struct io_notif_data *nd = container_of(uarg, struct io_notif_data, uarg);struct io_kiocb *notif =cmd_to_io_kiocb (nd); +if (nd->zc_report) { +if (success && !nd->zc_used && skb) +WRITE_ONCE (nd->zc_used, true); +else if (!success && !nd->zc_copied) +WRITE_ONCE (nd->zc_copied, true); + } +if (refcount_dec_and_test (&uarg->refcnt)) { notif->io_task_work.func = __io_notif_complete_tw;io_req_task_work_add (notif);
@@ -55,6 +66,7 @@ struct io_kiocb *io_alloc_notif(struct io_ring_ctx *ctx) nd->account_pages =0 ; nd->uarg.flags =SKBFL_ZEROCOPY_FRAG |SKBFL_DONT_ORPHAN ; nd->uarg.callback = io_uring_tx_zerocopy_callback; + nd->zc_report = nd->zc_used = nd->zc_copied = false;refcount_set (&nd->uarg.refcnt,1 );return notif; }
io_uring/notif.h +3 -0
@@ -13,6 +13,9 @@ struct io_notif_data {struct file *file;struct ubuf_info uarg;unsigned long account_pages; +bool zc_report; +bool zc_used; +bool zc_copied; };void io_notif_flush (struct io_kiocb *notif);
Signature · 1 line
2.34.1
Quoted 7 lines · Stefan Metzmacher
It might be useful for applications to detect if a zero copy transfer with SEND[MSG]_ZC was actually possible or not. The application can fallback to plain SEND[MSG] in order to avoid the overhead of two cqes per request.
Or it can generate a log message that could indicate to an administrator that no zero copy was possible and could explain degraded performance.
From a quick look seems good, I'll test and double check when I'm back on tuesday
Quoted 118 lines
Link https://lore.kernel.org/io-uring/[email protected]/T/#m2b0d9df94ce43b0e69e6c089bdff0ce6babbdfaa open hereCc Pavel Begunkov <[email protected]>Signed-off-by Stefan Metzmacher <[email protected]>--- include/uapi/linux/io_uring.h | 18 ++++++++++++++++++ io_uring/net.c | 6 +++++- io_uring/notif.c | 12 ++++++++++++ io_uring/notif.h | 3 +++ 4 files changed, 38 insertions(+), 1 deletion(-)include/uapi/linux/io_uring.h +18
-0@@ -296,10 +296,28 @@ enum io_uring_op {* * IORING_RECVSEND_FIXED_BUF Use registered buffers, the index is stored in * the buf_index field. +* +* IORING_SEND_ZC_REPORT_USAGE +* If set, SEND[MSG]_ZC should report +* the zerocopy usage in cqe.res +* for the IORING_CQE_F_NOTIF cqe. +* 0 is reported if zerocopy was actually possible. +* IORING_NOTIF_USAGE_ZC_COPIED if data was copied +* (at least partially). */ #define IORING_RECVSEND_POLL_FIRST (1U << 0)#define IORING_RECV_MULTISHOT (1U << 1)#define IORING_RECVSEND_FIXED_BUF (1U << 2) +#define IORING_SEND_ZC_REPORT_USAGE (1U << 3) + +/* +* cqe.res for IORING_CQE_F_NOTIF if +* IORING_SEND_ZC_REPORT_USAGE was requested +* +* It should be treated as a flag, all other +* bits of cqe.res should be treated as reserved! +*/ +#define IORING_NOTIF_USAGE_ZC_COPIED (1U << 31)/* * accept flags stored in sqe->ioprio io_uring/net.c +5
-1@@ -939,7 +939,8 @@ int io_send_zc_prep(struct io_kiocb *req, const struct io_uring_sqe *sqe) zc->flags =READ_ONCE (sqe->ioprio);if (zc->flags & ~(IORING_RECVSEND_POLL_FIRST | -IORING_RECVSEND_FIXED_BUF )) +IORING_RECVSEND_FIXED_BUF | +IORING_SEND_ZC_REPORT_USAGE ))return -EINVAL ; notif = zc->notif =io_alloc_notif (ctx);if (!notif)@@ -957,6 +958,9 @@ int io_send_zc_prep(struct io_kiocb *req, const struct io_uring_sqe *sqe) req->imu =READ_ONCE (ctx->user_bufs[idx]);io_req_set_rsrc_node (notif, ctx,0 ); } +if (zc->flags &IORING_SEND_ZC_REPORT_USAGE ) { +io_notif_to_data (notif)->zc_report = true; + }if (req->opcode ==IORING_OP_SEND_ZC ) {if (READ_ONCE (sqe->__pad3[0 ]))io_uring/notif.c +12
-0@@ -18,6 +18,10 @@ static void __io_notif_complete_tw(struct io_kiocb *notif, bool *locked)__io_unaccount_mem (ctx->user, nd->account_pages); nd->account_pages =0 ; } + +if (nd->zc_report && (nd->zc_copied || !nd->zc_used)) + notif->cqe.res |=IORING_NOTIF_USAGE_ZC_COPIED ; +io_req_task_complete (notif, locked); }@@ -28,6 +32,13 @@ static void io_uring_tx_zerocopy_callback(struct sk_buff *skb, struct io_notif_data *nd = container_of(uarg, struct io_notif_data, uarg);struct io_kiocb *notif =cmd_to_io_kiocb (nd); +if (nd->zc_report) { +if (success && !nd->zc_used && skb) +WRITE_ONCE (nd->zc_used, true); +else if (!success && !nd->zc_copied) +WRITE_ONCE (nd->zc_copied, true); + } +if (refcount_dec_and_test (&uarg->refcnt)) { notif->io_task_work.func = __io_notif_complete_tw;io_req_task_work_add (notif);@@ -55,6 +66,7 @@ struct io_kiocb *io_alloc_notif(struct io_ring_ctx *ctx) nd->account_pages =0 ; nd->uarg.flags =SKBFL_ZEROCOPY_FRAG |SKBFL_DONT_ORPHAN ; nd->uarg.callback = io_uring_tx_zerocopy_callback; + nd->zc_report = nd->zc_used = nd->zc_copied = false;refcount_set (&nd->uarg.refcnt,1 );return notif; }io_uring/notif.h +3
-0@@ -13,6 +13,9 @@ struct io_notif_data {struct file *file;struct ubuf_info uarg;unsigned long account_pages; +bool zc_report; +bool zc_used; +bool zc_copied; };void io_notif_flush (struct io_kiocb *notif);
Signature · 1 line
Pavel Begunkov
Quoted 7 lines · Stefan Metzmacher
It might be useful for applications to detect if a zero copy transfer with SEND[MSG]_ZC was actually possible or not. The application can fallback to plain SEND[MSG] in order to avoid the overhead of two cqes per request.
Or it can generate a log message that could indicate to an administrator that no zero copy was possible and could explain degraded performance.
Looks good,
Quoted 118 lines
Link https://lore.kernel.org/io-uring/[email protected]/T/#m2b0d9df94ce43b0e69e6c089bdff0ce6babbdfaa open hereCc Pavel Begunkov <[email protected]>Signed-off-by Stefan Metzmacher <[email protected]>--- include/uapi/linux/io_uring.h | 18 ++++++++++++++++++ io_uring/net.c | 6 +++++- io_uring/notif.c | 12 ++++++++++++ io_uring/notif.h | 3 +++ 4 files changed, 38 insertions(+), 1 deletion(-)include/uapi/linux/io_uring.h +18
-0@@ -296,10 +296,28 @@ enum io_uring_op {* * IORING_RECVSEND_FIXED_BUF Use registered buffers, the index is stored in * the buf_index field. +* +* IORING_SEND_ZC_REPORT_USAGE +* If set, SEND[MSG]_ZC should report +* the zerocopy usage in cqe.res +* for the IORING_CQE_F_NOTIF cqe. +* 0 is reported if zerocopy was actually possible. +* IORING_NOTIF_USAGE_ZC_COPIED if data was copied +* (at least partially). */ #define IORING_RECVSEND_POLL_FIRST (1U << 0)#define IORING_RECV_MULTISHOT (1U << 1)#define IORING_RECVSEND_FIXED_BUF (1U << 2) +#define IORING_SEND_ZC_REPORT_USAGE (1U << 3) + +/* +* cqe.res for IORING_CQE_F_NOTIF if +* IORING_SEND_ZC_REPORT_USAGE was requested +* +* It should be treated as a flag, all other +* bits of cqe.res should be treated as reserved! +*/ +#define IORING_NOTIF_USAGE_ZC_COPIED (1U << 31)/* * accept flags stored in sqe->ioprio io_uring/net.c +5
-1@@ -939,7 +939,8 @@ int io_send_zc_prep(struct io_kiocb *req, const struct io_uring_sqe *sqe) zc->flags =READ_ONCE (sqe->ioprio);if (zc->flags & ~(IORING_RECVSEND_POLL_FIRST | -IORING_RECVSEND_FIXED_BUF )) +IORING_RECVSEND_FIXED_BUF | +IORING_SEND_ZC_REPORT_USAGE ))return -EINVAL ; notif = zc->notif =io_alloc_notif (ctx);if (!notif)@@ -957,6 +958,9 @@ int io_send_zc_prep(struct io_kiocb *req, const struct io_uring_sqe *sqe) req->imu =READ_ONCE (ctx->user_bufs[idx]);io_req_set_rsrc_node (notif, ctx,0 ); } +if (zc->flags &IORING_SEND_ZC_REPORT_USAGE ) { +io_notif_to_data (notif)->zc_report = true; + }if (req->opcode ==IORING_OP_SEND_ZC ) {if (READ_ONCE (sqe->__pad3[0 ]))io_uring/notif.c +12
-0@@ -18,6 +18,10 @@ static void __io_notif_complete_tw(struct io_kiocb *notif, bool *locked)__io_unaccount_mem (ctx->user, nd->account_pages); nd->account_pages =0 ; } + +if (nd->zc_report && (nd->zc_copied || !nd->zc_used)) + notif->cqe.res |=IORING_NOTIF_USAGE_ZC_COPIED ; +io_req_task_complete (notif, locked); }@@ -28,6 +32,13 @@ static void io_uring_tx_zerocopy_callback(struct sk_buff *skb, struct io_notif_data *nd = container_of(uarg, struct io_notif_data, uarg);struct io_kiocb *notif =cmd_to_io_kiocb (nd); +if (nd->zc_report) { +if (success && !nd->zc_used && skb) +WRITE_ONCE (nd->zc_used, true); +else if (!success && !nd->zc_copied) +WRITE_ONCE (nd->zc_copied, true); + } +if (refcount_dec_and_test (&uarg->refcnt)) { notif->io_task_work.func = __io_notif_complete_tw;io_req_task_work_add (notif);@@ -55,6 +66,7 @@ struct io_kiocb *io_alloc_notif(struct io_ring_ctx *ctx) nd->account_pages =0 ; nd->uarg.flags =SKBFL_ZEROCOPY_FRAG |SKBFL_DONT_ORPHAN ; nd->uarg.callback = io_uring_tx_zerocopy_callback; + nd->zc_report = nd->zc_used = nd->zc_copied = false;refcount_set (&nd->uarg.refcnt,1 );return notif; }io_uring/notif.h +3
-0@@ -13,6 +13,9 @@ struct io_notif_data {struct file *file;struct ubuf_info uarg;unsigned long account_pages; +bool zc_report; +bool zc_used; +bool zc_copied; };void io_notif_flush (struct io_kiocb *notif);
Signature · 1 line
Pavel Begunkov
Quoted 9 lines · Stefan Metzmacher
It might be useful for applications to detect if a zero copy transfer with SEND[MSG]_ZC was actually possible or not. The application can fallback to plain SEND[MSG] in order to avoid the overhead of two cqes per request.
Or it can generate a log message that could indicate to an administrator that no zero copy was possible and could explain degraded performance.[...]
Applied, thanks!
[1/1] io_uring/net: introduce IORING_SEND_ZC_REPORT_USAGE flag
commit: 4847a0eae62976ac27f192cd59b9de72b390eff3
Best regards,
Signature · 1 line
Jens Axboe