cmd infra for caching iovec/bvec
v2 cover cmd infra for caching iovec/bvec
Add infrastructure that is going to be used by commands for importing vectored registered buffers. It can also be reused later for iovec caching.
v2: clear the vec on first ->async_data allocation
fix a memory leak
Pavel Begunkov (2):
io_uring/cmd: add iovec cache for commands io_uring/cmd: introduce io_uring_cmd_import_fixed_vec
include/linux/io_uring/cmd.h | 13 ++++++++++++ io_uring/io_uring.c | 5 +++-- io_uring/opdef.c | 1 + io_uring/uring_cmd.c | 39 +++++++++++++++++++++++++++++++++++- io_uring/uring_cmd.h | 11 ++++++++++ 5 files changed, 66 insertions(+), 3 deletions(-)
Signature · 1 line
2.48.1
PATCH v2 1/2 io_uring/cmd: add iovec cache for commands
Add iou_vec to commands and wire caching for it, but don't expose it to users just yet. We need the vec cleared on initial alloc, but since we can't place it at the beginning at the moment, zero the entire async_data. It's cached, and the performance effects only the initial allocation, and it might be not a bad idea since we're exposing those bits to outside drivers.
--- io_uring/io_uring.c | 5 +++-- io_uring/opdef.c | 1 + io_uring/uring_cmd.c | 20 +++++++++++++++++++- io_uring/uring_cmd.h | 5 +++++ 4 files changed, 28 insertions(+), 3 deletions(-)
io_uring/io_uring.c +3 -2
@@ -289,7 +289,7 @@ static void io_free_alloc_caches(struct io_ring_ctx *ctx)io_alloc_cache_free (&ctx->apoll_cache, kfree);io_alloc_cache_free (&ctx->netmsg_cache, io_netmsg_cache_free);io_alloc_cache_free (&ctx->rw_cache, io_rw_cache_free); -io_alloc_cache_free (&ctx->cmd_cache, kfree); +io_alloc_cache_free (&ctx->cmd_cache, io_cmd_cache_free);io_alloc_cache_free (&ctx->msg_cache, kfree);io_futex_cache_free (ctx);io_rsrc_cache_free (ctx);
@@ -335,7 +335,8 @@ static __cold struct io_ring_ctx *io_ring_ctx_alloc(struct io_uring_params *p)sizeof (struct io_async_rw ), offsetof(struct io_async_rw, clear)); ret |=io_alloc_cache_init (&ctx->cmd_cache,IO_ALLOC_CACHE_MAX , -sizeof (struct io_async_cmd ),0 ); +sizeof (struct io_async_cmd ), +sizeof (struct io_async_cmd ));spin_lock_init (&ctx->msg_lock); ret |=io_alloc_cache_init (&ctx->msg_cache,IO_ALLOC_CACHE_MAX ,sizeof (struct io_kiocb ),0 );
io_uring/opdef.c +1 -0
@@ -755,6 +755,7 @@ const struct io_cold_def io_cold_defs[] = { }, [IORING_OP_URING_CMD] = { .name = "URING_CMD", + .cleanup = io_uring_cmd_cleanup, }, [IORING_OP_SEND_ZC] = { .name = "SEND_ZC",
io_uring/uring_cmd.c +19 -1
@@ -16,6 +16,14 @@#include "rsrc.h" #include "uring_cmd.h" +void io_cmd_cache_free (const void *entry) +{ +struct io_async_cmd *ac = (struct io_async_cmd *)entry; + +io_vec_free (&ac->vec); +kfree (ac); +} +static void io_req_uring_cleanup (struct io_kiocb *req,unsigned int issue_flags) { struct io_uring_cmd *ioucmd = io_kiocb_to_cmd(req, struct io_uring_cmd);
@@ -29,13 +37,23 @@ static void io_req_uring_cleanup(struct io_kiocb *req, unsigned int issue_flags)if (issue_flags &IO_URING_F_UNLOCKED )return ; + +io_alloc_cache_vec_kasan (&ac->vec); +if (ac->vec.nr >IO_VEC_CACHE_SOFT_CAP ) +io_vec_free (&ac->vec); +if (io_alloc_cache_put (&req->ctx->cmd_cache, cache)) { ioucmd->sqe =NULL ; req->async_data =NULL ; - req->flags &= ~REQ_F_ASYNC_DATA ; + req->flags &= ~(REQ_F_ASYNC_DATA |REQ_F_NEED_CLEANUP ); } } +void io_uring_cmd_cleanup (struct io_kiocb *req) +{ +io_req_uring_cleanup (req,0 ); +} +bool io_uring_try_cancel_uring_cmd (struct io_ring_ctx *ctx,struct io_uring_task *tctx,bool cancel_all) {
io_uring/uring_cmd.h +5 -0
@@ -1,13 +1,18 @@// SPDX-License-Identifier: GPL-2.0 #include <linux/io_uring/cmd.h> +#include <linux/io_uring_types.h> struct io_async_cmd {struct io_uring_cmd_data data; +struct iou_vec vec; };int io_uring_cmd (struct io_kiocb *req,unsigned int issue_flags);int io_uring_cmd_prep (struct io_kiocb *req,const struct io_uring_sqe *sqe); +void io_uring_cmd_cleanup (struct io_kiocb *req);bool io_uring_try_cancel_uring_cmd (struct io_ring_ctx *ctx,struct io_uring_task *tctx,bool cancel_all); + +void io_cmd_cache_free (const void *entry);
Signature · 1 line
2.48.1
PATCH v2 2/2 io_uring/cmd: introduce io_uring_cmd_import_fixed_vec
In reply to Pavel Begunkovio_uring_cmd_import_fixed_vec() is a cmd helper around vectored registered buffer import functions, which caches the memory under the hood. The lifetime of the vectore and hence the iterator is bound to the request. Furthermore, the user is not allowed to call it multiple times for a single request.
--- include/linux/io_uring/cmd.h | 13 +++++++++++++ io_uring/uring_cmd.c | 19 +++++++++++++++++++ io_uring/uring_cmd.h | 6 ++++++ 3 files changed, 38 insertions(+)
include/linux/io_uring/cmd.h +13 -0
@@ -43,6 +43,11 @@ int io_uring_cmd_import_fixed(u64 ubuf, unsigned long len, int rw,struct iov_iter *iter, struct io_uring_cmd *ioucmd, unsigned int issue_flags); +int io_uring_cmd_import_fixed_vec (struct io_uring_cmd *ioucmd, +const struct iovec __user *uvec, +size_t uvec_segs, +int ddir,struct iov_iter *iter, +unsigned issue_flags);/* * Completes the request, i.e. posts an io_uring CQE and deallocates @ioucmd
@@ -76,6 +81,14 @@ io_uring_cmd_import_fixed(u64 ubuf, unsigned long len, int rw, {return -EOPNOTSUPP ; } +static inline int io_uring_cmd_import_fixed_vec (struct io_uring_cmd *ioucmd, +const struct iovec __user *uvec, +size_t uvec_segs, +int ddir,struct iov_iter *iter, +unsigned issue_flags) +{ +return -EOPNOTSUPP ; +}static inline void io_uring_cmd_done (struct io_uring_cmd *cmd,ssize_t ret,u64 ret2,unsigned issue_flags) {
io_uring/uring_cmd.c +19 -0
@@ -277,6 +277,25 @@ int io_uring_cmd_import_fixed(u64 ubuf, unsigned long len, int rw, }EXPORT_SYMBOL_GPL (io_uring_cmd_import_fixed); +int io_uring_cmd_import_fixed_vec (struct io_uring_cmd *ioucmd, +const struct iovec __user *uvec, +size_t uvec_segs, +int ddir,struct iov_iter *iter, +unsigned issue_flags) +{ +struct io_kiocb *req =cmd_to_io_kiocb (ioucmd); +struct io_async_cmd *ac = req->async_data; +int ret; + + ret =io_prep_reg_iovec (req, &ac->vec, uvec, uvec_segs); +if (ret) +return ret; + +return io_import_reg_vec (ddir, iter, req, &ac->vec, uvec_segs, + issue_flags); +} +EXPORT_SYMBOL_GPL (io_uring_cmd_import_fixed_vec); +void io_uring_cmd_issue_blocking (struct io_uring_cmd *ioucmd) {struct io_kiocb *req =cmd_to_io_kiocb (ioucmd);
io_uring/uring_cmd.h +6 -0
@@ -16,3 +16,9 @@ bool io_uring_try_cancel_uring_cmd(struct io_ring_ctx *ctx, struct io_uring_task *tctx, bool cancel_all);void io_cmd_cache_free (const void *entry); + +int io_uring_cmd_import_fixed_vec (struct io_uring_cmd *ioucmd, +const struct iovec __user *uvec, +size_t uvec_segs, +int ddir,struct iov_iter *iter, +unsigned issue_flags);
Signature · 1 line
2.48.1
Jens Axboe
maintainerQuoted 11 lines · Pavel Begunkov
Add infrastructure that is going to be used by commands for importing vectored registered buffers. It can also be reused later for iovec caching.
v2: clear the vec on first ->async_data allocation
fix a memory leakPavel Begunkov (2):
io_uring/cmd: add iovec cache for commandsio_uring/cmd: introduceio_uring_cmd_import_fixed_vec
Quoted 6 lines
include/linux/io_uring/cmd.h | 13 ++++++++++++ io_uring/io_uring.c | 5 +++-- io_uring/opdef.c | 1 + io_uring/uring_cmd.c | 39 +++++++++++++++++++++++++++++++++++- io_uring/uring_cmd.h | 11 ++++++++++ 5 files changed, 66 insertions(+), 3 deletions(-)
This version works for me - adding in Ming, so he can test and verify as well.
Signature · 1 line
Jens Axboe
Quoted 8 lines · Pavel Begunkov
Add infrastructure that is going to be used by commands for importing vectored registered buffers. It can also be reused later for iovec caching.
v2: clear the vec on first ->async_data allocation
fix a memory leak[...]
Applied, thanks!
[1/2] io_uring/cmd: add iovec cache for commands
commit: 3a4689ac109f18f23ea0d0c1c79e055142796858
[2/2] io_uring/cmd: introduce io_uring_cmd_import_fixed_vec
commit: ef490275297267d9461733ecd9b02bd3b798b3a4Best regards,
Signature · 1 line
Jens Axboe
Quoted 19 lines · Jens Axboe
Quoted 11 lines · Pavel Begunkov
Add infrastructure that is going to be used by commands for importing vectored registered buffers. It can also be reused later for iovec caching.
v2: clear the vec on first ->async_data allocation
fix a memory leakPavel Begunkov (2):
io_uring/cmd: add iovec cache for commandsio_uring/cmd: introduceio_uring_cmd_import_fixed_vecQuoted 6 lines
include/linux/io_uring/cmd.h | 13 ++++++++++++ io_uring/io_uring.c | 5 +++-- io_uring/opdef.c | 1 + io_uring/uring_cmd.c | 39 +++++++++++++++++++++++++++++++++++- io_uring/uring_cmd.h | 11 ++++++++++ 5 files changed, 66 insertions(+), 3 deletions(-)
Quoted 2 lines
This version works for me - adding in Ming, so he can test and verify as well.
With the two patches, all ublk selftest can run to pass, and kernel doesn't panic any more.
BTW, I meant vectored fix kernel buffer support for FS read/write, which looks not supported yet.
And it can be useful in the following case:
https://lore.kernel.org/linux-block/[email protected]/T/#u
Thanks,
Ming
Quoted 20 lines · Ming Lei
Quoted 19 lines · Jens Axboe
Quoted 17 lines · Pavel Begunkov
Add infrastructure that is going to be used by commands for importing vectored registered buffers. It can also be reused later for iovec caching.
v2: clear the vec on first ->async_data allocation
fix a memory leakPavel Begunkov (2):
io_uring/cmd: add iovec cache for commandsio_uring/cmd: introduceio_uring_cmd_import_fixed_vecinclude/linux/io_uring/cmd.h | 13 ++++++++++++ io_uring/io_uring.c | 5 +++-- io_uring/opdef.c | 1 + io_uring/uring_cmd.c | 39 +++++++++++++++++++++++++++++++++++- io_uring/uring_cmd.h | 11 ++++++++++ 5 files changed, 66 insertions(+), 3 deletions(-)
Quoted 5 lines
Quoted 2 lines
This version works for me - adding in Ming, so he can test and verify as well.
With the two patches, all ublk selftest can run to pass, and kernel doesn't panic any more.
thanks for testing.
Quoted 3 lines
BTW, I meant vectored fix kernel buffer support for FS read/write, which looks not supported yet.
Right, you can't use do a vectored registered buffer read/write/etc. with a kernel buffer (ala ublk zc provided one). That will likely need a special version of io_vec_fill_bvec() that would account for bvec entries not being uniform in size.
Signature · 1 line
Pavel Begunkov