io_uring: extend bvec registration
Subsystems (4)
v6 cover io_uring: extend bvec registration
This series refactors and extends the io_uring registered buffers infrastructure to allow external subsystems to register pre-existing bvec arrays directly.
The motivation for the patches in this series is to make fuse zero-copy possible. These patches are split out from a previous larger fuse-over-io_uring series [1]. The fuse zero-copy work that builds on top of this is in [2].
Thanks,
Joanne
[1] https://lore.kernel.org/linux-fsdevel/[email protected]/
[2] https://lore.kernel.org/linux-fsdevel/[email protected]/
Changelog:
v5: https://lore.kernel.org/io-uring/[email protected]/
v5 -> v6:
* rebase to origin/for-next
v4: https://lore.kernel.org/io-uring/[email protected]/
v4 -> v5:
* rebase to origin/for-7.1/io_uring
* drop the io_uring_registered_mem_region_get() patch
v3: https://lore.kernel.org/io-uring/[email protected]/
v3 -> v4:
* Add comment about io_uring_registered_mem_region_get() locking (Jens)
* Return info in a new struct io_uring_mem_region_info (Jens)
v2: https://lore.kernel.org/io-uring/[email protected]/
v2 -> v3:
* drop patch that makes buffer release callback optional
* add patch for renaming/exporting IO_IMU_DEST / IO_IMU_SOURCE
v1: https://lore.kernel.org/io-uring/[email protected]/
v1 -> v2:
* update io_kernel_buffer_init() to take bitmasked dir directly so callers can set both dest and source
Joanne Koong (4):
io_uring/rsrc: rename
io_buffer_register_bvec()/io_buffer_unregister_bvec()
io_uring/rsrc: split io_buffer_register_request() logic io_uring/rsrc: add io_buffer_register_bvec() io_uring/rsrc: rename and export IO_IMU_DEST / IO_IMU_SOURCE
Documentation/block/ublk.rst | 14 ++-- drivers/block/ublk_drv.c | 22 +++--- include/linux/io_uring/cmd.h | 38 ++++++++-- include/linux/io_uring_types.h | 5 ++ io_uring/io_uring.c | 2 +- io_uring/rsrc.c | 127 +++++++++++++++++++++++---------- io_uring/rsrc.h | 5 -- 7 files changed, 146 insertions(+), 67 deletions(-)
Signature · 1 line
2.52.0
PATCH v6 1/4 io_uring/rsrc: rename io_buffer_register_bvec()/io_buffer_unregister_bvec()
Currently, io_buffer_register_bvec() takes in a request. In preparation for supporting kernel-populated buffers in fuse io-uring (which will need to register bvecs directly, not through a struct request), rename this to io_buffer_register_request().
A subsequent patch will commandeer the "io_buffer_register_bvec()" function name to support registering bvecs directly.
Rename io_buffer_unregister_bvec() to a more generic name, io_buffer_unregister(), as both io_buffer_register_request() and io_buffer_register_bvec() callers will use it for unregistration.
--- Documentation/block/ublk.rst | 14 +++++++------- drivers/block/ublk_drv.c | 22 +++++++++++----------- include/linux/io_uring/cmd.h | 25 +++++++++++++++++++------ io_uring/rsrc.c | 14 +++++++------- 4 files changed, 44 insertions(+), 31 deletions(-)
Documentation/block/ublk.rst +7 -7
@@ -382,17 +382,17 @@ Zero copy --------- ublk zero copy relies on io_uring's fixed kernel buffer, which provides -two APIs: `io_buffer_register_bvec()` and `io_buffer_unregister_bvec`. +two APIs: `io_buffer_register_request()` and `io_buffer_unregister`. ublk adds IO command of `UBLK_IO_REGISTER_IO_BUF` to call -`io_buffer_register_bvec()` for ublk server to register client request +`io_buffer_register_request()` for ublk server to register client request buffer into io_uring buffer table, then ublk server can submit io_uring IOs with the registered buffer index. IO command of `UBLK_IO_UNREGISTER_IO_BUF` -calls `io_buffer_unregister_bvec()` to unregister the buffer, which is -guaranteed to be live between calling `io_buffer_register_bvec()` and -`io_buffer_unregister_bvec()`. Any io_uring operation which supports this -kind of kernel buffer will grab one reference of the buffer until the -operation is completed. +calls `io_buffer_unregister()` to unregister the buffer, which is guaranteed +to be live between calling `io_buffer_register_request()` and +`io_buffer_unregister()`. Any io_uring operation which supports this kind of +kernel buffer will grab one reference of the buffer until the operation is +completed. ublk server implementing zero copy or user copy has to be CAP_SYS_ADMIN and be trusted, because it is ublk server's responsibility to make sure IO buffer
drivers/block/ublk_drv.c +11 -11
@@ -1627,8 +1627,8 @@ ublk_auto_buf_register(const struct ublk_queue *ubq, struct request *req, {int ret; - ret =io_buffer_register_bvec (cmd, req, ublk_io_release, - io->buf.auto_reg.index, issue_flags); + ret =io_buffer_register_request (cmd, req, ublk_io_release, + io->buf.auto_reg.index, issue_flags);if (ret) {if (io->buf.auto_reg.flags &UBLK_AUTO_BUF_REG_FALLBACK ) {ublk_auto_buf_reg_fallback (ubq, req->tag);
@@ -1828,7 +1828,7 @@ static noinline void ublk_batch_dispatch_fail(struct ublk_queue *ubq,ublk_io_unlock (io);if (index !=-1 ) -io_buffer_unregister_bvec (data->cmd, index, +io_buffer_unregister (data->cmd, index, data->issue_flags); }
@@ -3097,8 +3097,8 @@ static int ublk_register_io_buf(struct io_uring_cmd *cmd,if (!req)return -EINVAL ; - ret =io_buffer_register_bvec (cmd, req, ublk_io_release, index, - issue_flags); + ret =io_buffer_register_request (cmd, req, ublk_io_release, index, + issue_flags);if (ret) {ublk_put_req_ref (io, req);return ret;
@@ -3129,8 +3129,8 @@ ublk_daemon_register_io_buf(struct io_uring_cmd *cmd,if (!ublk_dev_support_zero_copy (ub) || !ublk_rq_has_data (req))return -EINVAL ; - ret =io_buffer_register_bvec (cmd, req, ublk_io_release, index, - issue_flags); + ret =io_buffer_register_request (cmd, req, ublk_io_release, index, + issue_flags);if (ret)return ret;
@@ -3145,7 +3145,7 @@ static int ublk_unregister_io_buf(struct io_uring_cmd *cmd,if (!(ub->dev_info.flags &UBLK_F_SUPPORT_ZERO_COPY ))return -EINVAL ; -return io_buffer_unregister_bvec (cmd, index, issue_flags); +return io_buffer_unregister (cmd, index, issue_flags); }static int ublk_check_fetch_buf (const struct ublk_device *ub,__u64 buf_addr)
@@ -3286,7 +3286,7 @@ static int ublk_ch_uring_cmd_local(struct io_uring_cmd *cmd, goto out;/* -* io_buffer_unregister_bvec() doesn't access the ubq or io, +* io_buffer_unregister() doesn't access the ubq or io, * so no need to validate the q_id, tag, or task */ if (_IOC_NR (cmd_op) ==UBLK_IO_UNREGISTER_IO_BUF )
@@ -3353,7 +3353,7 @@ static int ublk_ch_uring_cmd_local(struct io_uring_cmd *cmd, req =ublk_fill_io_cmd (io, cmd); ret =ublk_config_io_buf (ub, io, cmd, addr, &buf_idx);if (buf_idx !=UBLK_INVALID_BUF_IDX ) -io_buffer_unregister_bvec (cmd, buf_idx, issue_flags); +io_buffer_unregister (cmd, buf_idx, issue_flags); compl =ublk_need_complete_req (ub, io);if (req_op (req) ==REQ_OP_ZONE_APPEND )
@@ -3688,7 +3688,7 @@ static int ublk_batch_commit_io(struct ublk_queue *ubq, }if (buf_idx !=UBLK_INVALID_BUF_IDX ) -io_buffer_unregister_bvec (data->cmd, buf_idx, data->issue_flags); +io_buffer_unregister (data->cmd, buf_idx, data->issue_flags);if (req_op (req) ==REQ_OP_ZONE_APPEND ) req->__sector =ublk_batch_zone_lba (uc, elem);if (compl)
include/linux/io_uring/cmd.h +19 -6
@@ -91,6 +91,11 @@ struct io_br_sel io_uring_cmd_buffer_select(struct io_uring_cmd *ioucmd,bool io_uring_mshot_cmd_post_cqe (struct io_uring_cmd *ioucmd,struct io_br_sel *sel,unsigned int issue_flags); +int io_buffer_register_request (struct io_uring_cmd *cmd,struct request *rq, +void (*release)(void *),unsigned int index, +unsigned int issue_flags); +int io_buffer_unregister (struct io_uring_cmd *cmd,unsigned int index, +unsigned int issue_flags);#else static inline int io_uring_cmd_import_fixed(u64 ubuf, unsigned long len, int rw,
@@ -133,6 +138,20 @@ static inline bool io_uring_mshot_cmd_post_cqe(struct io_uring_cmd *ioucmd, {return true; } +static inline int io_buffer_register_request (struct io_uring_cmd *cmd, +struct request *rq, +void (*release)(void *), +unsigned int index, +unsigned int issue_flags) +{ +return -EOPNOTSUPP ; +} +static inline int io_buffer_unregister (struct io_uring_cmd *cmd, +unsigned int index, +unsigned int issue_flags) +{ +return -EOPNOTSUPP ; +}#endif static inline struct io_uring_cmd *io_uring_cmd_from_tw (struct io_tw_req tw_req)
@@ -182,10 +201,4 @@ static inline void io_uring_cmd_done32(struct io_uring_cmd *ioucmd, s32 ret, return __io_uring_cmd_done(ioucmd, ret, res2, issue_flags, true); } -int io_buffer_register_bvec (struct io_uring_cmd *cmd,struct request *rq, -void (*release)(void *),unsigned int index, -unsigned int issue_flags); -int io_buffer_unregister_bvec (struct io_uring_cmd *cmd,unsigned int index, -unsigned int issue_flags); -#endif /* _LINUX_IO_URING_CMD_H */
io_uring/rsrc.c +7 -7
@@ -924,9 +924,9 @@ int io_sqe_buffers_register(struct io_ring_ctx *ctx, void __user *arg,return ret; } -int io_buffer_register_bvec (struct io_uring_cmd *cmd,struct request *rq, -void (*release)(void *),unsigned int index, -unsigned int issue_flags) +int io_buffer_register_request (struct io_uring_cmd *cmd,struct request *rq, +void (*release)(void *),unsigned int index, +unsigned int issue_flags) {struct io_ring_ctx *ctx =cmd_to_io_kiocb (cmd)->ctx;struct io_rsrc_data *data = &ctx->buf_table;
@@ -986,10 +986,10 @@ int io_buffer_register_bvec(struct io_uring_cmd *cmd, struct request *rq,io_ring_submit_unlock (ctx, issue_flags);return ret; } -EXPORT_SYMBOL_GPL (io_buffer_register_bvec); +EXPORT_SYMBOL_GPL (io_buffer_register_request); -int io_buffer_unregister_bvec (struct io_uring_cmd *cmd,unsigned int index, -unsigned int issue_flags) +int io_buffer_unregister (struct io_uring_cmd *cmd,unsigned int index, +unsigned int issue_flags) {struct io_ring_ctx *ctx =cmd_to_io_kiocb (cmd)->ctx;struct io_rsrc_data *data = &ctx->buf_table;
@@ -1019,7 +1019,7 @@ int io_buffer_unregister_bvec(struct io_uring_cmd *cmd, unsigned int index,io_ring_submit_unlock (ctx, issue_flags);return ret; } -EXPORT_SYMBOL_GPL (io_buffer_unregister_bvec); +EXPORT_SYMBOL_GPL (io_buffer_unregister);static int validate_fixed_range (u64 buf_addr,size_t len,const struct io_mapped_ubuf *imu)
Signature · 1 line
2.52.0
PATCH v6 2/4 io_uring/rsrc: split io_buffer_register_request() logic
In reply to Joanne KoongSplit the main initialization logic in io_buffer_register_request() into a helper function.
This is a preparatory patch for supporting kernel-populated buffers in fuse io-uring, which will be reusing this logic.
--- io_uring/rsrc.c | 84 ++++++++++++++++++++++++++++++------------------- 1 file changed, 51 insertions(+), 33 deletions(-)
io_uring/rsrc.c +51 -33
@@ -924,64 +924,82 @@ int io_sqe_buffers_register(struct io_ring_ctx *ctx, void __user *arg,return ret; } -int io_buffer_register_request (struct io_uring_cmd *cmd,struct request *rq, -void (*release)(void *),unsigned int index, -unsigned int issue_flags) +static struct io_mapped_ubuf *io_kernel_buffer_init (struct io_ring_ctx *ctx, +unsigned int nr_bvecs, +unsigned int total_bytes, +u8 dir, +void (*release)(void *), +void *priv, +unsigned int index) { -struct io_ring_ctx *ctx =cmd_to_io_kiocb (cmd)->ctx;struct io_rsrc_data *data = &ctx->buf_table; -struct req_iterator rq_iter;struct io_mapped_ubuf *imu;struct io_rsrc_node *node; -struct bio_vec bv; -unsigned int nr_bvecs =0 ; -int ret =0 ; -io_ring_submit_lock (ctx, issue_flags); -if (index >= data->nr) { - ret = -EINVAL ; - goto unlock; - } +if (index >= data->nr) +return ERR_PTR (-EINVAL ); index =array_index_nospec (index, data->nr); -if (data->nodes[index]) { - ret = -EBUSY ; - goto unlock; - } +if (data->nodes[index]) +return ERR_PTR (-EBUSY ); node =io_rsrc_node_alloc (ctx,IORING_RSRC_BUFFER ); -if (!node) { - ret = -ENOMEM ; - goto unlock; - } +if (!node) +return ERR_PTR (-ENOMEM ); -/* -* blk_rq_nr_phys_segments() may overestimate the number of bvecs -* but avoids needing to iterate over the bvecs -*/ - imu =io_alloc_imu (ctx,blk_rq_nr_phys_segments (rq)); + imu =io_alloc_imu (ctx, nr_bvecs);if (!imu) {io_cache_free (&ctx->node_cache, node); - ret = -ENOMEM ; - goto unlock; +return ERR_PTR (-ENOMEM ); } imu->ubuf =0 ; - imu->len =blk_rq_bytes (rq); + imu->len = total_bytes; imu->acct_pages =0 ; imu->folio_shift =PAGE_SHIFT ; + imu->nr_bvecs = nr_bvecs;refcount_set (&imu->refs,1 ); imu->release = release; - imu->priv = rq; + imu->priv = priv; + imu->dir = dir; imu->flags =IO_REGBUF_F_KBUF ; - imu->dir =1 <<rq_data_dir (rq); + node->buf = imu; + data->nodes[index] = node; + +return imu; +} + +int io_buffer_register_request (struct io_uring_cmd *cmd,struct request *rq, +void (*release)(void *),unsigned int index, +unsigned int issue_flags) +{ +struct io_ring_ctx *ctx =cmd_to_io_kiocb (cmd)->ctx; +struct req_iterator rq_iter; +struct io_mapped_ubuf *imu; +struct bio_vec bv; +/* +* blk_rq_nr_phys_segments() may overestimate the number of bvecs +* but avoids needing to iterate over the bvecs +*/ +unsigned int nr_bvecs =blk_rq_nr_phys_segments (rq); +unsigned int total_bytes =blk_rq_bytes (rq); +int ret =0 ; + +io_ring_submit_lock (ctx, issue_flags); + + imu =io_kernel_buffer_init (ctx, nr_bvecs, total_bytes, +1 <<rq_data_dir (rq), release, rq, index); +if (IS_ERR (imu)) { + ret =PTR_ERR (imu); + goto unlock; + } + + nr_bvecs =0 ;rq_for_each_bvec (bv, rq, rq_iter) imu->bvec[nr_bvecs++] = bv; imu->nr_bvecs = nr_bvecs; - node->buf = imu; - data->nodes[index] = node; unlock:io_ring_submit_unlock (ctx, issue_flags);return ret;
Signature · 1 line
2.52.0
PATCH v6 3/4 io_uring/rsrc: add io_buffer_register_bvec()
In reply to Joanne KoongAdd io_buffer_register_bvec() for registering a bvec array.
This is a preparatory patch for fuse-over-io-uring zero-copy.
--- include/linux/io_uring/cmd.h | 13 +++++++++++++ io_uring/rsrc.c | 35 +++++++++++++++++++++++++++++++++++ 2 files changed, 48 insertions(+)
include/linux/io_uring/cmd.h +13 -0
@@ -94,6 +94,10 @@ bool io_uring_mshot_cmd_post_cqe(struct io_uring_cmd *ioucmd,int io_buffer_register_request (struct io_uring_cmd *cmd,struct request *rq,void (*release)(void *),unsigned int index,unsigned int issue_flags); +int io_buffer_register_bvec (struct io_uring_cmd *cmd,const struct bio_vec *bvs, +unsigned int nr_bvecs,void (*release)(void *), +void *priv,u8 dir,unsigned int index, +unsigned int issue_flags);int io_buffer_unregister (struct io_uring_cmd *cmd,unsigned int index,unsigned int issue_flags);#else
@@ -146,6 +150,15 @@ static inline int io_buffer_register_request(struct io_uring_cmd *cmd, {return -EOPNOTSUPP ; } +static inline int io_buffer_register_bvec (struct io_uring_cmd *cmd, +const struct bio_vec *bvs, +unsigned int nr_bvecs, +void (*release)(void *),void *priv, +u8 dir,unsigned int index, +unsigned int issue_flags) +{ +return -EOPNOTSUPP ; +}static inline int io_buffer_unregister (struct io_uring_cmd *cmd,unsigned int index,unsigned int issue_flags)
io_uring/rsrc.c +35 -0
@@ -1006,6 +1006,41 @@ int io_buffer_register_request(struct io_uring_cmd *cmd, struct request *rq, }EXPORT_SYMBOL_GPL (io_buffer_register_request); +/* +* bvs is copied internally. caller may free it on return. +*/ +int io_buffer_register_bvec (struct io_uring_cmd *cmd,const struct bio_vec *bvs, +unsigned int nr_bvecs,void (*release)(void *), +void *priv,u8 dir,unsigned int index, +unsigned int issue_flags) +{ +struct io_ring_ctx *ctx =cmd_to_io_kiocb (cmd)->ctx; +struct io_mapped_ubuf *imu; +struct bio_vec *bvec; +unsigned int i, total_bytes =0 ; +int ret =0 ; + +for (i =0 ; i < nr_bvecs; i++) + total_bytes += bvs[i].bv_len; + +io_ring_submit_lock (ctx, issue_flags); + imu =io_kernel_buffer_init (ctx, nr_bvecs, total_bytes, dir, release, + priv, index); +if (IS_ERR (imu)) { + ret =PTR_ERR (imu); + goto unlock; + } + + bvec = imu->bvec; +for (i =0 ; i < nr_bvecs; i++) + bvec[i] = bvs[i]; + +unlock: +io_ring_submit_unlock (ctx, issue_flags); +return ret; +} +EXPORT_SYMBOL_GPL (io_buffer_register_bvec); +int io_buffer_unregister (struct io_uring_cmd *cmd,unsigned int index,unsigned int issue_flags) {
Signature · 1 line
2.52.0
PATCH v6 4/4 io_uring/rsrc: rename and export IO_IMU_DEST / IO_IMU_SOURCE
In reply to Joanne KoongRename IO_IMU_DEST and IO_IMU_SOURCE to IO_BUF_DEST and IO_BUF_SOURCE and export it so subsystems may use it.
This is needed by the io_buffer_register_bvec() path for callers who may need the buffer to be both readable and writable.
--- include/linux/io_uring_types.h | 5 +++++ io_uring/io_uring.c | 2 +- io_uring/rsrc.c | 2 +- io_uring/rsrc.h | 5 ----- 4 files changed, 7 insertions(+), 7 deletions(-)
include/linux/io_uring_types.h +5 -0
@@ -44,6 +44,11 @@ enum io_uring_cmd_flags { IO_URING_F_COMPAT = (1 << 12), }; +enum { +IO_BUF_DEST =1 <<ITER_DEST , +IO_BUF_SOURCE =1 <<ITER_SOURCE , +}; +struct iou_loop_params ;struct io_wq_work_node {
io_uring/io_uring.c +1 -1
@@ -3218,7 +3218,7 @@ static int __init io_uring_init(void)io_uring_optable_init ();/* imu->dir is u8 */ -BUILD_BUG_ON ((IO_IMU_DEST |IO_IMU_SOURCE ) >U8_MAX ); +BUILD_BUG_ON ((IO_BUF_DEST |IO_BUF_SOURCE ) >U8_MAX );/* * Allow user copy in the per-command field, which starts after the
io_uring/rsrc.c +1 -1
@@ -820,7 +820,7 @@ static struct io_rsrc_node *io_sqe_buffer_register(struct io_ring_ctx *ctx, imu->release = io_release_ubuf; imu->priv = imu; imu->flags =0 ; - imu->dir =IO_IMU_DEST |IO_IMU_SOURCE ; + imu->dir =IO_BUF_DEST |IO_BUF_SOURCE ;if (coalesced) imu->folio_shift = data.folio_shift;refcount_set (&imu->refs,1 );
io_uring/rsrc.h +0 -5
@@ -23,11 +23,6 @@ struct io_rsrc_node { }; }; -enum { -IO_IMU_DEST =1 <<ITER_DEST , -IO_IMU_SOURCE =1 <<ITER_SOURCE , -}; -enum {IO_REGBUF_F_KBUF =1 , };
Signature · 1 line
2.52.0
Quoted 10 lines · Joanne Koong
This series refactors and extends the
io_uringregistered buffers infrastructure to allow external subsystems to register pre-existing bvec arrays directly.The motivation for the patches in this series is to make fuse zero-copy possible. These patches are split out from a previous larger fuse-over-io_uring series [1]. The fuse zero-copy work that builds on top of this is in [2].
[...]
Applied, thanks!
[1/4] io_uring/rsrc: rename io_buffer_register_bvec()/io_buffer_unregister_bvec()
commit: 29ebfdd7db89514b9a6dea9feba6e2f9b90079bc
[2/4] io_uring/rsrc: split io_buffer_register_request() logic
commit: 367978400e3b792b702e0710d9faaaf0acf2cecf
[3/4] io_uring/rsrc: add io_buffer_register_bvec()
commit: 33ee911b8e6230fb5c939ed19d159c227673321f
[4/4] io_uring/rsrc: rename and export IO_IMU_DEST / IO_IMU_SOURCE
commit: b09efaddf67196b74160e54a3fc45efa3fa100eeBest regards,
Signature · 1 line
Jens Axboe