Add support for vectored registered buffers
cover Add support for vectored registered buffers
Add registered buffer support for vectored io_uring operations. That allows to pass an iovec, all entries of which must belong to and point into the same registered buffer specified by sqe->buf_index.
The series covers zerocopy sendmsg and reads / writes. Reads and writes are implemented as new opcodes, while zerocopy sendmsg reuses IORING_RECVSEND_FIXED_BUF for the api.
Results are aligned to what one would expect from registered buffers:
t/io_uring + nullblk, single segment 16K:
34 -> 46 GiB/sexamples/send-zerocopy.c default send size (64KB):
82558 -> 123855 MB/s
The series is placed on top of 6.15 + zcrx.
Some tests:
https://github.com/isilence/liburing.git regbuf-import
Pavel Begunkov (8):
io_uring: introduce struct iou_vec
io_uring: add infra for importing vectored reg buffers io_uring/rw: implement vectored registered rw io_uring/rw: defer reg buf vec import io_uring/net: combine msghdr copy io_uring/net: pull vec alloc out of msghdr import io_uring/net: convert to struct iou_vec io_uring/net: implement vectored reg bufs for zctx
include/linux/io_uring_types.h | 11 ++ include/uapi/linux/io_uring.h | 2 + io_uring/alloc_cache.h | 9 -- io_uring/net.c | 177 ++++++++++++++++++++------------- io_uring/net.h | 6 +- io_uring/opdef.c | 39 ++++++++ io_uring/rsrc.c | 133 +++++++++++++++++++++++++ io_uring/rsrc.h | 22 ++++ io_uring/rw.c | 96 ++++++++++++++++-- io_uring/rw.h | 8 +- 10 files changed, 411 insertions(+), 92 deletions(-)
Signature · 1 line
2.48.1
PATCH 1/8 io_uring: introduce struct iou_vec
I need a convenient way to pass around and work with iovec+size pair, put them into a structure and makes use of it in rw.c
--- include/linux/io_uring_types.h | 5 +++++ io_uring/rsrc.c | 9 +++++++++ io_uring/rsrc.h | 17 +++++++++++++++++ io_uring/rw.c | 17 +++++++---------- io_uring/rw.h | 6 ++++-- 5 files changed, 42 insertions(+), 12 deletions(-)
include/linux/io_uring_types.h +5 -0
@@ -110,6 +110,11 @@ struct io_uring_task { }____cacheline_aligned_in_smp ; }; +struct iou_vec { +struct iovec *iovec; +unsigned nr; +}; +struct io_uring {u32 head;u32 tail;
io_uring/rsrc.c +9 -0
@@ -1264,3 +1264,12 @@ int io_register_clone_buffers(struct io_ring_ctx *ctx, void __user *arg)fput (file);return ret; } + +void io_vec_free (struct iou_vec *iv) +{ +if (!iv->iovec) +return ; +kfree (iv->iovec); + iv->iovec =NULL ; + iv->nr =0 ; +}
io_uring/rsrc.h +17 -0
@@ -3,6 +3,7 @@#define IOU_RSRC_H #include <linux/lockdep.h> +#include <linux/io_uring_types.h> enum {IORING_RSRC_FILE =0 ,
@@ -144,4 +145,20 @@ static inline void __io_unaccount_mem(struct user_struct *user, atomic_long_sub(nr_pages, &user->locked_vm); } +void io_vec_free (struct iou_vec *iv); + +static inline void io_vec_reset_iovec (struct iou_vec *iv, +struct iovec *iovec,unsigned nr) +{ +io_vec_free (iv); + iv->iovec = iovec; + iv->nr = nr; +} + +static inline void io_alloc_cache_vec_kasan (struct iou_vec *iv) +{ +if (IS_ENABLED (CONFIG_KASAN )) +io_vec_free (iv); +} +#endif
io_uring/rw.c +7 -10
@@ -87,9 +87,9 @@ static int io_import_vec(int ddir, struct io_kiocb *req,int ret, nr_segs;struct iovec *iov; -if (io->free_iovec) { - nr_segs = io->free_iov_nr; - iov = io->free_iovec; +if (io->vec.iovec) { + nr_segs = io->vec.nr; + iov = io->vec.iovec; }else { nr_segs =1 ; iov = &io->fast_iov;
@@ -101,9 +101,7 @@ static int io_import_vec(int ddir, struct io_kiocb *req,return ret;if (iov) { req->flags |=REQ_F_NEED_CLEANUP ; - io->free_iov_nr = io->iter.nr_segs; -kfree (io->free_iovec); - io->free_iovec = iov; +io_vec_reset_iovec (&io->vec, iov, io->iter.nr_segs); }return 0 ; }
@@ -151,7 +149,7 @@ static void io_rw_recycle(struct io_kiocb *req, unsigned int issue_flags)if (unlikely (issue_flags &IO_URING_F_UNLOCKED ))return ; -io_alloc_cache_kasan (&rw->free_iovec, &rw->free_iov_nr); +io_alloc_cache_vec_kasan (&rw->vec);if (io_alloc_cache_put (&req->ctx->rw_cache, rw)) { req->async_data =NULL ; req->flags &= ~REQ_F_ASYNC_DATA ;
@@ -201,7 +199,7 @@ static int io_rw_alloc_async(struct io_kiocb *req) rw =io_uring_alloc_async_data (&ctx->rw_cache, req);if (!rw)return -ENOMEM ; -if (rw->free_iovec) +if (rw->vec.iovec) req->flags |=REQ_F_NEED_CLEANUP ; rw->bytes_done =0 ;return 0 ;
@@ -1327,7 +1325,6 @@ void io_rw_cache_free(const void *entry) {struct io_async_rw *rw = (struct io_async_rw *) entry; -if (rw->free_iovec) -kfree (rw->free_iovec); +io_vec_free (&rw->vec);kfree (rw); }
io_uring/rw.h +4 -2
@@ -3,19 +3,21 @@#include <linux/io_uring_types.h> #include <linux/pagemap.h> +#include "rsrc.h" +struct io_meta_state {u32 seed;struct iov_iter_state iter_meta; };struct io_async_rw { +struct iou_vec vec;size_t bytes_done; -struct iovec *free_iovec; + struct_group(clear, struct iov_iter iter;struct iov_iter_state iter_state;struct iovec fast_iov; -int free_iov_nr;/* * wpq is for buffered io, while meta fields are used with * direct io
Signature · 1 line
2.48.1
PATCH 2/8 io_uring: add infra for importing vectored reg buffers
In reply to Pavel BegunkovAdd io_import_reg_vec(), which will be responsible for importing vectored registered buffers. iovecs are overlapped with the resulting bvec in memory, which is why the iovec is expected to be padded in iou_vec.
--- include/linux/io_uring_types.h | 5 +- io_uring/rsrc.c | 124 +++++++++++++++++++++++++++++++++ io_uring/rsrc.h | 5 ++ 3 files changed, 133 insertions(+), 1 deletion(-)
include/linux/io_uring_types.h +4 -1
@@ -111,7 +111,10 @@ struct io_uring_task { };struct iou_vec { -struct iovec *iovec; +union { +struct iovec *iovec; +struct bio_vec *bvec; + };unsigned nr; };
io_uring/rsrc.c +124 -0
133 lines not on this page · raw message
io_uring/rsrc.h +5 -0
@@ -61,6 +61,10 @@ struct io_rsrc_node *io_find_buf_node(struct io_kiocb *req,int io_import_reg_buf (struct io_kiocb *req,struct iov_iter *iter,u64 buf_addr,size_t len,int ddir,unsigned issue_flags); +int io_import_reg_vec (int ddir,struct iov_iter *iter, +struct io_kiocb *req,struct iou_vec *vec, +int nr_iovs,unsigned iovec_off, +unsigned issue_flags);int io_register_clone_buffers (struct io_ring_ctx *ctx,void __user *arg);int io_sqe_buffers_unregister (struct io_ring_ctx *ctx);
@@ -146,6 +150,7 @@ static inline void __io_unaccount_mem(struct user_struct *user, }void io_vec_free (struct iou_vec *iv); +int io_vec_realloc (struct iou_vec *iv,unsigned nr_entries);static inline void io_vec_reset_iovec (struct iou_vec *iv,struct iovec *iovec,unsigned nr)
Signature · 1 line
2.48.1
PATCH 3/8 io_uring/rw: implement vectored registered rw
In reply to Pavel BegunkovImplement registered buffer vectored reads with new opcodes IORING_OP_WRITEV_FIXED and IORING_OP_READV_FIXED.
--- include/uapi/linux/io_uring.h | 2 ++ io_uring/opdef.c | 39 +++++++++++++++++++++++++++ io_uring/rw.c | 51 +++++++++++++++++++++++++++++++++++ io_uring/rw.h | 2 ++ 4 files changed, 94 insertions(+)
include/uapi/linux/io_uring.h +2 -0
@@ -280,6 +280,8 @@ enum io_uring_op {IORING_OP_BIND ,IORING_OP_LISTEN ,IORING_OP_RECV_ZC , +IORING_OP_READV_FIXED , +IORING_OP_WRITEV_FIXED ,/* this goes last, obviously */ IORING_OP_LAST,
io_uring/opdef.c +39 -0
@@ -529,6 +529,35 @@ const struct io_issue_def io_issue_defs[] = { .prep = io_eopnotsupp_prep, #endif }, + [IORING_OP_READV_FIXED ] = { + .needs_file =1 , + .unbound_nonreg_file =1 , + .pollin =1 , + .plug =1 , + .audit_skip =1 , + .ioprio =1 , + .iopoll =1 , + .iopoll_queue =1 , + .vectored =1 , + .async_size =sizeof (struct io_async_rw ), + .prep = io_prep_readv_fixed, + .issue = io_read, + }, + [IORING_OP_WRITEV_FIXED ] = { + .needs_file =1 , + .hash_reg_file =1 , + .unbound_nonreg_file =1 , + .pollout =1 , + .plug =1 , + .audit_skip =1 , + .ioprio =1 , + .iopoll =1 , + .iopoll_queue =1 , + .vectored =1 , + .async_size =sizeof (struct io_async_rw ), + .prep = io_prep_writev_fixed, + .issue = io_write, + }, };const struct io_cold_def io_cold_defs[] = {
@@ -761,6 +790,16 @@ const struct io_cold_def io_cold_defs[] = { [IORING_OP_RECV_ZC ] = { .name ="RECV_ZC" , }, + [IORING_OP_READV_FIXED ] = { + .name ="READV_FIXED" , + .cleanup = io_readv_writev_cleanup, + .fail = io_rw_fail, + }, + [IORING_OP_WRITEV_FIXED ] = { + .name ="WRITEV_FIXED" , + .cleanup = io_readv_writev_cleanup, + .fail = io_rw_fail, + }, };const char *io_uring_get_opcode (u8 opcode)
io_uring/rw.c +51 -0
57 lines not on this page · raw message
io_uring/rw.h +2 -0
@@ -34,6 +34,8 @@ struct io_async_rw {int io_prep_read_fixed (struct io_kiocb *req,const struct io_uring_sqe *sqe);int io_prep_write_fixed (struct io_kiocb *req,const struct io_uring_sqe *sqe); +int io_prep_readv_fixed (struct io_kiocb *req,const struct io_uring_sqe *sqe); +int io_prep_writev_fixed (struct io_kiocb *req,const struct io_uring_sqe *sqe);int io_prep_readv (struct io_kiocb *req,const struct io_uring_sqe *sqe);int io_prep_writev (struct io_kiocb *req,const struct io_uring_sqe *sqe);int io_prep_read (struct io_kiocb *req,const struct io_uring_sqe *sqe);
Signature · 1 line
2.48.1
PATCH 4/8 io_uring/rw: defer reg buf vec import
In reply to Pavel BegunkovImport registered buffers for vectored reads and writes later at issue time as we now do for other fixed ops.
--- include/linux/io_uring_types.h | 3 +++ io_uring/rw.c | 36 +++++++++++++++++++++++++++++----- 2 files changed, 34 insertions(+), 5 deletions(-)
include/linux/io_uring_types.h +3 -0
@@ -502,6 +502,7 @@ enum {REQ_F_BUFFERS_COMMIT_BIT ,REQ_F_BUF_NODE_BIT ,REQ_F_HAS_METADATA_BIT , +REQ_F_IMPORT_BUFFER_BIT ,/* not a real bit, just to check we're not overflowing the space */ __REQ_F_LAST_BIT,
@@ -584,6 +585,8 @@ enum {REQ_F_BUF_NODE =IO_REQ_FLAG (REQ_F_BUF_NODE_BIT ),/* request has read/write metadata assigned */ REQ_F_HAS_METADATA =IO_REQ_FLAG (REQ_F_HAS_METADATA_BIT ), +/* resolve padded iovec to registered buffers */ + REQ_F_IMPORT_BUFFER = IO_REQ_FLAG(REQ_F_IMPORT_BUFFER_BIT), };typedef void (*io_req_tw_func_t )(struct io_kiocb *req,io_tw_token_t tw);
io_uring/rw.c +31 -5
@@ -381,6 +381,24 @@ int io_prep_write_fixed(struct io_kiocb *req, const struct io_uring_sqe *sqe)return __io_prep_rw (req, sqe,ITER_SOURCE ); } +static int io_rw_import_reg_vec (struct io_kiocb *req, +struct io_async_rw *io, +int ddir,unsigned int issue_flags) +{ + struct io_rw *rw = io_kiocb_to_cmd(req, struct io_rw); +unsigned uvec_segs = rw->len; +unsigned iovec_off = io->vec.nr - uvec_segs; +int ret; + + ret =io_import_reg_vec (ddir, &io->iter, req, &io->vec, + uvec_segs, iovec_off, issue_flags); +if (unlikely (ret)) +return ret; +iov_iter_save_state (&io->iter, &io->iter_state); + req->flags &= ~REQ_F_IMPORT_BUFFER ; +return 0 ; +} +static int io_rw_prep_reg_vec (struct io_kiocb *req,int ddir) { struct io_rw *rw = io_kiocb_to_cmd(req, struct io_rw);
@@ -406,10 +424,8 @@ static int io_rw_prep_reg_vec(struct io_kiocb *req, int ddir)if (IS_ERR (res))return PTR_ERR (res); - ret =io_import_reg_vec (ddir, &io->iter, req, &io->vec, - uvec_segs, iovec_off,0 ); -iov_iter_save_state (&io->iter, &io->iter_state); -return ret; + req->flags |=REQ_F_IMPORT_BUFFER ; +return 0 ; }int io_prep_readv_fixed (struct io_kiocb *req,const struct io_uring_sqe *sqe)
@@ -906,7 +922,11 @@ static int __io_read(struct io_kiocb *req, unsigned int issue_flags)ssize_t ret;loff_t *ppos; -if (io_do_buffer_select (req)) { +if (req->flags &REQ_F_IMPORT_BUFFER ) { + ret =io_rw_import_reg_vec (req, io,ITER_DEST , issue_flags); +if (unlikely (ret)) +return ret; + }else if (io_do_buffer_select (req)) { ret =io_import_rw_buffer (ITER_DEST , req, io, issue_flags);if (unlikely (ret <0 ))return ret;
@@ -1117,6 +1137,12 @@ int io_write(struct io_kiocb *req, unsigned int issue_flags)ssize_t ret, ret2;loff_t *ppos; +if (req->flags &REQ_F_IMPORT_BUFFER ) { + ret =io_rw_import_reg_vec (req, io,ITER_SOURCE , issue_flags); +if (unlikely (ret)) +return ret; + } + ret =io_rw_init_file (req,FMODE_WRITE ,WRITE );if (unlikely (ret))return ret;
Signature · 1 line
2.48.1
PATCH 5/8 io_uring/net: combine msghdr copy
In reply to Pavel BegunkovCall the compat version from inside of io_msg_copy_hdr() and don't duplicate it in callers.
--- io_uring/net.c | 46 +++++++++++++++++++--------------------------- 1 file changed, 19 insertions(+), 27 deletions(-)
io_uring/net.c +19 -27
@@ -287,6 +287,24 @@ static int io_msg_copy_hdr(struct io_kiocb *req, struct io_async_msghdr *iomsg,struct user_msghdr __user *umsg = sr->umsg;int ret; + iomsg->msg.msg_name = &iomsg->addr; + iomsg->msg.msg_iter.nr_segs =0 ; + +if (io_is_compat (req->ctx)) { +struct compat_msghdr cmsg; + + ret =io_compat_msg_copy_hdr (req, iomsg, &cmsg, ddir, save_addr); +if (ret) +return ret; + +memset (&msg,0 ,sizeof (msg)); + msg->msg_namelen = cmsg.msg_namelen; + msg->msg_controllen = cmsg.msg_controllen; + msg->msg_iov =compat_ptr (cmsg.msg_iov); + msg->msg_iovlen = cmsg.msg_iovlen; +return 0 ; + } + ret =io_copy_msghdr_from_user (msg, umsg);if (unlikely (ret))return ret;
@@ -323,18 +341,6 @@ static int io_sendmsg_copy_hdr(struct io_kiocb *req,struct user_msghdr msg;int ret; - iomsg->msg.msg_name = &iomsg->addr; - iomsg->msg.msg_iter.nr_segs =0 ; - -if (io_is_compat (req->ctx)) { -struct compat_msghdr cmsg; - - ret =io_compat_msg_copy_hdr (req, iomsg, &cmsg,ITER_SOURCE , -NULL ); - sr->msg_control = iomsg->msg.msg_control_user; -return ret; - } - ret =io_msg_copy_hdr (req, iomsg, &msg,ITER_SOURCE ,NULL );/* save msg_control as sys_sendmsg() overwrites it */ sr->msg_control = iomsg->msg.msg_control_user;
@@ -710,21 +716,7 @@ static int io_recvmsg_copy_hdr(struct io_kiocb *req,struct user_msghdr msg;int ret; - iomsg->msg.msg_name = &iomsg->addr; - iomsg->msg.msg_iter.nr_segs =0 ; - -if (io_is_compat (req->ctx)) { -struct compat_msghdr cmsg; - - ret =io_compat_msg_copy_hdr (req, iomsg, &cmsg,ITER_DEST , - &iomsg->uaddr); -memset (&msg,0 ,sizeof (msg)); - msg.msg_namelen = cmsg.msg_namelen; - msg.msg_controllen = cmsg.msg_controllen; - }else { - ret =io_msg_copy_hdr (req, iomsg, &msg,ITER_DEST , &iomsg->uaddr); - } - + ret =io_msg_copy_hdr (req, iomsg, &msg,ITER_DEST , &iomsg->uaddr);if (unlikely (ret))return ret; return io_recvmsg_mshot_prep(req, iomsg, msg.msg_namelen,
Signature · 1 line
2.48.1
PATCH 6/8 io_uring/net: pull vec alloc out of msghdr import
In reply to Pavel BegunkovI'll need more control over iovec management, move io_net_import_vec() out of io_msg_copy_hdr().
--- io_uring/net.c | 23 +++++++++++++++-------- 1 file changed, 15 insertions(+), 8 deletions(-)
io_uring/net.c +15 -8
@@ -253,12 +253,8 @@ static int io_compat_msg_copy_hdr(struct io_kiocb *req,return -EFAULT ; sr->len = tmp_iov.iov_len; } - -return 0 ; } - -return io_net_import_vec (req, iomsg, (struct iovec __user *)uiov, - msg->msg_iovlen, ddir); +return 0 ; } static int io_copy_msghdr_from_user(struct user_msghdr *msg,
@@ -328,10 +324,8 @@ static int io_msg_copy_hdr(struct io_kiocb *req, struct io_async_msghdr *iomsg,return -EFAULT ; sr->len = tmp_iov.iov_len; } -return 0 ; } - -return io_net_import_vec (req, iomsg, msg->msg_iov, msg->msg_iovlen, ddir); +return 0 ; } static int io_sendmsg_copy_hdr(struct io_kiocb *req,
@@ -342,6 +336,12 @@ static int io_sendmsg_copy_hdr(struct io_kiocb *req,int ret; ret =io_msg_copy_hdr (req, iomsg, &msg,ITER_SOURCE ,NULL ); +if (unlikely (ret)) +return ret; + +if (!(req->flags &REQ_F_BUFFER_SELECT )) + ret =io_net_import_vec (req, iomsg, msg.msg_iov, msg.msg_iovlen, +ITER_SOURCE );/* save msg_control as sys_sendmsg() overwrites it */ sr->msg_control = iomsg->msg.msg_control_user;return ret;
@@ -719,6 +719,13 @@ static int io_recvmsg_copy_hdr(struct io_kiocb *req, ret =io_msg_copy_hdr (req, iomsg, &msg,ITER_DEST , &iomsg->uaddr);if (unlikely (ret))return ret; + +if (!(req->flags &REQ_F_BUFFER_SELECT )) { + ret =io_net_import_vec (req, iomsg, msg.msg_iov, msg.msg_iovlen, +ITER_DEST ); +if (unlikely (ret)) +return ret; + }return io_recvmsg_mshot_prep (req, iomsg, msg.msg_namelen, msg.msg_controllen); }
Signature · 1 line
2.48.1
PATCH 7/8 io_uring/net: convert to struct iou_vec
In reply to Pavel BegunkovConvert net.c to use struct iou_vec.
--- io_uring/alloc_cache.h | 9 -------- io_uring/net.c | 51 ++++++++++++++++++------------------------ io_uring/net.h | 6 ++--- 3 files changed, 25 insertions(+), 41 deletions(-)
io_uring/alloc_cache.h +0 -9
@@ -16,15 +16,6 @@ bool io_alloc_cache_init(struct io_alloc_cache *cache,void *io_cache_alloc_new (struct io_alloc_cache *cache,gfp_t gfp); -static inline void io_alloc_cache_kasan (struct iovec **iov,int *nr) -{ -if (IS_ENABLED (CONFIG_KASAN )) { -kfree (*iov); - *iov =NULL ; - *nr =0 ; - } -} -static inline bool io_alloc_cache_put (struct io_alloc_cache *cache,void *entry) {
io_uring/net.c +22 -29
117 lines not on this page · raw message
io_uring/net.h +3 -3
@@ -2,12 +2,12 @@#include <linux/net.h> #include <linux/uio.h> +#include <linux/io_uring_types.h> struct io_async_msghdr {#if defined(CONFIG_NET ) -struct iovec *free_iov; -/* points to an allocated iov, if NULL we use fast_iov instead */ -int free_iov_nr; +struct iou_vec vec; + struct_group(clear, int namelen;struct iovec fast_iov;
Signature · 1 line
2.48.1
PATCH 8/8 io_uring/net: implement vectored reg bufs for zctx
In reply to Pavel BegunkovAdd support for vectored registered buffers for send zc.
--- io_uring/net.c | 59 ++++++++++++++++++++++++++++++++++++++++++++++---- 1 file changed, 55 insertions(+), 4 deletions(-)
io_uring/net.c +55 -4
@@ -395,6 +395,44 @@ static int io_sendmsg_setup(struct io_kiocb *req, const struct io_uring_sqe *sqereturn io_sendmsg_copy_hdr (req, kmsg); } +static int io_sendmsg_zc_setup (struct io_kiocb *req,const struct io_uring_sqe *sqe) +{ + struct io_sr_msg *sr = io_kiocb_to_cmd(req, struct io_sr_msg); +struct io_async_msghdr *kmsg = req->async_data; +struct user_msghdr msg; +int ret, iovec_off; +struct iovec *iov; +void *res; + +if (!(sr->flags &IORING_RECVSEND_FIXED_BUF )) +return io_sendmsg_setup (req, sqe); + + sr->umsg =u64_to_user_ptr (READ_ONCE (sqe->addr)); + + ret =io_msg_copy_hdr (req, kmsg, &msg,ITER_SOURCE ,NULL ); +if (unlikely (ret)) +return ret; + sr->msg_control = kmsg->msg.msg_control_user; + +if (msg.msg_iovlen > kmsg->vec.nr ||WARN_ON_ONCE (!kmsg->vec.iovec)) { + ret =io_vec_realloc (&kmsg->vec, msg.msg_iovlen); +if (ret) +return ret; + req->flags |=REQ_F_NEED_CLEANUP ; + } + iovec_off = kmsg->vec.nr - msg.msg_iovlen; + iov = kmsg->vec.iovec + iovec_off; + + res =iovec_from_user (msg.msg_iov, msg.msg_iovlen, kmsg->vec.nr, iov, +io_is_compat (req->ctx)); +if (IS_ERR (res)) +return PTR_ERR (res); + + kmsg->msg.msg_iter.nr_segs = msg.msg_iovlen; + req->flags |=REQ_F_IMPORT_BUFFER ; +return ret; +} +#define SENDMSG_FLAGS (IORING_RECVSEND_POLL_FIRST | IORING_RECVSEND_BUNDLE)int io_sendmsg_prep (struct io_kiocb *req,const struct io_uring_sqe *sqe)
@@ -1333,8 +1371,6 @@ int io_send_zc_prep(struct io_kiocb *req, const struct io_uring_sqe *sqe)if (req->opcode !=IORING_OP_SEND_ZC ) {if (unlikely (sqe->addr2 || sqe->file_index))return -EINVAL ; -if (unlikely (zc->flags &IORING_RECVSEND_FIXED_BUF )) -return -EINVAL ; } zc->len =READ_ONCE (sqe->len);
@@ -1350,7 +1386,7 @@ int io_send_zc_prep(struct io_kiocb *req, const struct io_uring_sqe *sqe)return -ENOMEM ;if (req->opcode !=IORING_OP_SENDMSG_ZC )return io_send_setup (req, sqe); -return io_sendmsg_setup (req, sqe); +return io_sendmsg_zc_setup (req, sqe); } static int io_sg_from_iter_iovec(struct sk_buff *skb,
@@ -1506,6 +1542,22 @@ int io_sendmsg_zc(struct io_kiocb *req, unsigned int issue_flags)unsigned flags;int ret, min_ret =0 ; + kmsg->msg.sg_from_iter = io_sg_from_iter_iovec; + +if (req->flags &REQ_F_IMPORT_BUFFER ) { +unsigned uvec_segs = kmsg->msg.msg_iter.nr_segs; +unsigned iovec_off = kmsg->vec.nr - uvec_segs; +int ret; + + ret =io_import_reg_vec (ITER_SOURCE , &kmsg->msg.msg_iter, req, + &kmsg->vec, uvec_segs, iovec_off, + issue_flags); +if (unlikely (ret)) +return ret; + kmsg->msg.sg_from_iter = io_sg_from_iter; + req->flags &= ~REQ_F_IMPORT_BUFFER ; + } + sock =sock_from_file (req->file);if (unlikely (!sock))return -ENOTSOCK ;
@@ -1524,7 +1576,6 @@ int io_sendmsg_zc(struct io_kiocb *req, unsigned int issue_flags) kmsg->msg.msg_control_user = sr->msg_control; kmsg->msg.msg_ubuf = &io_notif_to_data (sr->notif)->uarg; - kmsg->msg.sg_from_iter = io_sg_from_iter_iovec; ret =__sys_sendmsg_sock (sock, &kmsg->msg, flags);if (unlikely (ret < min_ret)) {
Signature · 1 line
2.48.1
Re: [PATCH 1/8] io_uring: introduce struct iou_vec
In reply to Pavel BegunkovQuoted 51 lines · Pavel Begunkov
I need a convenient way to pass around and work with iovec+size pair, put them into a structure and makes use of it in
rw.cSigned-off-by Pavel Begunkov <[email protected]>--- include/linux/io_uring_types.h | 5 +++++ io_uring/rsrc.c | 9 +++++++++ io_uring/rsrc.h | 17 +++++++++++++++++ io_uring/rw.c | 17 +++++++---------- io_uring/rw.h | 6 ++++-- 5 files changed, 42 insertions(+), 12 deletions(-)include/linux/io_uring_types.h +5
-0@@ -110,6 +110,11 @@ struct io_uring_task { }____cacheline_aligned_in_smp ; }; +struct iou_vec { +struct iovec *iovec; +unsigned nr; +}; +struct io_uring {u32 head;u32 tail;io_uring/rsrc.c +9
-0@@ -1264,3 +1264,12 @@ int io_register_clone_buffers(struct io_ring_ctx *ctx, void __user *arg)fput (file);return ret; } + +void io_vec_free (struct iou_vec *iv) +{ +if (!iv->iovec) +return ; +kfree (iv->iovec); + iv->iovec =NULL ; + iv->nr =0 ; +}io_uring/rsrc.h +0
-0
io_uring/rsrc.h line 6
Quoted 4 lines
io_uring/rsrc.h +1
-0@@ -3,6 +3,7 @@#define IOU_RSRC_H #include <linux/lockdep.h> +#include <linux/io_uring_types.h>
I sent out a separate patch for this a couple days ago:
https://lore.kernel.org/io-uring/[email protected]/T/#u
Quoted 84 lines
io_uring/rsrc.h +16
-0enum {IORING_RSRC_FILE =0 ,@@ -144,4 +145,20 @@ static inline void __io_unaccount_mem(struct user_struct *user, atomic_long_sub(nr_pages, &user->locked_vm); } +void io_vec_free (struct iou_vec *iv); + +static inline void io_vec_reset_iovec (struct iou_vec *iv, +struct iovec *iovec,unsigned nr) +{ +io_vec_free (iv); + iv->iovec = iovec; + iv->nr = nr; +} + +static inline void io_alloc_cache_vec_kasan (struct iou_vec *iv) +{ +if (IS_ENABLED (CONFIG_KASAN )) +io_vec_free (iv); +} +#endif io_uring/rw.c +7
-10@@ -87,9 +87,9 @@ static int io_import_vec(int ddir, struct io_kiocb *req,int ret, nr_segs;struct iovec *iov; -if (io->free_iovec) { - nr_segs = io->free_iov_nr; - iov = io->free_iovec; +if (io->vec.iovec) { + nr_segs = io->vec.nr; + iov = io->vec.iovec; }else { nr_segs =1 ; iov = &io->fast_iov;@@ -101,9 +101,7 @@ static int io_import_vec(int ddir, struct io_kiocb *req,return ret;if (iov) { req->flags |=REQ_F_NEED_CLEANUP ; - io->free_iov_nr = io->iter.nr_segs; -kfree (io->free_iovec); - io->free_iovec = iov; +io_vec_reset_iovec (&io->vec, iov, io->iter.nr_segs); }return 0 ; }@@ -151,7 +149,7 @@ static void io_rw_recycle(struct io_kiocb *req, unsigned int issue_flags)if (unlikely (issue_flags &IO_URING_F_UNLOCKED ))return ; -io_alloc_cache_kasan (&rw->free_iovec, &rw->free_iov_nr); +io_alloc_cache_vec_kasan (&rw->vec);if (io_alloc_cache_put (&req->ctx->rw_cache, rw)) { req->async_data =NULL ; req->flags &= ~REQ_F_ASYNC_DATA ;@@ -201,7 +199,7 @@ static int io_rw_alloc_async(struct io_kiocb *req) rw =io_uring_alloc_async_data (&ctx->rw_cache, req);if (!rw)return -ENOMEM ; -if (rw->free_iovec) +if (rw->vec.iovec) req->flags |=REQ_F_NEED_CLEANUP ; rw->bytes_done =0 ;return 0 ;@@ -1327,7 +1325,6 @@ void io_rw_cache_free(const void *entry) {struct io_async_rw *rw = (struct io_async_rw *) entry; -if (rw->free_iovec) -kfree (rw->free_iovec); +io_vec_free (&rw->vec);kfree (rw); }io_uring/rw.h +0
-0
io_uring/rw.h line 6
Quoted 4 lines
io_uring/rw.h +1
-0@@ -3,19 +3,21 @@#include <linux/io_uring_types.h> #include <linux/pagemap.h> +#include "rsrc.h"
Why is this include necessary? struct iou_vec is defined in io_uring_types.h. Seems like the include would make more sense in rw.c.
Best,
Caleb
Quoted 23 lines
io_uring/rw.h +3
-3+struct io_meta_state {u32 seed;struct iov_iter_state iter_meta; };struct io_async_rw { +struct iou_vec vec;size_t bytes_done; -struct iovec *free_iovec; + struct_group(clear, struct iov_iter iter;struct iov_iter_state iter_state;struct iovec fast_iov; -int free_iov_nr;/* * wpq is for buffered io, while meta fields are used with * direct io -- 2.48.1
Re: [PATCH 2/8] io_uring: add infra for importing vectored reg buffers
In reply to Pavel BegunkovQuoted 18 lines · Pavel Begunkov
Add
io_import_reg_vec(), which will be responsible for importing vectored registered buffers. iovecs are overlapped with the resulting bvec in memory, which is why the iovec is expected to be padded iniou_vec.Signed-off-by Pavel Begunkov <[email protected]>--- include/linux/io_uring_types.h | 5 +- io_uring/rsrc.c | 124 +++++++++++++++++++++++++++++++++ io_uring/rsrc.h | 5 ++ 3 files changed, 133 insertions(+), 1 deletion(-)include/linux/io_uring_types.h +0
-0
include/linux/io_uring_types.h struct io_uring_task { · line 117
Quoted 8 lines
include/linux/io_uring_types.h +4
-1@@ -111,7 +111,10 @@ struct io_uring_task { };struct iou_vec { -struct iovec *iovec; +union { +struct iovec *iovec; +struct bio_vec *bvec; + };
If I understand correctly, io_import_reg_vec() converts the iovecs to bio_vecs in place. If an iovec expands to more than one bio_vec (i.e. crosses a folio boundary), wouldn't the bio_vecs overwrite iovecs that hadn't been processed yet?
Quoted 42 lines
include/linux/io_uring_types.h +0
-0unsigned nr; };io_uring/rsrc.c +25
-0@@ -1267,9 +1267,133 @@ int io_register_clone_buffers(struct io_ring_ctx *ctx, void __user *arg)void io_vec_free (struct iou_vec *iv) { +BUILD_BUG_ON (sizeof (struct bio_vec ) >sizeof (struct iovec )); +if (!iv->iovec)return ;kfree (iv->iovec); iv->iovec =NULL ; iv->nr =0 ; } + +int io_vec_realloc (struct iou_vec *iv,unsigned nr_entries) +{ +struct iovec *iov; + +WARN_ON_ONCE (nr_entries <=0 ); + + iov =kmalloc_array (nr_entries,sizeof (iov[0 ]),GFP_KERNEL ); +if (!iov) +return -ENOMEM ; + +io_vec_free (iv); + iv->iovec = iov; + iv->nr = nr_entries; +return 0 ; +} + +static int io_vec_fill_bvec (int ddir,struct iov_iter *iter, +struct io_mapped_ubuf *imu, +struct iovec *iovec,int nr_iovs, +struct iou_vec *vec) +{ +unsigned long folio_size = (1 << imu->folio_shift);
io_uring/rsrc.c int io_register_clone_buffers(struct io_ring_ctx *ctx, void __user *arg) · line 1310
Quoted 10 lines
io_uring/rsrc.c +10
-0+unsigned long folio_mask = folio_size -1 ; +struct bio_vec *res_bvec = vec->bvec; +size_t total_len =0 ; +int bvec_idx =0 ; +int iov_idx; + +for (iov_idx =0 ; iov_idx < nr_iovs; iov_idx++) { +size_t iov_len = iovec[iov_idx].iov_len; +u64 buf_addr = (u64 )iovec[iov_idx].iov_base; +u64 folio_addr = imu->ubuf & ~folio_mask;
The computation of folio_addr could be moved out of the loop.
Quoted 12 lines
io_uring/rsrc.c +12
-0+struct bio_vec *src_bvec; +size_t offset; +u64 buf_end; + +if (unlikely (check_add_overflow (buf_addr, (u64 )iov_len, &buf_end))) +return -EFAULT ; +if (unlikely (buf_addr < imu->ubuf || buf_end > (imu->ubuf + imu->len))) +return -EFAULT ; + + total_len += iov_len; +/* by using folio address it also accounts for bvec offset */ + offset = buf_addr - folio_addr;
io_uring/rsrc.c int io_register_clone_buffers(struct io_ring_ctx *ctx, void __user *arg) · line 1332
Quoted 10 lines
io_uring/rsrc.c +10
-0+ src_bvec = imu->bvec + (offset >> imu->folio_shift); + offset &= folio_mask; + +for (; iov_len; offset =0 , bvec_idx++, src_bvec++) { +size_t seg_size =min_t (size_t, iov_len, + folio_size - offset); + + res_bvec[bvec_idx].bv_page = src_bvec->bv_page; + res_bvec[bvec_idx].bv_offset = offset; + res_bvec[bvec_idx].bv_len = seg_size;
Could just increment res_bvec to avoid the variable bvec_idx?
Quoted 9 lines
io_uring/rsrc.c +9
-0+ iov_len -= seg_size; + } + } +if (total_len >MAX_RW_COUNT ) +return -EINVAL ; + +iov_iter_bvec (iter, ddir, res_bvec, bvec_idx, total_len); +return 0 ; +}
io_uring/rsrc.c int io_register_clone_buffers(struct io_ring_ctx *ctx, void __user *arg) · line 1351
Quoted 10 lines
io_uring/rsrc.c +10
-0+ +static int io_estimate_bvec_size (struct iovec *iov,unsigned nr_iovs, +struct io_mapped_ubuf *imu) +{ +unsigned shift = imu->folio_shift; +size_t max_segs =0 ; +unsigned i; + +for (i =0 ; i < nr_iovs; i++) + max_segs += (iov[i].iov_len >> shift) +2 ;
Sees like this may overestimate a bit. I think something like this would give the exact number of segments for each iovec? (((u64)iov_base & folio_mask) + iov_len + folio_mask) >> folio_shift
Quoted 16 lines
io_uring/rsrc.c +16
-0+return max_segs; +} + +int io_import_reg_vec (int ddir,struct iov_iter *iter, +struct io_kiocb *req,struct iou_vec *vec, +int nr_iovs,unsigned iovec_off, +unsigned issue_flags) +{ +struct io_rsrc_node *node; +struct io_mapped_ubuf *imu; +unsigned cache_nr; +struct iovec *iov; +unsigned nr_segs; +int ret; + + node =io_find_buf_node (req, issue_flags);
io_uring/rsrc.c int io_register_clone_buffers(struct io_ring_ctx *ctx, void __user *arg) · line 1377
Quoted 10 lines
io_uring/rsrc.c +10
-0+if (!node) +return -EFAULT ; + imu = node->buf; +if (imu->is_kbuf) +return -EOPNOTSUPP ; + + iov = vec->iovec + iovec_off; + ret =io_estimate_bvec_size (iov, nr_iovs, imu); +if (ret <0 ) +return ret;
io_estimate_bvec_size() doesn't (intentionally) return an error code, just an unsigned value cast to an int.
Best,
Caleb
Quoted 49 lines
io_uring/rsrc.c +22
-0+ nr_segs = ret; + cache_nr = vec->nr; + +if (WARN_ON_ONCE (iovec_off + nr_iovs != cache_nr) || + nr_segs > cache_nr) { +struct iou_vec tmp_vec = {}; + + ret =io_vec_realloc (&tmp_vec, nr_segs); +if (ret) +return ret; + + iovec_off = tmp_vec.nr - nr_iovs; +memcpy (tmp_vec.iovec + iovec_off, iov,sizeof (*iov) * nr_iovs); +io_vec_free (vec); + + *vec = tmp_vec; + iov = vec->iovec + iovec_off; + req->flags |=REQ_F_NEED_CLEANUP ; + } + +return io_vec_fill_bvec (ddir, iter, imu, iov, nr_iovs, vec); +}io_uring/rsrc.h +5
-1@@ -61,6 +61,10 @@ struct io_rsrc_node *io_find_buf_node(struct io_kiocb *req,int io_import_reg_buf (struct io_kiocb *req,struct iov_iter *iter,u64 buf_addr,size_t len,int ddir,unsigned issue_flags); +int io_import_reg_vec (int ddir,struct iov_iter *iter, +struct io_kiocb *req,struct iou_vec *vec, +int nr_iovs,unsigned iovec_off, +unsigned issue_flags);int io_register_clone_buffers (struct io_ring_ctx *ctx,void __user *arg);int io_sqe_buffers_unregister (struct io_ring_ctx *ctx);@@ -146,6 +150,7 @@ static inline void __io_unaccount_mem(struct user_struct *user, }void io_vec_free (struct iou_vec *iv); +int io_vec_realloc (struct iou_vec *iv,unsigned nr_entries);static inline void io_vec_reset_iovec (struct iou_vec *iv,struct iovec *iovec,unsigned nr) --2.48.1
Re: [PATCH 2/8] io_uring: add infra for importing vectored reg buffers
Quoted 14 lines · Caleb Sander Mateos
Quoted 14 lines
io_uring/rsrc.c +14
-0+u64 buf_end; + +if (unlikely (check_add_overflow (buf_addr, (u64 )iov_len, &buf_end))) +return -EFAULT ; +if (unlikely (buf_addr < imu->ubuf || buf_end > (imu->ubuf + imu->len))) +return -EFAULT ; + + total_len += iov_len; +/* by using folio address it also accounts for bvec offset */ + offset = buf_addr - folio_addr; + src_bvec = imu->bvec + (offset >> imu->folio_shift); + offset &= folio_mask; + +for (; iov_len; offset =0 , bvec_idx++, src_bvec++) {
Quoted 8 lines
io_uring/rsrc.c int io_register_clone_buffers(struct io_ring_ctx *ctx, void __user *arg) · line 1332
Quoted 6 lines
io_uring/rsrc.c +6
-0+size_t seg_size =min_t (size_t, iov_len, + folio_size - offset); + + res_bvec[bvec_idx].bv_page = src_bvec->bv_page; + res_bvec[bvec_idx].bv_offset = offset; + res_bvec[bvec_idx].bv_len = seg_size;Could just increment
res_bvecto avoid the variablebvec_idx?
And utilizing bvec_set_page() to initialize looks a bit cleaner too.
Hi,
Quoted 3 lines · Pavel Begunkov
Add registered buffer support for vectored
io_uringoperations. That allows to pass an iovec, all entries of which must belong to and point into the same registered buffer specified bysqe->buf_index.
This is very much appreciated!
Quoted 8 lines
The series covers zerocopy sendmsg and reads / writes. Reads and writes are implemented as new opcodes, while zerocopy sendmsg reuses
IORING_RECVSEND_FIXED_BUFfor the api.Results are aligned to what one would expect from registered buffers:
t/io_uring + nullblk, single segment 16K:
34 -> 46 GiB/s
FWIW, I'd expect bigger wins with real IO when using 1GB huge pages. I encountered when there were a lot of reads from a large nvme raid into a small set of shared huge pages (database buffer pool), by many proceses concurrently. The constant pinning/unpinning of the relevant folio caused a lot of contention.
Unfortunately switching to registered buffers would, until now, have required using non-vectored IO, which causes significant performance regressions in other cases...
Greetings,
Andres Freund
Re: [PATCH 3/8] io_uring/rw: implement vectored registered rw
In reply to Pavel BegunkovQuoted 107 lines · Pavel Begunkov
Implement registered buffer vectored reads with new opcodes
IORING_OP_WRITEV_FIXEDandIORING_OP_READV_FIXED.Signed-off-by Pavel Begunkov <[email protected]>--- include/uapi/linux/io_uring.h | 2 ++ io_uring/opdef.c | 39 +++++++++++++++++++++++++++ io_uring/rw.c | 51 +++++++++++++++++++++++++++++++++++ io_uring/rw.h | 2 ++ 4 files changed, 94 insertions(+)include/uapi/linux/io_uring.h +2
-0@@ -280,6 +280,8 @@ enum io_uring_op {IORING_OP_BIND ,IORING_OP_LISTEN ,IORING_OP_RECV_ZC , +IORING_OP_READV_FIXED , +IORING_OP_WRITEV_FIXED ,/* this goes last, obviously */ IORING_OP_LAST,io_uring/opdef.c +39
-0@@ -529,6 +529,35 @@ const struct io_issue_def io_issue_defs[] = { .prep = io_eopnotsupp_prep, #endif }, + [IORING_OP_READV_FIXED ] = { + .needs_file =1 , + .unbound_nonreg_file =1 , + .pollin =1 , + .plug =1 , + .audit_skip =1 , + .ioprio =1 , + .iopoll =1 , + .iopoll_queue =1 , + .vectored =1 , + .async_size =sizeof (struct io_async_rw ), + .prep = io_prep_readv_fixed, + .issue = io_read, + }, + [IORING_OP_WRITEV_FIXED ] = { + .needs_file =1 , + .hash_reg_file =1 , + .unbound_nonreg_file =1 , + .pollout =1 , + .plug =1 , + .audit_skip =1 , + .ioprio =1 , + .iopoll =1 , + .iopoll_queue =1 , + .vectored =1 , + .async_size =sizeof (struct io_async_rw ), + .prep = io_prep_writev_fixed, + .issue = io_write, + }, };const struct io_cold_def io_cold_defs[] = {@@ -761,6 +790,16 @@ const struct io_cold_def io_cold_defs[] = { [IORING_OP_RECV_ZC ] = { .name ="RECV_ZC" , }, + [IORING_OP_READV_FIXED ] = { + .name ="READV_FIXED" , + .cleanup = io_readv_writev_cleanup, + .fail = io_rw_fail, + }, + [IORING_OP_WRITEV_FIXED ] = { + .name ="WRITEV_FIXED" , + .cleanup = io_readv_writev_cleanup, + .fail = io_rw_fail, + }, };const char *io_uring_get_opcode (u8 opcode)io_uring/rw.c +17
-0@@ -381,6 +381,57 @@ int io_prep_write_fixed(struct io_kiocb *req, const struct io_uring_sqe *sqe)return __io_prep_rw (req, sqe,ITER_SOURCE ); } +static int io_rw_prep_reg_vec (struct io_kiocb *req,int ddir) +{ + struct io_rw *rw = io_kiocb_to_cmd(req, struct io_rw); +struct io_async_rw *io = req->async_data; +const struct iovec __user *uvec; +size_t uvec_segs = rw->len; +struct iovec *iov; +int iovec_off, ret; +void *res; + +if (uvec_segs > io->vec.nr) { + ret =io_vec_realloc (&io->vec, uvec_segs); +if (ret) +return ret; + req->flags |=REQ_F_NEED_CLEANUP ; + } +/* pad iovec to the right */
io_uring/rw.c int io_prep_write_fixed(struct io_kiocb *req, const struct io_uring_sqe *sqe) · line 410
Quoted 10 lines
io_uring/rw.c +10
-0+ iovec_off = io->vec.nr - uvec_segs; + iov = io->vec.iovec + iovec_off; + uvec =u64_to_user_ptr (rw->addr); + res =iovec_from_user (uvec, uvec_segs, uvec_segs, iov, +io_is_compat (req->ctx)); +if (IS_ERR (res)) +return PTR_ERR (res); + + ret =io_import_reg_vec (ddir, &io->iter, req, &io->vec, + uvec_segs, iovec_off,0 );
So the iovecs are being imported at prep time rather than issue time? I suppose since only user registered buffers are allowed and not kernel bvecs, you aren't concerned about interactions with the ublk bvec register/unregister operations? I think in principle the difference between prep and issue time is still observable if the same registered buffer index is being used alternately for user and kernel registered buffers.
Best,
Caleb
Quoted 44 lines
io_uring/rw.c +24
-0+iov_iter_save_state (&io->iter, &io->iter_state); +return ret; +} + +int io_prep_readv_fixed (struct io_kiocb *req,const struct io_uring_sqe *sqe) +{ +int ret; + + ret =__io_prep_rw (req, sqe,ITER_DEST ); +if (unlikely (ret)) +return ret; +return io_rw_prep_reg_vec (req,ITER_DEST ); +} + +int io_prep_writev_fixed (struct io_kiocb *req,const struct io_uring_sqe *sqe) +{ +int ret; + + ret =__io_prep_rw (req, sqe,ITER_SOURCE ); +if (unlikely (ret)) +return ret; +return io_rw_prep_reg_vec (req,ITER_SOURCE ); +} +/* * Multishot read is prepared just like a normal read/write request, only * difference is that we set the MULTISHOT flag. io_uring/rw.h +2
-1@@ -34,6 +34,8 @@ struct io_async_rw {int io_prep_read_fixed (struct io_kiocb *req,const struct io_uring_sqe *sqe);int io_prep_write_fixed (struct io_kiocb *req,const struct io_uring_sqe *sqe); +int io_prep_readv_fixed (struct io_kiocb *req,const struct io_uring_sqe *sqe); +int io_prep_writev_fixed (struct io_kiocb *req,const struct io_uring_sqe *sqe);int io_prep_readv (struct io_kiocb *req,const struct io_uring_sqe *sqe);int io_prep_writev (struct io_kiocb *req,const struct io_uring_sqe *sqe);int io_prep_read (struct io_kiocb *req,const struct io_uring_sqe *sqe); --2.48.1
Re: [PATCH 3/8] io_uring/rw: implement vectored registered rw
Quoted 120 lines · Caleb Sander Mateos
Quoted 107 lines · Pavel Begunkov
Implement registered buffer vectored reads with new opcodes
IORING_OP_WRITEV_FIXEDandIORING_OP_READV_FIXED.Signed-off-by Pavel Begunkov <[email protected]>--- include/uapi/linux/io_uring.h | 2 ++ io_uring/opdef.c | 39 +++++++++++++++++++++++++++ io_uring/rw.c | 51 +++++++++++++++++++++++++++++++++++ io_uring/rw.h | 2 ++ 4 files changed, 94 insertions(+)include/uapi/linux/io_uring.h +2
-0@@ -280,6 +280,8 @@ enum io_uring_op {IORING_OP_BIND ,IORING_OP_LISTEN ,IORING_OP_RECV_ZC , +IORING_OP_READV_FIXED , +IORING_OP_WRITEV_FIXED ,/* this goes last, obviously */ IORING_OP_LAST,io_uring/opdef.c +39
-0@@ -529,6 +529,35 @@ const struct io_issue_def io_issue_defs[] = { .prep = io_eopnotsupp_prep, #endif }, + [IORING_OP_READV_FIXED ] = { + .needs_file =1 , + .unbound_nonreg_file =1 , + .pollin =1 , + .plug =1 , + .audit_skip =1 , + .ioprio =1 , + .iopoll =1 , + .iopoll_queue =1 , + .vectored =1 , + .async_size =sizeof (struct io_async_rw ), + .prep = io_prep_readv_fixed, + .issue = io_read, + }, + [IORING_OP_WRITEV_FIXED ] = { + .needs_file =1 , + .hash_reg_file =1 , + .unbound_nonreg_file =1 , + .pollout =1 , + .plug =1 , + .audit_skip =1 , + .ioprio =1 , + .iopoll =1 , + .iopoll_queue =1 , + .vectored =1 , + .async_size =sizeof (struct io_async_rw ), + .prep = io_prep_writev_fixed, + .issue = io_write, + }, };const struct io_cold_def io_cold_defs[] = {@@ -761,6 +790,16 @@ const struct io_cold_def io_cold_defs[] = { [IORING_OP_RECV_ZC ] = { .name ="RECV_ZC" , }, + [IORING_OP_READV_FIXED ] = { + .name ="READV_FIXED" , + .cleanup = io_readv_writev_cleanup, + .fail = io_rw_fail, + }, + [IORING_OP_WRITEV_FIXED ] = { + .name ="WRITEV_FIXED" , + .cleanup = io_readv_writev_cleanup, + .fail = io_rw_fail, + }, };const char *io_uring_get_opcode (u8 opcode)io_uring/rw.c +17
-0@@ -381,6 +381,57 @@ int io_prep_write_fixed(struct io_kiocb *req, const struct io_uring_sqe *sqe)return __io_prep_rw (req, sqe,ITER_SOURCE ); } +static int io_rw_prep_reg_vec (struct io_kiocb *req,int ddir) +{ + struct io_rw *rw = io_kiocb_to_cmd(req, struct io_rw); +struct io_async_rw *io = req->async_data; +const struct iovec __user *uvec; +size_t uvec_segs = rw->len; +struct iovec *iov; +int iovec_off, ret; +void *res; + +if (uvec_segs > io->vec.nr) { + ret =io_vec_realloc (&io->vec, uvec_segs); +if (ret) +return ret; + req->flags |=REQ_F_NEED_CLEANUP ; + } +/* pad iovec to the right */ io_uring/rw.c int io_prep_write_fixed(struct io_kiocb *req, const struct io_uring_sqe *sqe) · line 410
Quoted 10 lines
io_uring/rw.c +10
-0+ iovec_off = io->vec.nr - uvec_segs; + iov = io->vec.iovec + iovec_off; + uvec =u64_to_user_ptr (rw->addr); + res =iovec_from_user (uvec, uvec_segs, uvec_segs, iov, +io_is_compat (req->ctx)); +if (IS_ERR (res)) +return PTR_ERR (res); + + ret =io_import_reg_vec (ddir, &io->iter, req, &io->vec, + uvec_segs, iovec_off,0 );
Quoted 7 lines
So the iovecs are being imported at prep time rather than issue time? I suppose since only user registered buffers are allowed and not kernel bvecs, you aren't concerned about interactions with the ublk bvec register/unregister operations? I think in principle the difference between prep and issue time is still observable if the same registered buffer index is being used alternately for user and kernel registered buffers.
Never mind, I see you change this in the next patch.
Best,
Caleb
Re: [PATCH 4/8] io_uring/rw: defer reg buf vec import
In reply to Pavel BegunkovQuoted 61 lines · Pavel Begunkov
Import registered buffers for vectored reads and writes later at issue time as we now do for other fixed ops.
Signed-off-by Pavel Begunkov <[email protected]>--- include/linux/io_uring_types.h | 3 +++ io_uring/rw.c | 36 +++++++++++++++++++++++++++++----- 2 files changed, 34 insertions(+), 5 deletions(-)include/linux/io_uring_types.h +3
-0@@ -502,6 +502,7 @@ enum {REQ_F_BUFFERS_COMMIT_BIT ,REQ_F_BUF_NODE_BIT ,REQ_F_HAS_METADATA_BIT , +REQ_F_IMPORT_BUFFER_BIT ,/* not a real bit, just to check we're not overflowing the space */ __REQ_F_LAST_BIT,@@ -584,6 +585,8 @@ enum {REQ_F_BUF_NODE =IO_REQ_FLAG (REQ_F_BUF_NODE_BIT ),/* request has read/write metadata assigned */ REQ_F_HAS_METADATA =IO_REQ_FLAG (REQ_F_HAS_METADATA_BIT ), +/* resolve padded iovec to registered buffers */ + REQ_F_IMPORT_BUFFER = IO_REQ_FLAG(REQ_F_IMPORT_BUFFER_BIT), };typedef void (*io_req_tw_func_t )(struct io_kiocb *req,io_tw_token_t tw);io_uring/rw.c +18
-0@@ -381,6 +381,24 @@ int io_prep_write_fixed(struct io_kiocb *req, const struct io_uring_sqe *sqe)return __io_prep_rw (req, sqe,ITER_SOURCE ); } +static int io_rw_import_reg_vec (struct io_kiocb *req, +struct io_async_rw *io, +int ddir,unsigned int issue_flags) +{ + struct io_rw *rw = io_kiocb_to_cmd(req, struct io_rw); +unsigned uvec_segs = rw->len; +unsigned iovec_off = io->vec.nr - uvec_segs; +int ret; + + ret =io_import_reg_vec (ddir, &io->iter, req, &io->vec, + uvec_segs, iovec_off, issue_flags); +if (unlikely (ret)) +return ret; +iov_iter_save_state (&io->iter, &io->iter_state); + req->flags &= ~REQ_F_IMPORT_BUFFER ; +return 0 ; +} +static int io_rw_prep_reg_vec (struct io_kiocb *req,int ddir) { struct io_rw *rw = io_kiocb_to_cmd(req, struct io_rw);
io_uring/rw.c static int io_rw_prep_reg_vec(struct io_kiocb *req, int ddir) · line 428
Quoted 9 lines
io_uring/rw.c +2
-4@@ -406,10 +424,8 @@ static int io_rw_prep_reg_vec(struct io_kiocb *req, int ddir)if (IS_ERR (res))return PTR_ERR (res); - ret =io_import_reg_vec (ddir, &io->iter, req, &io->vec, - uvec_segs, iovec_off,0 ); -iov_iter_save_state (&io->iter, &io->iter_state); -return ret; + req->flags |=REQ_F_IMPORT_BUFFER ; +return 0 ;
Looks like ddir is now unused in this function?
Best,
Caleb
Quoted 33 lines
io_uring/rw.c +11
-2}int io_prep_readv_fixed (struct io_kiocb *req,const struct io_uring_sqe *sqe)@@ -906,7 +922,11 @@ static int __io_read(struct io_kiocb *req, unsigned int issue_flags)ssize_t ret;loff_t *ppos; -if (io_do_buffer_select (req)) { +if (req->flags &REQ_F_IMPORT_BUFFER ) { + ret =io_rw_import_reg_vec (req, io,ITER_DEST , issue_flags); +if (unlikely (ret)) +return ret; + }else if (io_do_buffer_select (req)) { ret =io_import_rw_buffer (ITER_DEST , req, io, issue_flags);if (unlikely (ret <0 ))return ret;@@ -1117,6 +1137,12 @@ int io_write(struct io_kiocb *req, unsigned int issue_flags)ssize_t ret, ret2;loff_t *ppos; +if (req->flags &REQ_F_IMPORT_BUFFER ) { + ret =io_rw_import_reg_vec (req, io,ITER_SOURCE , issue_flags); +if (unlikely (ret)) +return ret; + } + ret =io_rw_init_file (req,FMODE_WRITE ,WRITE );if (unlikely (ret))return ret; --2.48.1
Re: [PATCH 7/8] io_uring/net: convert to struct iou_vec
In reply to Pavel BegunkovQuoted 35 lines · Pavel Begunkov
Convert
net.cto use structiou_vec.Signed-off-by Pavel Begunkov <[email protected]>--- io_uring/alloc_cache.h | 9 -------- io_uring/net.c | 51 ++++++++++++++++++------------------------ io_uring/net.h | 6 ++--- 3 files changed, 25 insertions(+), 41 deletions(-)io_uring/alloc_cache.h +0
-9@@ -16,15 +16,6 @@ bool io_alloc_cache_init(struct io_alloc_cache *cache,void *io_cache_alloc_new (struct io_alloc_cache *cache,gfp_t gfp); -static inline void io_alloc_cache_kasan (struct iovec **iov,int *nr) -{ -if (IS_ENABLED (CONFIG_KASAN )) { -kfree (*iov); - *iov =NULL ; - *nr =0 ; - } -} -static inline bool io_alloc_cache_put (struct io_alloc_cache *cache,void *entry) {io_uring/net.c +0
-0
io_uring/net.c static bool io_net_retry(struct socket *sock, int flags) · line 140
Quoted 10 lines
io_uring/net.c +2
-5@@ -136,11 +136,8 @@ static bool io_net_retry(struct socket *sock, int flags)static void io_netmsg_iovec_free (struct io_async_msghdr *kmsg) { -if (kmsg->free_iov) { -kfree (kmsg->free_iov); - kmsg->free_iov_nr =0 ; - kmsg->free_iov =NULL ; - } +if (kmsg->vec.iovec) +io_vec_free (&kmsg->vec);
io_vec_free() already checks vec.iovec, is it necessary to duplicate it? If it's an unlikely case and you'd like to avoid the function call overhead, then move io_vec_free() to the header file so it can be inlined?
Best,
Caleb
Quoted 141 lines
io_uring/net.c +20
-24}static void io_netmsg_recycle (struct io_kiocb *req,unsigned int issue_flags)@@ -154,7 +151,7 @@ static void io_netmsg_recycle(struct io_kiocb *req, unsigned int issue_flags) }/* Let normal cleanup path reap it if we fail adding to the cache */ -io_alloc_cache_kasan (&hdr->free_iov, &hdr->free_iov_nr); +io_alloc_cache_vec_kasan (&hdr->vec);if (io_alloc_cache_put (&req->ctx->netmsg_cache, hdr)) { req->async_data =NULL ; req->flags &= ~REQ_F_ASYNC_DATA ;@@ -171,7 +168,7 @@ static struct io_async_msghdr *io_msg_alloc_async(struct io_kiocb *req)return NULL ;/* If the async data was cached, we might have an iov cached inside. */ -if (hdr->free_iov) +if (hdr->vec.iovec) req->flags |=REQ_F_NEED_CLEANUP ;return hdr; }@@ -182,10 +179,7 @@ static void io_net_vec_assign(struct io_kiocb *req, struct io_async_msghdr *kmsg {if (iov) { req->flags |=REQ_F_NEED_CLEANUP ; - kmsg->free_iov_nr = kmsg->msg.msg_iter.nr_segs; -if (kmsg->free_iov) -kfree (kmsg->free_iov); - kmsg->free_iov = iov; +io_vec_reset_iovec (&kmsg->vec, iov, kmsg->msg.msg_iter.nr_segs); } }@@ -208,9 +202,9 @@ static int io_net_import_vec(struct io_kiocb *req, struct io_async_msghdr *iomsgstruct iovec *iov;int ret, nr_segs; -if (iomsg->free_iov) { - nr_segs = iomsg->free_iov_nr; - iov = iomsg->free_iov; +if (iomsg->vec.iovec) { + nr_segs = iomsg->vec.nr; + iov = iomsg->vec.iovec; }else { nr_segs =1 ; iov = &iomsg->fast_iov;@@ -468,7 +462,7 @@ static int io_bundle_nbufs(struct io_async_msghdr *kmsg, int ret)if (iter_is_ubuf (&kmsg->msg.msg_iter))return 1 ; - iov = kmsg->free_iov; + iov = kmsg->vec.iovec;if (!iov) iov = &kmsg->fast_iov;@@ -584,9 +578,9 @@ static int io_send_select_buffer(struct io_kiocb *req, unsigned int issue_flags, .nr_iovs = 1, }; -if (kmsg->free_iov) { - arg.nr_iovs = kmsg->free_iov_nr; - arg.iovs = kmsg->free_iov; +if (kmsg->vec.iovec) { + arg.nr_iovs = kmsg->vec.nr; + arg.iovs = kmsg->vec.iovec; arg.mode =KBUF_MODE_FREE ; }@@ -599,9 +593,9 @@ static int io_send_select_buffer(struct io_kiocb *req, unsigned int issue_flags,if (unlikely (ret <0 ))return ret; -if (arg.iovs != &kmsg->fast_iov && arg.iovs != kmsg->free_iov) { - kmsg->free_iov_nr = ret; - kmsg->free_iov = arg.iovs; +if (arg.iovs != &kmsg->fast_iov && arg.iovs != kmsg->vec.iovec) { + kmsg->vec.nr = ret; + kmsg->vec.iovec = arg.iovs; req->flags |=REQ_F_NEED_CLEANUP ; } sr->len = arg.out_len;@@ -1085,9 +1079,9 @@ static int io_recv_buf_select(struct io_kiocb *req, struct io_async_msghdr *kmsg .mode = KBUF_MODE_EXPAND, }; -if (kmsg->free_iov) { - arg.nr_iovs = kmsg->free_iov_nr; - arg.iovs = kmsg->free_iov; +if (kmsg->vec.iovec) { + arg.nr_iovs = kmsg->vec.nr; + arg.iovs = kmsg->vec.iovec; arg.mode |=KBUF_MODE_FREE ; }@@ -1106,9 +1100,9 @@ static int io_recv_buf_select(struct io_kiocb *req, struct io_async_msghdr *kmsg }iov_iter_init (&kmsg->msg.msg_iter,ITER_DEST , arg.iovs, ret, arg.out_len); -if (arg.iovs != &kmsg->fast_iov && arg.iovs != kmsg->free_iov) { - kmsg->free_iov_nr = ret; - kmsg->free_iov = arg.iovs; +if (arg.iovs != &kmsg->fast_iov && arg.iovs != kmsg->vec.iovec) { + kmsg->vec.nr = ret; + kmsg->vec.iovec = arg.iovs; req->flags |=REQ_F_NEED_CLEANUP ; } }else {@@ -1874,8 +1868,7 @@ void io_netmsg_cache_free(const void *entry) {struct io_async_msghdr *kmsg = (struct io_async_msghdr *) entry; -if (kmsg->free_iov) -io_netmsg_iovec_free (kmsg); +io_vec_free (&kmsg->vec);kfree (kmsg); }#endif io_uring/net.h +3
-4@@ -2,12 +2,12 @@#include <linux/net.h> #include <linux/uio.h> +#include <linux/io_uring_types.h> struct io_async_msghdr {#if defined(CONFIG_NET ) -struct iovec *free_iov; -/* points to an allocated iov, if NULL we use fast_iov instead */ -int free_iov_nr; +struct iou_vec vec; + struct_group(clear, int namelen;struct iovec fast_iov; --2.48.1
Quoted 9 lines · Pavel Begunkov
Add registered buffer support for vectored
io_uringoperations. That allows to pass an iovec, all entries of which must belong to and point into the same registered buffer specified bysqe->buf_index.The series covers zerocopy sendmsg and reads / writes. Reads and writes are implemented as new opcodes, while zerocopy sendmsg reuses
IORING_RECVSEND_FIXED_BUFfor the api.
Quoted 6 lines
Results are aligned to what one would expect from registered buffers:
t/io_uring + nullblk, single segment 16K:
34 -> 46 GiB/sexamples/send-zerocopy.cdefault send size (64KB):
82558 -> 123855 MB/s
Thanks for implementing this, it's great to be able to combine these 2 optimizations! Though I suspect many applications will want to perform vectorized I/O using iovecs that come from different registered buffers (e.g. separate header and data allocations). Perhaps a future improvement could allow a list of buffer indices to be specified.
Thanks,
Caleb
Re: [PATCH 2/8] io_uring: add infra for importing vectored reg buffers
In reply to Caleb Sander MateosQuoted 1 line · Caleb Sander Mateos
On Mon, Mar 3, 2025 at 7:51 AM Pavel Begunkov <[email protected]> wrote:
...
Quoted 5 lines
If I understand correctly,
io_import_reg_vec()converts the iovecs tobio_vecsin place. If an iovec expands to more than onebio_vec(i.e. crosses a folio boundary), wouldn't thebio_vecsoverwrite iovecs that hadn't been processed yet?
It's handled, obviously, you missed that the vectors are offset'ed from each other.
Quoted 4 lines
Quoted 4 lines
io_uring/rsrc.c
Quoted 1 line
io_uring/rsrc.c +0
-0index
9b05e614819e..1ec1f5b3e385100644io_uring/rsrc.c +0
-0
...
Quoted 9 lines
io_uring/rsrc.c int io_register_clone_buffers(struct io_ring_ctx *ctx, void __user *arg) · line 1332
Quoted 7 lines
io_uring/rsrc.c +7
-0+for (; iov_len; offset =0 , bvec_idx++, src_bvec++) { +size_t seg_size =min_t (size_t, iov_len, + folio_size - offset); + + res_bvec[bvec_idx].bv_page = src_bvec->bv_page; + res_bvec[bvec_idx].bv_offset = offset; + res_bvec[bvec_idx].bv_len = seg_size;Could just increment
res_bvecto avoid the variablebvec_idx?
I don't see the benefit.
Quoted 6 lines
io_uring/rsrc.c int io_register_clone_buffers(struct io_ring_ctx *ctx, void __user *arg) · line 1351
Quoted 2 lines
io_uring/rsrc.c +2
-0+for (i =0 ; i < nr_iovs; i++) + max_segs += (iov[i].iov_len >> shift) +2 ;Sees like this may overestimate a bit. I think something like this would give the exact number of segments for each iovec? (((u64)iov_base &
folio_mask) +iov_len+folio_mask) >>folio_shift
It's overestimated exactly to avoid a beast like this.
Signature · 1 line
Pavel Begunkov
Re: [PATCH 3/8] io_uring/rw: implement vectored registered rw
In reply to Caleb Sander MateosOn 3/3/25 23:01, Caleb Sander Mateos wrote:
...
Quoted 12 lines
io_uring/rw.c int io_prep_write_fixed(struct io_kiocb *req, const struct io_uring_sqe *sqe) · line 410
Quoted 11 lines
io_uring/rw.c +11
-0+/* pad iovec to the right */ + iovec_off = io->vec.nr - uvec_segs; + iov = io->vec.iovec + iovec_off; + uvec =u64_to_user_ptr (rw->addr); + res =iovec_from_user (uvec, uvec_segs, uvec_segs, iov, +io_is_compat (req->ctx)); +if (IS_ERR (res)) +return PTR_ERR (res); + + ret =io_import_reg_vec (ddir, &io->iter, req, &io->vec, + uvec_segs, iovec_off,0 );
Quoted 3 lines
So the iovecs are being imported at prep time rather than issue time? I suppose since only user registered buffers are allowed and not kernel bvecs, you aren't concerned about interactions with the ublk
It's a question of generic io_uring policy and has nothing to do with ublk. Thinking about it in terms of a specific user wouldn't be productive.
Signature · 1 line
Pavel Begunkov
Quoted 8 lines · Andres Freund
Hi,
Quoted 3 lines · Pavel Begunkov
Add registered buffer support for vectored
io_uringoperations. That allows to pass an iovec, all entries of which must belong to and point into the same registered buffer specified bysqe->buf_index.This is very much appreciated!'
Glad to hear. I do remember you mentioning the contention issue in the list. A bunch of other people who were interested as well.
Quoted 10 lines
Quoted 8 lines
The series covers zerocopy sendmsg and reads / writes. Reads and writes are implemented as new opcodes, while zerocopy sendmsg reuses
IORING_RECVSEND_FIXED_BUFfor the api.Results are aligned to what one would expect from registered buffers:
t/io_uring + nullblk, single segment 16K:
34 -> 46 GiB/sFWIW, I'd expect bigger wins with real IO when using 1GB huge pages. I
I didn't even benchmark it meaningfully as we should be able to extrapolate results from registered buffer test, but I agree, such contention might make it even more desirable.
Quoted 8 lines
encountered when there were a lot of reads from a large nvme raid into a small set of shared huge pages (database buffer pool), by many proceses concurrently. The constant pinning/unpinning of the relevant folio caused a lot of contention.
Unfortunately switching to registered buffers would, until now, have required using non-vectored IO, which causes significant performance regressions in other cases...
Signature · 1 line
Pavel Begunkov
Quoted 17 lines · Caleb Sander Mateos
Quoted 9 lines · Pavel Begunkov
Add registered buffer support for vectored
io_uringoperations. That allows to pass an iovec, all entries of which must belong to and point into the same registered buffer specified bysqe->buf_index.The series covers zerocopy sendmsg and reads / writes. Reads and writes are implemented as new opcodes, while zerocopy sendmsg reuses
IORING_RECVSEND_FIXED_BUFfor the api.Quoted 6 lines
Results are aligned to what one would expect from registered buffers:
t/io_uring + nullblk, single segment 16K:
34 -> 46 GiB/s examples/send-zerocopy.c default send size (64KB): 82558 -> 123855 MB/s
Quoted 5 lines
Thanks for implementing this, it's great to be able to combine these 2 optimizations! Though I suspect many applications will want to perform vectorized I/O using iovecs that come from different registered buffers (e.g. separate header and data allocations). Perhaps a future improvement could allow a list of buffer indices to be specified.
That's the design decision made, otherwise the API is becoming a mess as well as handling. The user has to be smart and keep a small number of large registered buffers and potentially growing them, which is a good thing regardless of this feature.
Signature · 1 line
Pavel Begunkov
Hi Pavel,
Quoted 15 lines
Quoted 8 lines
Add registered buffer support for vectored
io_uringoperations. That allows to pass an iovec, all entries of which must belong to and point into the same registered buffer specified bysqe->buf_index.The series covers zerocopy sendmsg and reads / writes. Reads and writes are implemented as new opcodes, while zerocopy sendmsg reuses
IORING_RECVSEND_FIXED_BUFfor the api.Quoted 6 lines
Results are aligned to what one would expect from registered buffers:
t/io_uring + nullblk, single segment 16K:
34 -> 46 GiB/s examples/send-zerocopy.c default send size (64KB): 82558 -> 123855 MB/s
Quoted 5 lines
Thanks for implementing this, it's great to be able to combine these 2 optimizations! Though I suspect many applications will want to perform vectorized I/O using iovecs that come from different registered buffers (e.g. separate header and data allocations). Perhaps a future improvement could allow a list of buffer indices to be specified.
I'm wondering about the same. And it's not completely clear to me what the value of iov_base is in this case, is it the offset into the buffer, or the real pointer address that must within the range of the registered buffer?
It might also be very useful to have some vector elements pointing into one of the registered buffer, while others refer to non-registered buffers.
metze
Quoted 23 lines · Stefan Metzmacher
Hi Pavel,
Quoted 15 lines
Quoted 8 lines
Add registered buffer support for vectored
io_uringoperations. That allows to pass an iovec, all entries of which must belong to and point into the same registered buffer specified bysqe->buf_index.The series covers zerocopy sendmsg and reads / writes. Reads and writes are implemented as new opcodes, while zerocopy sendmsg reuses
IORING_RECVSEND_FIXED_BUFfor the api.Quoted 6 lines
Results are aligned to what one would expect from registered buffers:
t/io_uring + nullblk, single segment 16K:
34 -> 46 GiB/sexamples/send-zerocopy.cdefault send size (64KB):
82558 -> 123855 MB/sQuoted 5 lines
Thanks for implementing this, it's great to be able to combine these 2 optimizations! Though I suspect many applications will want to perform vectorized I/O using iovecs that come from different registered buffers (e.g. separate header and data allocations). Perhaps a future improvement could allow a list of buffer indices to be specified.
Quoted 4 lines
I'm wondering about the same. And it's not completely clear to me what the value of
iov_baseis in this case, is it the offset into the buffer, or the real pointer address that must within the range of the registered buffer?
Same as with other registered buffer requests. It's a pointer into the initial buffer you specified when registering it, which serves to calculate the offset.
See the io_uring_register(2) man, addr and len are iov_base and iov_len and there are multiple of them. You can call it confusing, and I'd agree, but that's how it was done from the very beginning, so staying consistent here.
https://github.com/axboe/liburing/blob/master/man/io_uring_register.2#L87
Quoted 3 lines
It might also be very useful to have some vector elements pointing into one of the registered buffer, while others refer to non-registered buffers.
See the other reply.
Signature · 1 line
Pavel Begunkov
Re: [PATCH 2/8] io_uring: add infra for importing vectored reg buffers
In reply to Pavel BegunkovQuoted 11 lines · Pavel Begunkov
Quoted 1 line · Caleb Sander Mateos
On Mon, Mar 3, 2025 at 7:51 AM Pavel Begunkov <[email protected]> wrote:
...
Quoted 6 lines
Quoted 2 lines
+ for (i = 0; i <
nr_iovs; i++)
+max_segs+= (iov[i].iov_len >> shift) + 2;Sees like this may overestimate a bit. I think something like this would give the exact number of segments for each iovec? (((u64)iov_base &
folio_mask) +iov_len+folio_mask) >>folio_shiftIt's overestimated exactly to avoid a beast like this.
And it'd be broken as well for 0 len.
Signature · 1 line
Pavel Begunkov