Add support for vectored registered buffers
v4 cover Add support for vectored registered buffers
Add registered buffer support for vectored io_uring operations. That allows to pass an iovec, all entries of which must belong to and point into the same registered buffer specified by sqe->buf_index.
The series covers zerocopy sendmsg and reads / writes. Reads and writes are implemented as new opcodes, while zerocopy sendmsg reuses IORING_RECVSEND_FIXED_BUF for the user API.
Results are aligned to what one would expect from registered buffers:
t/io_uring + nullblk, single segment 16K:
34 -> 46 GiB/sexamples/send-zerocopy.c default send size (64KB):
82558 -> 123855 MB/s
The series is placed on top of 6.15 + zcrx + epoll.
v4: Address cast and include warnings
v3: Handle 32 bit where bvec is larger than iovec
v2: Nowarn alloc
Cap bvec caching
Check length overflow
Reject 0 len segments
Check op direction
Other minor changes
Pavel Begunkov (9):
io_uring: introduce struct iou_vec
io_uring: add infra for importing vectored reg buffers io_uring/rw: implement vectored registered rw io_uring/rw: defer reg buf vec import io_uring/net: combine msghdr copy io_uring/net: pull vec alloc out of msghdr import io_uring/net: convert to struct iou_vec io_uring/net: implement vectored reg bufs for zctx
io_uring: cap cached iovec/bvec size
include/linux/io_uring_types.h | 11 ++ include/uapi/linux/io_uring.h | 2 + io_uring/alloc_cache.h | 9 -- io_uring/net.c | 180 +++++++++++++++++++++------------ io_uring/net.h | 6 +- io_uring/opdef.c | 39 +++++++ io_uring/rsrc.c | 137 +++++++++++++++++++++++++ io_uring/rsrc.h | 23 +++++ io_uring/rw.c | 99 ++++++++++++++++-- io_uring/rw.h | 6 +- 10 files changed, 420 insertions(+), 92 deletions(-)
Signature · 1 line
2.48.1
PATCH v4 1/9 io_uring: introduce struct iou_vec
I need a convenient way to pass around and work with iovec+size pair, put them into a structure and makes use of it in rw.c
--- include/linux/io_uring_types.h | 5 +++++ io_uring/rsrc.c | 9 +++++++++ io_uring/rsrc.h | 16 ++++++++++++++++ io_uring/rw.c | 17 +++++++---------- io_uring/rw.h | 4 ++-- 5 files changed, 39 insertions(+), 12 deletions(-)
include/linux/io_uring_types.h +5 -0
@@ -110,6 +110,11 @@ struct io_uring_task { }____cacheline_aligned_in_smp ; }; +struct iou_vec { +struct iovec *iovec; +unsigned nr; +}; +struct io_uring {u32 head;u32 tail;
io_uring/rsrc.c +9 -0
@@ -1260,3 +1260,12 @@ int io_register_clone_buffers(struct io_ring_ctx *ctx, void __user *arg)fput (file);return ret; } + +void io_vec_free (struct iou_vec *iv) +{ +if (!iv->iovec) +return ; +kfree (iv->iovec); + iv->iovec =NULL ; + iv->nr =0 ; +}
io_uring/rsrc.h +16 -0
@@ -145,4 +145,20 @@ static inline void __io_unaccount_mem(struct user_struct *user, atomic_long_sub(nr_pages, &user->locked_vm); } +void io_vec_free (struct iou_vec *iv); + +static inline void io_vec_reset_iovec (struct iou_vec *iv, +struct iovec *iovec,unsigned nr) +{ +io_vec_free (iv); + iv->iovec = iovec; + iv->nr = nr; +} + +static inline void io_alloc_cache_vec_kasan (struct iou_vec *iv) +{ +if (IS_ENABLED (CONFIG_KASAN )) +io_vec_free (iv); +} +#endif
io_uring/rw.c +7 -10
@@ -87,9 +87,9 @@ static int io_import_vec(int ddir, struct io_kiocb *req,int ret, nr_segs;struct iovec *iov; -if (io->free_iovec) { - nr_segs = io->free_iov_nr; - iov = io->free_iovec; +if (io->vec.iovec) { + nr_segs = io->vec.nr; + iov = io->vec.iovec; }else { nr_segs =1 ; iov = &io->fast_iov;
@@ -101,9 +101,7 @@ static int io_import_vec(int ddir, struct io_kiocb *req,return ret;if (iov) { req->flags |=REQ_F_NEED_CLEANUP ; - io->free_iov_nr = io->iter.nr_segs; -kfree (io->free_iovec); - io->free_iovec = iov; +io_vec_reset_iovec (&io->vec, iov, io->iter.nr_segs); }return 0 ; }
@@ -151,7 +149,7 @@ static void io_rw_recycle(struct io_kiocb *req, unsigned int issue_flags)if (unlikely (issue_flags &IO_URING_F_UNLOCKED ))return ; -io_alloc_cache_kasan (&rw->free_iovec, &rw->free_iov_nr); +io_alloc_cache_vec_kasan (&rw->vec);if (io_alloc_cache_put (&req->ctx->rw_cache, rw)) { req->async_data =NULL ; req->flags &= ~REQ_F_ASYNC_DATA ;
@@ -201,7 +199,7 @@ static int io_rw_alloc_async(struct io_kiocb *req) rw =io_uring_alloc_async_data (&ctx->rw_cache, req);if (!rw)return -ENOMEM ; -if (rw->free_iovec) +if (rw->vec.iovec) req->flags |=REQ_F_NEED_CLEANUP ; rw->bytes_done =0 ;return 0 ;
@@ -1327,7 +1325,6 @@ void io_rw_cache_free(const void *entry) {struct io_async_rw *rw = (struct io_async_rw *) entry; -if (rw->free_iovec) -kfree (rw->free_iovec); +io_vec_free (&rw->vec);kfree (rw); }
io_uring/rw.h +2 -2
@@ -9,13 +9,13 @@ struct io_meta_state { };struct io_async_rw { +struct iou_vec vec;size_t bytes_done; -struct iovec *free_iovec; + struct_group(clear, struct iov_iter iter;struct iov_iter_state iter_state;struct iovec fast_iov; -int free_iov_nr;/* * wpq is for buffered io, while meta fields are used with * direct io
Signature · 1 line
2.48.1
PATCH v4 2/9 io_uring: add infra for importing vectored reg buffers
In reply to Pavel BegunkovAdd io_import_reg_vec(), which will be responsible for importing vectored registered buffers. The function might reallocate the vector, but it'd try to do the conversion in place first, which is why it's required of the user to pad the iovec to the right border of the cache.
Overlapping also depends on struct iovec being larger than bvec, which is not the case on e.g. 32 bit architectures. Don't try to complicate this case and make sure vectors never overlap, it'll be improved later.
--- include/linux/io_uring_types.h | 7 +- io_uring/rsrc.c | 128 +++++++++++++++++++++++++++++++++ io_uring/rsrc.h | 5 ++ 3 files changed, 138 insertions(+), 2 deletions(-)
include/linux/io_uring_types.h +5 -2
@@ -111,8 +111,11 @@ struct io_uring_task { };struct iou_vec { -struct iovec *iovec; -unsigned nr; +union { +struct iovec *iovec; +struct bio_vec *bvec; + }; +unsigned nr;/* number of struct iovec it can hold */ };struct io_uring {
io_uring/rsrc.c +128 -0
131 lines not on this page · raw message
io_uring/rsrc.h +5 -0
@@ -61,6 +61,10 @@ struct io_rsrc_node *io_find_buf_node(struct io_kiocb *req,int io_import_reg_buf (struct io_kiocb *req,struct iov_iter *iter,u64 buf_addr,size_t len,int ddir,unsigned issue_flags); +int io_import_reg_vec (int ddir,struct iov_iter *iter, +struct io_kiocb *req,struct iou_vec *vec, +unsigned nr_iovs,unsigned iovec_off, +unsigned issue_flags);int io_register_clone_buffers (struct io_ring_ctx *ctx,void __user *arg);int io_sqe_buffers_unregister (struct io_ring_ctx *ctx);
@@ -146,6 +150,7 @@ static inline void __io_unaccount_mem(struct user_struct *user, }void io_vec_free (struct iou_vec *iv); +int io_vec_realloc (struct iou_vec *iv,unsigned nr_entries);static inline void io_vec_reset_iovec (struct iou_vec *iv,struct iovec *iovec,unsigned nr)
Signature · 1 line
2.48.1
PATCH v4 3/9 io_uring/rw: implement vectored registered rw
In reply to Pavel BegunkovImplement registered buffer vectored reads with new opcodes IORING_OP_WRITEV_FIXED and IORING_OP_READV_FIXED.
--- include/uapi/linux/io_uring.h | 2 ++ io_uring/opdef.c | 39 +++++++++++++++++++++++++++ io_uring/rw.c | 51 +++++++++++++++++++++++++++++++++++ io_uring/rw.h | 2 ++ 4 files changed, 94 insertions(+)
include/uapi/linux/io_uring.h +2 -0
@@ -281,6 +281,8 @@ enum io_uring_op {IORING_OP_LISTEN ,IORING_OP_RECV_ZC ,IORING_OP_EPOLL_WAIT , +IORING_OP_READV_FIXED , +IORING_OP_WRITEV_FIXED ,/* this goes last, obviously */ IORING_OP_LAST,
io_uring/opdef.c +39 -0
@@ -540,6 +540,35 @@ const struct io_issue_def io_issue_defs[] = { .prep = io_eopnotsupp_prep, #endif }, + [IORING_OP_READV_FIXED ] = { + .needs_file =1 , + .unbound_nonreg_file =1 , + .pollin =1 , + .plug =1 , + .audit_skip =1 , + .ioprio =1 , + .iopoll =1 , + .iopoll_queue =1 , + .vectored =1 , + .async_size =sizeof (struct io_async_rw ), + .prep = io_prep_readv_fixed, + .issue = io_read, + }, + [IORING_OP_WRITEV_FIXED ] = { + .needs_file =1 , + .hash_reg_file =1 , + .unbound_nonreg_file =1 , + .pollout =1 , + .plug =1 , + .audit_skip =1 , + .ioprio =1 , + .iopoll =1 , + .iopoll_queue =1 , + .vectored =1 , + .async_size =sizeof (struct io_async_rw ), + .prep = io_prep_writev_fixed, + .issue = io_write, + }, };const struct io_cold_def io_cold_defs[] = {
@@ -775,6 +804,16 @@ const struct io_cold_def io_cold_defs[] = { [IORING_OP_EPOLL_WAIT ] = { .name ="EPOLL_WAIT" , }, + [IORING_OP_READV_FIXED ] = { + .name ="READV_FIXED" , + .cleanup = io_readv_writev_cleanup, + .fail = io_rw_fail, + }, + [IORING_OP_WRITEV_FIXED ] = { + .name ="WRITEV_FIXED" , + .cleanup = io_readv_writev_cleanup, + .fail = io_rw_fail, + }, };const char *io_uring_get_opcode (u8 opcode)
io_uring/rw.c +51 -0
57 lines not on this page · raw message
io_uring/rw.h +2 -0
@@ -32,6 +32,8 @@ struct io_async_rw {int io_prep_read_fixed (struct io_kiocb *req,const struct io_uring_sqe *sqe);int io_prep_write_fixed (struct io_kiocb *req,const struct io_uring_sqe *sqe); +int io_prep_readv_fixed (struct io_kiocb *req,const struct io_uring_sqe *sqe); +int io_prep_writev_fixed (struct io_kiocb *req,const struct io_uring_sqe *sqe);int io_prep_readv (struct io_kiocb *req,const struct io_uring_sqe *sqe);int io_prep_writev (struct io_kiocb *req,const struct io_uring_sqe *sqe);int io_prep_read (struct io_kiocb *req,const struct io_uring_sqe *sqe);
Signature · 1 line
2.48.1
PATCH v4 4/9 io_uring/rw: defer reg buf vec import
In reply to Pavel BegunkovImport registered buffers for vectored reads and writes later at issue time as we now do for other fixed ops.
--- include/linux/io_uring_types.h | 3 +++ io_uring/rw.c | 42 +++++++++++++++++++++++++++------- 2 files changed, 37 insertions(+), 8 deletions(-)
include/linux/io_uring_types.h +3 -0
@@ -502,6 +502,7 @@ enum {REQ_F_BUFFERS_COMMIT_BIT ,REQ_F_BUF_NODE_BIT ,REQ_F_HAS_METADATA_BIT , +REQ_F_IMPORT_BUFFER_BIT ,/* not a real bit, just to check we're not overflowing the space */ __REQ_F_LAST_BIT,
@@ -584,6 +585,8 @@ enum {REQ_F_BUF_NODE =IO_REQ_FLAG (REQ_F_BUF_NODE_BIT ),/* request has read/write metadata assigned */ REQ_F_HAS_METADATA =IO_REQ_FLAG (REQ_F_HAS_METADATA_BIT ), +/* resolve padded iovec to registered buffers */ + REQ_F_IMPORT_BUFFER = IO_REQ_FLAG(REQ_F_IMPORT_BUFFER_BIT), };typedef void (*io_req_tw_func_t )(struct io_kiocb *req,io_tw_token_t tw);
io_uring/rw.c +34 -8
78 lines not on this page · raw message
Signature · 1 line
2.48.1
PATCH v4 5/9 io_uring/net: combine msghdr copy
In reply to Pavel BegunkovCall the compat version from inside of io_msg_copy_hdr() and don't duplicate it in callers.
--- io_uring/net.c | 46 +++++++++++++++++++--------------------------- 1 file changed, 19 insertions(+), 27 deletions(-)
io_uring/net.c +19 -27
@@ -287,6 +287,24 @@ static int io_msg_copy_hdr(struct io_kiocb *req, struct io_async_msghdr *iomsg,struct user_msghdr __user *umsg = sr->umsg;int ret; + iomsg->msg.msg_name = &iomsg->addr; + iomsg->msg.msg_iter.nr_segs =0 ; + +if (io_is_compat (req->ctx)) { +struct compat_msghdr cmsg; + + ret =io_compat_msg_copy_hdr (req, iomsg, &cmsg, ddir, save_addr); +if (ret) +return ret; + +memset (&msg,0 ,sizeof (msg)); + msg->msg_namelen = cmsg.msg_namelen; + msg->msg_controllen = cmsg.msg_controllen; + msg->msg_iov =compat_ptr (cmsg.msg_iov); + msg->msg_iovlen = cmsg.msg_iovlen; +return 0 ; + } + ret =io_copy_msghdr_from_user (msg, umsg);if (unlikely (ret))return ret;
@@ -323,18 +341,6 @@ static int io_sendmsg_copy_hdr(struct io_kiocb *req,struct user_msghdr msg;int ret; - iomsg->msg.msg_name = &iomsg->addr; - iomsg->msg.msg_iter.nr_segs =0 ; - -if (io_is_compat (req->ctx)) { -struct compat_msghdr cmsg; - - ret =io_compat_msg_copy_hdr (req, iomsg, &cmsg,ITER_SOURCE , -NULL ); - sr->msg_control = iomsg->msg.msg_control_user; -return ret; - } - ret =io_msg_copy_hdr (req, iomsg, &msg,ITER_SOURCE ,NULL );/* save msg_control as sys_sendmsg() overwrites it */ sr->msg_control = iomsg->msg.msg_control_user;
@@ -710,21 +716,7 @@ static int io_recvmsg_copy_hdr(struct io_kiocb *req,struct user_msghdr msg;int ret; - iomsg->msg.msg_name = &iomsg->addr; - iomsg->msg.msg_iter.nr_segs =0 ; - -if (io_is_compat (req->ctx)) { -struct compat_msghdr cmsg; - - ret =io_compat_msg_copy_hdr (req, iomsg, &cmsg,ITER_DEST , - &iomsg->uaddr); -memset (&msg,0 ,sizeof (msg)); - msg.msg_namelen = cmsg.msg_namelen; - msg.msg_controllen = cmsg.msg_controllen; - }else { - ret =io_msg_copy_hdr (req, iomsg, &msg,ITER_DEST , &iomsg->uaddr); - } - + ret =io_msg_copy_hdr (req, iomsg, &msg,ITER_DEST , &iomsg->uaddr);if (unlikely (ret))return ret; return io_recvmsg_mshot_prep(req, iomsg, msg.msg_namelen,
Signature · 1 line
2.48.1
PATCH v4 6/9 io_uring/net: pull vec alloc out of msghdr import
In reply to Pavel BegunkovI'll need more control over iovec management, move io_net_import_vec() out of io_msg_copy_hdr().
--- io_uring/net.c | 23 +++++++++++++++-------- 1 file changed, 15 insertions(+), 8 deletions(-)
io_uring/net.c +15 -8
@@ -253,12 +253,8 @@ static int io_compat_msg_copy_hdr(struct io_kiocb *req,return -EFAULT ; sr->len = tmp_iov.iov_len; } - -return 0 ; } - -return io_net_import_vec (req, iomsg, (struct iovec __user *)uiov, - msg->msg_iovlen, ddir); +return 0 ; } static int io_copy_msghdr_from_user(struct user_msghdr *msg,
@@ -328,10 +324,8 @@ static int io_msg_copy_hdr(struct io_kiocb *req, struct io_async_msghdr *iomsg,return -EFAULT ; sr->len = tmp_iov.iov_len; } -return 0 ; } - -return io_net_import_vec (req, iomsg, msg->msg_iov, msg->msg_iovlen, ddir); +return 0 ; } static int io_sendmsg_copy_hdr(struct io_kiocb *req,
@@ -342,6 +336,12 @@ static int io_sendmsg_copy_hdr(struct io_kiocb *req,int ret; ret =io_msg_copy_hdr (req, iomsg, &msg,ITER_SOURCE ,NULL ); +if (unlikely (ret)) +return ret; + +if (!(req->flags &REQ_F_BUFFER_SELECT )) + ret =io_net_import_vec (req, iomsg, msg.msg_iov, msg.msg_iovlen, +ITER_SOURCE );/* save msg_control as sys_sendmsg() overwrites it */ sr->msg_control = iomsg->msg.msg_control_user;return ret;
@@ -719,6 +719,13 @@ static int io_recvmsg_copy_hdr(struct io_kiocb *req, ret =io_msg_copy_hdr (req, iomsg, &msg,ITER_DEST , &iomsg->uaddr);if (unlikely (ret))return ret; + +if (!(req->flags &REQ_F_BUFFER_SELECT )) { + ret =io_net_import_vec (req, iomsg, msg.msg_iov, msg.msg_iovlen, +ITER_DEST ); +if (unlikely (ret)) +return ret; + }return io_recvmsg_mshot_prep (req, iomsg, msg.msg_namelen, msg.msg_controllen); }
Signature · 1 line
2.48.1
PATCH v4 7/9 io_uring/net: convert to struct iou_vec
In reply to Pavel BegunkovConvert net.c to use struct iou_vec.
--- io_uring/alloc_cache.h | 9 -------- io_uring/net.c | 51 ++++++++++++++++++------------------------ io_uring/net.h | 6 ++--- 3 files changed, 25 insertions(+), 41 deletions(-)
io_uring/alloc_cache.h +0 -9
@@ -16,15 +16,6 @@ bool io_alloc_cache_init(struct io_alloc_cache *cache,void *io_cache_alloc_new (struct io_alloc_cache *cache,gfp_t gfp); -static inline void io_alloc_cache_kasan (struct iovec **iov,int *nr) -{ -if (IS_ENABLED (CONFIG_KASAN )) { -kfree (*iov); - *iov =NULL ; - *nr =0 ; - } -} -static inline bool io_alloc_cache_put (struct io_alloc_cache *cache,void *entry) {
io_uring/net.c +22 -29
117 lines not on this page · raw message
io_uring/net.h +3 -3
@@ -2,12 +2,12 @@#include <linux/net.h> #include <linux/uio.h> +#include <linux/io_uring_types.h> struct io_async_msghdr {#if defined(CONFIG_NET ) -struct iovec *free_iov; -/* points to an allocated iov, if NULL we use fast_iov instead */ -int free_iov_nr; +struct iou_vec vec; + struct_group(clear, int namelen;struct iovec fast_iov;
Signature · 1 line
2.48.1
PATCH v4 8/9 io_uring/net: implement vectored reg bufs for zctx
In reply to Pavel BegunkovAdd support for vectored registered buffers for send zc.
--- io_uring/net.c | 59 ++++++++++++++++++++++++++++++++++++++++++++++---- 1 file changed, 55 insertions(+), 4 deletions(-)
io_uring/net.c +55 -4
@@ -395,6 +395,44 @@ static int io_sendmsg_setup(struct io_kiocb *req, const struct io_uring_sqe *sqereturn io_sendmsg_copy_hdr (req, kmsg); } +static int io_sendmsg_zc_setup (struct io_kiocb *req,const struct io_uring_sqe *sqe) +{ + struct io_sr_msg *sr = io_kiocb_to_cmd(req, struct io_sr_msg); +struct io_async_msghdr *kmsg = req->async_data; +struct user_msghdr msg; +int ret, iovec_off; +struct iovec *iov; +void *res; + +if (!(sr->flags &IORING_RECVSEND_FIXED_BUF )) +return io_sendmsg_setup (req, sqe); + + sr->umsg =u64_to_user_ptr (READ_ONCE (sqe->addr)); + + ret =io_msg_copy_hdr (req, kmsg, &msg,ITER_SOURCE ,NULL ); +if (unlikely (ret)) +return ret; + sr->msg_control = kmsg->msg.msg_control_user; + +if (msg.msg_iovlen > kmsg->vec.nr ||WARN_ON_ONCE (!kmsg->vec.iovec)) { + ret =io_vec_realloc (&kmsg->vec, msg.msg_iovlen); +if (ret) +return ret; + req->flags |=REQ_F_NEED_CLEANUP ; + } + iovec_off = kmsg->vec.nr - msg.msg_iovlen; + iov = kmsg->vec.iovec + iovec_off; + + res =iovec_from_user (msg.msg_iov, msg.msg_iovlen, kmsg->vec.nr, iov, +io_is_compat (req->ctx)); +if (IS_ERR (res)) +return PTR_ERR (res); + + kmsg->msg.msg_iter.nr_segs = msg.msg_iovlen; + req->flags |=REQ_F_IMPORT_BUFFER ; +return ret; +} +#define SENDMSG_FLAGS (IORING_RECVSEND_POLL_FIRST | IORING_RECVSEND_BUNDLE)int io_sendmsg_prep (struct io_kiocb *req,const struct io_uring_sqe *sqe)
@@ -1333,8 +1371,6 @@ int io_send_zc_prep(struct io_kiocb *req, const struct io_uring_sqe *sqe)if (req->opcode !=IORING_OP_SEND_ZC ) {if (unlikely (sqe->addr2 || sqe->file_index))return -EINVAL ; -if (unlikely (zc->flags &IORING_RECVSEND_FIXED_BUF )) -return -EINVAL ; } zc->len =READ_ONCE (sqe->len);
@@ -1350,7 +1386,7 @@ int io_send_zc_prep(struct io_kiocb *req, const struct io_uring_sqe *sqe)return -ENOMEM ;if (req->opcode !=IORING_OP_SENDMSG_ZC )return io_send_setup (req, sqe); -return io_sendmsg_setup (req, sqe); +return io_sendmsg_zc_setup (req, sqe); } static int io_sg_from_iter_iovec(struct sk_buff *skb,
@@ -1506,6 +1542,22 @@ int io_sendmsg_zc(struct io_kiocb *req, unsigned int issue_flags)unsigned flags;int ret, min_ret =0 ; + kmsg->msg.sg_from_iter = io_sg_from_iter_iovec; + +if (req->flags &REQ_F_IMPORT_BUFFER ) { +unsigned uvec_segs = kmsg->msg.msg_iter.nr_segs; +unsigned iovec_off = kmsg->vec.nr - uvec_segs; +int ret; + + ret =io_import_reg_vec (ITER_SOURCE , &kmsg->msg.msg_iter, req, + &kmsg->vec, uvec_segs, iovec_off, + issue_flags); +if (unlikely (ret)) +return ret; + kmsg->msg.sg_from_iter = io_sg_from_iter; + req->flags &= ~REQ_F_IMPORT_BUFFER ; + } + sock =sock_from_file (req->file);if (unlikely (!sock))return -ENOTSOCK ;
@@ -1524,7 +1576,6 @@ int io_sendmsg_zc(struct io_kiocb *req, unsigned int issue_flags) kmsg->msg.msg_control_user = sr->msg_control; kmsg->msg.msg_ubuf = &io_notif_to_data (sr->notif)->uarg; - kmsg->msg.sg_from_iter = io_sg_from_iter_iovec; ret =__sys_sendmsg_sock (sock, &kmsg->msg, flags);if (unlikely (ret < min_ret)) {
Signature · 1 line
2.48.1
PATCH v4 9/9 io_uring: cap cached iovec/bvec size
In reply to Pavel BegunkovBvecs can be large, put an arbitrary limit on the max vector size it can cache.
--- io_uring/net.c | 3 +++ io_uring/rsrc.h | 2 ++ io_uring/rw.c | 3 +++ 3 files changed, 8 insertions(+)
io_uring/net.c +3 -0
@@ -152,6 +152,9 @@ static void io_netmsg_recycle(struct io_kiocb *req, unsigned int issue_flags)/* Let normal cleanup path reap it if we fail adding to the cache */ io_alloc_cache_vec_kasan (&hdr->vec); +if (hdr->vec.nr >IO_VEC_CACHE_SOFT_CAP ) +io_vec_free (&hdr->vec); +if (io_alloc_cache_put (&req->ctx->netmsg_cache, hdr)) { req->async_data =NULL ; req->flags &= ~REQ_F_ASYNC_DATA ;
io_uring/rsrc.h +2 -0
@@ -5,6 +5,8 @@#include <linux/io_uring_types.h> #include <linux/lockdep.h> +#define IO_VEC_CACHE_SOFT_CAP 256 +enum {IORING_RSRC_FILE =0 ,IORING_RSRC_BUFFER =1 ,
io_uring/rw.c +3 -0
@@ -150,6 +150,9 @@ static void io_rw_recycle(struct io_kiocb *req, unsigned int issue_flags)return ;io_alloc_cache_vec_kasan (&rw->vec); +if (rw->vec.nr >IO_VEC_CACHE_SOFT_CAP ) +io_vec_free (&rw->vec); +if (io_alloc_cache_put (&req->ctx->rw_cache, rw)) { req->async_data =NULL ; req->flags &= ~REQ_F_ASYNC_DATA ;
Signature · 1 line
2.48.1
Quoted 9 lines · Pavel Begunkov
Add registered buffer support for vectored
io_uringoperations. That allows to pass an iovec, all entries of which must belong to and point into the same registered buffer specified bysqe->buf_index.The series covers zerocopy sendmsg and reads / writes. Reads and writes are implemented as new opcodes, while zerocopy sendmsg reuses
IORING_RECVSEND_FIXED_BUFfor the user API.[...]
Applied, thanks!
[1/9] io_uring: introduce struct iou_vec
commit: e1d499590977a492ae120d9263bd55076aabd460
[2/9] io_uring: add infra for importing vectored reg buffers
commit: 9ef4cbbcb4ac3786a1a4164507511b76b2a572c5
[3/9] io_uring/rw: implement vectored registered rw
commit: bdabba04bb1023e0327998b1eb0be096079bde65
[4/9] io_uring/rw: defer reg buf vec import
commit: 835c4bdf95d5c71fd5b41f77f2343b695b4494aa
[5/9] io_uring/net: combine msghdr copy
commit: 7fc9b27f8ee940ebb4035957e15225732e106d09
[6/9] io_uring/net: pull vec alloc out of msghdr import
commit: ae89ab8845288c751a3cc7f3215cb44ec6a900f7
[7/9] io_uring/net: convert to struct iou_vec
commit: 8066b67a8a0c4fdf93c043698c36474cc1fc0ff7
[8/9] io_uring/net: implement vectored reg bufs for zctx
commit: c983552d59531792533191186def14ea34d80590
[9/9] io_uring: cap cached iovec/bvec size
commit: 89176d5837668a429d862c9aff392226026ae7b3Best regards,
Signature · 1 line
Jens Axboe