Add support for cloning partial buffer sets
Jens Axboe
maintainercover Add support for cloning partial buffer sets
Hi,
6.12 added buffer cloning support, but it's an all-or-nothing kind of thing - if there's an existing buffer table in the destination ring, then nothing can be cloned to it.
This adds support for cloning partial buffer sets, specifying a source/dest offset and the number of buffers to clone. And it allows cloning to replace existing nodes as well, specified with a separate flag.
include/uapi/linux/io_uring.h | 8 +++-- io_uring/rsrc.c | 60 +++++++++++++++++++++++++++++------ 2 files changed, 56 insertions(+), 12 deletions(-)
Signature · 1 line
Jens Axboe
Jens Axboe
maintainerPATCH 1/2 io_uring/rsrc: allow cloning at an offset
Right now buffer cloning is an all-or-nothing kind of thing - either the whole table is cloned from a source to a destination ring, or nothing at all.
However, it's not always desired to clone the whole thing. Allow for the application to specify a source and destination offset, and a number of buffers to clone. If the destination offset is non-zero, then allocate sparse nodes upfront.
--- include/uapi/linux/io_uring.h | 5 ++++- io_uring/rsrc.c | 36 +++++++++++++++++++++++++++++------ 2 files changed, 34 insertions(+), 7 deletions(-)
include/uapi/linux/io_uring.h +4 -1
@@ -719,7 +719,10 @@ enum {struct io_uring_clone_buffers {__u32 src_fd;__u32 flags; -__u32 pad[6 ]; +__u32 src_off; +__u32 dst_off; +__u32 nr; +__u32 pad[3 ]; };struct io_uring_buf {
io_uring/rsrc.c +30 -6
@@ -924,10 +924,11 @@ int io_import_fixed(int ddir, struct iov_iter *iter,return 0 ; } -static int io_clone_buffers (struct io_ring_ctx *ctx,struct io_ring_ctx *src_ctx) +static int io_clone_buffers (struct io_ring_ctx *ctx,struct io_ring_ctx *src_ctx, +struct io_uring_clone_buffers *arg) { +int i, ret, nbufs, off, nr;struct io_rsrc_data data; -int i, ret, nbufs;/* * Drop our own lock here. We'll setup the data we need and reference
@@ -940,11 +941,33 @@ static int io_clone_buffers(struct io_ring_ctx *ctx, struct io_ring_ctx *src_ctx nbufs = src_ctx->buf_table.nr;if (!nbufs) goto out_unlock; - ret =io_rsrc_data_alloc (&data, nbufs); + ret = -EINVAL ; +if (!arg->nr) + arg->nr = nbufs; +else if (arg->nr > nbufs) + goto out_unlock; + ret = -EOVERFLOW ; +if (check_add_overflow (arg->nr, arg->src_off, &off)) + goto out_unlock; +if (off > nbufs) + goto out_unlock; +if (check_add_overflow (arg->nr, arg->dst_off, &off)) + goto out_unlock; + ret = -EINVAL ; +if (off >IORING_MAX_REG_BUFFERS ) + goto out_unlock; + ret =io_rsrc_data_alloc (&data, off);if (ret) goto out_unlock; -for (i =0 ; i < nbufs; i++) { +/* fill empty/sparse nodes, if needed */ +for (i =0 ; i < arg->dst_off; i++) + data.nodes[i] = rsrc_empty_node; + + off = arg->dst_off; + i = arg->src_off; + nr = arg->nr; +while (nr--) {struct io_rsrc_node *dst_node, *src_node; src_node =io_rsrc_node_lookup (&src_ctx->buf_table, i);
@@ -960,7 +983,8 @@ static int io_clone_buffers(struct io_ring_ctx *ctx, struct io_ring_ctx *src_ctxrefcount_inc (&src_node->buf->refs); dst_node->buf = src_node->buf; } - data.nodes[i] = dst_node; + data.nodes[off++] = dst_node; + i++; }/* Have a ref on the bufs now, drop src lock and re-grab our own lock */
@@ -1015,7 +1039,7 @@ int io_register_clone_buffers(struct io_ring_ctx *ctx, void __user *arg) file =io_uring_register_get_file (buf.src_fd, registered_src);if (IS_ERR (file))return PTR_ERR (file); - ret =io_clone_buffers (ctx, file->private_data); + ret =io_clone_buffers (ctx, file->private_data, &buf);if (!registered_src)fput (file);return ret;
Signature · 1 line
2.45.2
Jens Axboe
maintainerPATCH 2/2 io_uring/rsrc: allow cloning with node replacements
In reply to Jens AxboeCurrently cloning a buffer table will fail if the destination already has a table. But it should be possible to use it to replace existing elements. Add a IORING_REGISTER_DST_REPLACE cloning flag, which if set, will allow the destination to already having a buffer table. If that is the case, then entries designated by offset + nr buffers will be replaced if they already exist.
Note that it's allowed to use IORING_REGISTER_DST_REPLACE and not have an existing table, in which case it'll work just like not having the flag set and an empty table - it'll just assign the newly created table for that case.
--- include/uapi/linux/io_uring.h | 3 ++- io_uring/rsrc.c | 24 ++++++++++++++++++++---- 2 files changed, 22 insertions(+), 5 deletions(-)
include/uapi/linux/io_uring.h +2 -1
@@ -713,7 +713,8 @@ struct io_uring_clock_register { };enum { -IORING_REGISTER_SRC_REGISTERED =1 , +IORING_REGISTER_SRC_REGISTERED = (1U <<0 ), +IORING_REGISTER_DST_REPLACE = (1U <<1 ), };struct io_uring_clone_buffers {
io_uring/rsrc.c +20 -4
@@ -990,9 +990,25 @@ static int io_clone_buffers(struct io_ring_ctx *ctx, struct io_ring_ctx *src_ctx/* Have a ref on the bufs now, drop src lock and re-grab our own lock */ mutex_unlock (&src_ctx->uring_lock);mutex_lock (&ctx->uring_lock); -if (!ctx->buf_table.nr) { + +/* +* Not replacing, or replacing an empty table. Just install the +* new table. +*/ +if (!(arg->flags &IORING_REGISTER_DST_REPLACE ) || !ctx->buf_table.nr) { ctx->buf_table = data;return 0 ; + }else if (arg->flags &IORING_REGISTER_DST_REPLACE ) { +/* put nodes in overlapping spots, if any */ +for (i = arg->src_off; i < arg->nr; i++) { +if (data.nodes[i] == rsrc_empty_node) +continue ; +io_reset_rsrc_node (&ctx->buf_table, i); + ctx->buf_table.nodes[i] = data.nodes[i]; + data.nodes[i] =NULL ; + } +io_rsrc_data_free (&data); +return 0 ; }mutex_unlock (&ctx->uring_lock);
@@ -1026,12 +1042,12 @@ int io_register_clone_buffers(struct io_ring_ctx *ctx, void __user *arg)struct file *file;int ret; -if (ctx->buf_table.nr) -return -EBUSY ;if (copy_from_user (&buf, arg,sizeof (buf)))return -EFAULT ; -if (buf.flags & ~IORING_REGISTER_SRC_REGISTERED ) +if (buf.flags & ~(IORING_REGISTER_SRC_REGISTERED |IORING_REGISTER_DST_REPLACE ))return -EINVAL ; +if (!(buf.flags &IORING_REGISTER_DST_REPLACE ) && ctx->buf_table.nr) +return -EBUSY ;if (memchr_inv (buf.pad,0 ,sizeof (buf.pad)))return -EINVAL ;
Signature · 1 line
2.45.2