Skip to contentSkip to conversation

Add support for cloning partial buffer sets

PATCH Superseded Jens Axboe 1 participant 3 messages Oct 30, 2024 io_uring

v1 of 2 Superseded Latest is v2 →

lore
Share & tools
lore

Jens Axboe

author maintainer
#0 lore raw

cover Add support for cloning partial buffer sets

Hi,

6.12 added buffer cloning support, but it's an all-or-nothing kind of thing - if there's an existing buffer table in the destination ring, then nothing can be cloned to it.

This adds support for cloning partial buffer sets, specifying a source/dest offset and the number of buffers to clone. And it allows cloning to replace existing nodes as well, specified with a separate flag.

 include/uapi/linux/io_uring.h |  8 +++--
 io_uring/rsrc.c               | 60 +++++++++++++++++++++++++++++------
 2 files changed, 56 insertions(+), 12 deletions(-)
Signature · 1 line
Jens Axboe
author maintainer
#1 lore raw

PATCH 1/2 io_uring/rsrc: allow cloning at an offset

Right now buffer cloning is an all-or-nothing kind of thing - either the whole table is cloned from a source to a destination ring, or nothing at all.

However, it's not always desired to clone the whole thing. Allow for the application to specify a source and destination offset, and a number of buffers to clone. If the destination offset is non-zero, then allocate sparse nodes upfront.

---
 include/uapi/linux/io_uring.h |  5 ++++-
 io_uring/rsrc.c               | 36 +++++++++++++++++++++++++++++------
 2 files changed, 34 insertions(+), 7 deletions(-)
include/uapi/linux/io_uring.h +4 -1
index 024745283783..cc8dbe78c126 100644
@@ -719,7 +719,10 @@ enum {
 struct io_uring_clone_buffers {
 	__u32	src_fd;
 	__u32	flags;
-	__u32	pad[6];
+	__u32	src_off;
+	__u32	dst_off;
+	__u32	nr;
+	__u32	pad[3];
 };
 
 struct io_uring_buf {
io_uring/rsrc.c +30 -6
index af60d9f597be..4c149dc42fd7 100644
@@ -924,10 +924,11 @@ int io_import_fixed(int ddir, struct iov_iter *iter,
 	return 0;
 }
 
-static int io_clone_buffers(struct io_ring_ctx *ctx, struct io_ring_ctx *src_ctx)
+static int io_clone_buffers(struct io_ring_ctx *ctx, struct io_ring_ctx *src_ctx,
+			    struct io_uring_clone_buffers *arg)
 {
+	int i, ret, nbufs, off, nr;
 	struct io_rsrc_data data;
-	int i, ret, nbufs;
 
 	/*
 	 * Drop our own lock here. We'll setup the data we need and reference
@@ -940,11 +941,33 @@ static int io_clone_buffers(struct io_ring_ctx *ctx, struct io_ring_ctx *src_ctx
 	nbufs = src_ctx->buf_table.nr;
 	if (!nbufs)
 		goto out_unlock;
-	ret = io_rsrc_data_alloc(&data, nbufs);
+	ret = -EINVAL;
+	if (!arg->nr)
+		arg->nr = nbufs;
+	else if (arg->nr > nbufs)
+		goto out_unlock;
+	ret = -EOVERFLOW;
+	if (check_add_overflow(arg->nr, arg->src_off, &off))
+		goto out_unlock;
+	if (off > nbufs)
+		goto out_unlock;
+	if (check_add_overflow(arg->nr, arg->dst_off, &off))
+		goto out_unlock;
+	ret = -EINVAL;
+	if (off > IORING_MAX_REG_BUFFERS)
+		goto out_unlock;
+	ret = io_rsrc_data_alloc(&data, off);
 	if (ret)
 		goto out_unlock;
 
-	for (i = 0; i < nbufs; i++) {
+	/* fill empty/sparse nodes, if needed */
+	for (i = 0; i < arg->dst_off; i++)
+		data.nodes[i] = rsrc_empty_node;
+
+	off = arg->dst_off;
+	i = arg->src_off;
+	nr = arg->nr;
+	while (nr--) {
 		struct io_rsrc_node *dst_node, *src_node;
 
 		src_node = io_rsrc_node_lookup(&src_ctx->buf_table, i);
@@ -960,7 +983,8 @@ static int io_clone_buffers(struct io_ring_ctx *ctx, struct io_ring_ctx *src_ctx
 			refcount_inc(&src_node->buf->refs);
 			dst_node->buf = src_node->buf;
 		}
-		data.nodes[i] = dst_node;
+		data.nodes[off++] = dst_node;
+		i++;
 	}
 
 	/* Have a ref on the bufs now, drop src lock and re-grab our own lock */
@@ -1015,7 +1039,7 @@ int io_register_clone_buffers(struct io_ring_ctx *ctx, void __user *arg)
 	file = io_uring_register_get_file(buf.src_fd, registered_src);
 	if (IS_ERR(file))
 		return PTR_ERR(file);
-	ret = io_clone_buffers(ctx, file->private_data);
+	ret = io_clone_buffers(ctx, file->private_data, &buf);
 	if (!registered_src)
 		fput(file);
 	return ret;
Signature · 1 line
2.45.2
author maintainer
#2 lore raw

PATCH 2/2 io_uring/rsrc: allow cloning with node replacements

In reply to Jens Axboe

Currently cloning a buffer table will fail if the destination already has a table. But it should be possible to use it to replace existing elements. Add a IORING_REGISTER_DST_REPLACE cloning flag, which if set, will allow the destination to already having a buffer table. If that is the case, then entries designated by offset + nr buffers will be replaced if they already exist.

Note that it's allowed to use IORING_REGISTER_DST_REPLACE and not have an existing table, in which case it'll work just like not having the flag set and an empty table - it'll just assign the newly created table for that case.

---
 include/uapi/linux/io_uring.h |  3 ++-
 io_uring/rsrc.c               | 24 ++++++++++++++++++++----
 2 files changed, 22 insertions(+), 5 deletions(-)
include/uapi/linux/io_uring.h +2 -1
index cc8dbe78c126..ce58c4590de6 100644
@@ -713,7 +713,8 @@ struct io_uring_clock_register {
 };
 
 enum {
-	IORING_REGISTER_SRC_REGISTERED = 1,
+	IORING_REGISTER_SRC_REGISTERED	= (1U << 0),
+	IORING_REGISTER_DST_REPLACE	= (1U << 1),
 };
 
 struct io_uring_clone_buffers {
io_uring/rsrc.c +20 -4
index 4c149dc42fd7..9829c51105ed 100644
@@ -990,9 +990,25 @@ static int io_clone_buffers(struct io_ring_ctx *ctx, struct io_ring_ctx *src_ctx
 	/* Have a ref on the bufs now, drop src lock and re-grab our own lock */
 	mutex_unlock(&src_ctx->uring_lock);
 	mutex_lock(&ctx->uring_lock);
-	if (!ctx->buf_table.nr) {
+
+	/*
+	 * Not replacing, or replacing an empty table. Just install the
+	 * new table.
+	 */
+	if (!(arg->flags & IORING_REGISTER_DST_REPLACE) || !ctx->buf_table.nr) {
 		ctx->buf_table = data;
 		return 0;
+	} else if (arg->flags & IORING_REGISTER_DST_REPLACE) {
+		/* put nodes in overlapping spots, if any */
+		for (i = arg->src_off; i < arg->nr; i++) {
+			if (data.nodes[i] == rsrc_empty_node)
+				continue;
+			io_reset_rsrc_node(&ctx->buf_table, i);
+			ctx->buf_table.nodes[i] = data.nodes[i];
+			data.nodes[i] = NULL;
+		}
+		io_rsrc_data_free(&data);
+		return 0;
 	}
 
 	mutex_unlock(&ctx->uring_lock);
@@ -1026,12 +1042,12 @@ int io_register_clone_buffers(struct io_ring_ctx *ctx, void __user *arg)
 	struct file *file;
 	int ret;
 
-	if (ctx->buf_table.nr)
-		return -EBUSY;
 	if (copy_from_user(&buf, arg, sizeof(buf)))
 		return -EFAULT;
-	if (buf.flags & ~IORING_REGISTER_SRC_REGISTERED)
+	if (buf.flags & ~(IORING_REGISTER_SRC_REGISTERED|IORING_REGISTER_DST_REPLACE))
 		return -EINVAL;
+	if (!(buf.flags & IORING_REGISTER_DST_REPLACE) && ctx->buf_table.nr)
+		return -EBUSY;
 	if (memchr_inv(buf.pad, 0, sizeof(buf.pad)))
 		return -EINVAL;
 
Signature · 1 line
2.45.2