io_uring/register: add IORING_REGISTER_CLONE_FILES opcode
PATCH v2 1/1 io_uring/register: add IORING_REGISTER_CLONE_FILES opcode
Currently, if an application wants to duplicate registered file descriptors from one io_uring instance to another, it must manually unregister and re-register them, incurring unnecessary overhead.
Add IORING_REGISTER_CLONE_FILES to allow direct cloning of the file table from a source ring to a destination ring. This includes support for partial offsets and the IORING_REGISTER_DST_REPLACE flag.
---
v2: Dropped unrelated whitespace formatting changes from v1
--- include/uapi/linux/io_uring.h | 12 +++ io_uring/register.c | 6 ++ io_uring/rsrc.c | 160 ++++++++++++++++++++++++++++++++++ io_uring/rsrc.h | 1 + 4 files changed, 179 insertions(+)
include/uapi/linux/io_uring.h +12 -0
@@ -723,6 +723,9 @@ enum io_uring_register_op {/* register bpf filtering programs */ IORING_REGISTER_BPF_FILTER =37 , +/* clone file descriptors from another ring*/ +IORING_REGISTER_CLONE_FILES =38 , +/* this goes last */ IORING_REGISTER_LAST,
@@ -854,6 +857,15 @@ struct io_uring_clone_buffers {__u32 pad[3 ]; }; +struct io_uring_clone_files { +__u32 src_fd; +__u32 flags; +__u32 src_off; +__u32 dst_off; +__u32 nr; +__u32 pad[3 ]; +}; +struct io_uring_buf {__u64 addr;__u32 len;
io_uring/register.c +6 -0
@@ -924,6 +924,12 @@ static int __io_uring_register(struct io_ring_ctx *ctx, unsigned opcode,break ; ret =io_register_clone_buffers (ctx, arg);break ; +case IORING_REGISTER_CLONE_FILES : + ret = -EINVAL ; +if (!arg || nr_args !=1 ) +break ; + ret =io_register_clone_files (ctx, arg); +break ;case IORING_REGISTER_ZCRX_IFQ : ret = -EINVAL ;if (!arg || nr_args !=1 )
io_uring/rsrc.c +160 -0
166 lines not on this page · raw message
io_uring/rsrc.h +1 -0
@@ -75,6 +75,7 @@ int io_prep_reg_iovec(struct io_kiocb *req, struct iou_vec *iv, const struct iovec __user *uvec, size_t uvec_segs);int io_register_clone_buffers (struct io_ring_ctx *ctx,void __user *arg); +int io_register_clone_files (struct io_ring_ctx *ctx,void __user *arg);int io_sqe_buffers_unregister (struct io_ring_ctx *ctx);int io_sqe_buffers_register (struct io_ring_ctx *ctx,void __user *arg,unsigned int nr_args,u64 __user *tags);
Signature · 1 line
2.54.0
Jens Axboe
maintainerQuoted 12 lines · On 6/17/26 2:16 AM, harshal24-chavan wrote:
io_uring/rsrc.c +4
-0@@ -1303,6 +1303,166 @@ int io_register_clone_buffers(struct io_ring_ctx *ctx, void __user *arg)return ret; } + +static int io_clone_files (struct io_ring_ctx *ctx,struct io_ring_ctx *src_ctx, +struct io_uring_clone_files *arg) +{
io_uring/rsrc.c int io_register_clone_buffers(struct io_ring_ctx *ctx, void __user *arg) · line 1319
Quoted 10 lines
io_uring/rsrc.c +10
-0+struct io_file_table new_file_table; +int i, off, nr; +unsigned int src_nr; + +lockdep_assert_held (&ctx->uring_lock); +lockdep_assert_held (&src_ctx->uring_lock); + +/* if offsets are given, must have nr specified too */ +if (!arg->nr && (arg->dst_off || arg->src_off)) +return -EINVAL ;
Not sure the offsets and partial copies are going to be worth it, but I'm willing to have my mind changed. But that's a minor thing really.
Quoted 15 lines
io_uring/rsrc.c +15
-0+/* not allowed unless REPLACE is set */ +if (ctx->file_table.data.nr && + !(arg->flags &IORING_REGISTER_DST_REPLACE )) +return -EBUSY ; + + src_nr = src_ctx->file_table.data.nr; +if (!src_nr) +return -ENXIO ; +if (!arg->nr) + arg->nr = src_nr; +else if (arg->nr > src_nr) +return -EINVAL ; +else if (arg->nr >IORING_MAX_FIXED_FILES ) +return -EINVAL ; +if (check_add_overflow (arg->nr, arg->src_off, &off) || off > src_nr)
io_uring/rsrc.c int io_register_clone_buffers(struct io_ring_ctx *ctx, void __user *arg) · line 1344
Quoted 10 lines
io_uring/rsrc.c +10
-0+return -EOVERFLOW ; +if (check_add_overflow (arg->nr, arg->dst_off, &src_nr)) +return -EOVERFLOW ; +if (src_nr >IORING_MAX_FIXED_FILES ) +return -EINVAL ; +/* Allocate file tables memory {data + bitmap} into new_file_table */ +memset (&new_file_table,0 ,sizeof (new_file_table)); +if (!io_alloc_file_tables (ctx, &new_file_table, +max (src_nr, ctx->file_table.data.nr))) +return -ENOMEM ;
Also a question whether the destination should've already allocated a sparse table. This kind of bundles the two into one. In general, as mention on the GH link, I do think this should work exactly like cloning buffers. It'd be somewhat confusing if they don't match up, as it's essentially the same operation, just on a different node type.
io_uring/rsrc.c int io_register_clone_buffers(struct io_ring_ctx *ctx, void __user *arg) · line 1355
Quoted 10 lines
io_uring/rsrc.c +10
-0+/* Copy original dst nodes from before the cloned range */ +for (i =0 ; i <min (arg->dst_off, ctx->file_table.data.nr); i++) { +struct io_rsrc_node *node = ctx->file_table.data.nodes[i]; + +if (node) { + new_file_table.data.nodes[i] = node; + node->refs++; +io_file_bitmap_set (&new_file_table, i); + } + }
This definitely won't work - I also mentioned in the GH link that nodes cannot be shared, you have to allocate new nodes on the destination side.
Quoted 15 lines
io_uring/rsrc.c +15
-0+while (nr--) { +struct io_rsrc_node *dst_node, *src_node; + + src_node =io_rsrc_node_lookup (&src_ctx->file_table.data, i); +if (!src_node) { + dst_node =NULL ; + }else { + dst_node =io_rsrc_node_alloc (ctx,IORING_RSRC_FILE ); +if (!dst_node) { +io_free_file_tables (ctx, &new_file_table); +return -ENOMEM ; + } + +struct file *file =io_slot_file (src_node); +
io_uring/rsrc.c int io_register_clone_buffers(struct io_ring_ctx *ctx, void __user *arg) · line 1384
Quoted 10 lines
io_uring/rsrc.c +10
-0+get_file (file); +io_fixed_file_set (dst_node, file); + } + new_file_table.data.nodes[off] = dst_node; +if (dst_node) +io_file_bitmap_set (&new_file_table, off); + + i++; + off++; + }
Same here, it needs a new node that's private to the destination. Hence you'd need to _always_ allocate one, assign the file, and get a reference to it.
The file nodes rely on non-atomic refs when being used, which is protected by the ctx->uring_lock as that's always held for the fast path issue. If you just assign the node by reference, now you have two different rings manipulating the same node in memory, but they don't agree on synchronization.
Signature · 1 line
Jens Axboe
Quoted 25 lines · Jens Axboe
Not sure the offsets and partial copies are going to be worth it, but I'm willing to have my mind changed. But that's a minor thing really.
Also a question whether the destination should've already allocated a sparse table. This kind of bundles the two into one. In general, as mention on the GH link, I do think this should work exactly like cloning buffers. It'd be somewhat confusing if they don't match up, as it's essentially the same operation, just on a different node type.
io_uring/rsrc.c int io_register_clone_buffers(struct io_ring_ctx *ctx, void __user *arg) · line 1355
Quoted 10 lines
io_uring/rsrc.c +10
-0+/* Copy original dst nodes from before the cloned range */ +for (i =0 ; i <min (arg->dst_off, ctx->file_table.data.nr); i++) { +struct io_rsrc_node *node = ctx->file_table.data.nodes[i]; + +if (node) { + new_file_table.data.nodes[i] = node; + node->refs++; +io_file_bitmap_set (&new_file_table, i); + } + }This definitely won't work - I also mentioned in the GH link that nodes cannot be shared, you have to allocate new nodes on the destination side.
Quoted 5 lines
The file nodes rely on non-atomic refs when being used, which is protected by the
ctx->uring_lockas that's always held for the fast path issue. If you just assign the node by reference, now you have two different rings manipulating the same node in memory, but they don't agree on synchronization.
Thanks for the detailed explanation.
In v3, I have rewritten the cloning loops to utilize a new helper function. It strictly calls io_rsrc_node_alloc(), get_file(), and io_fixed_file_set() for every single transferred node. We no longer share references across rings; the destination ring is fully populated with its own newly allocated, private nodes to avoid any non-atomic reference races.
Quoted 161 lines · harshal24-chavan <[email protected]> writes:
Currently, if an application wants to duplicate registered file descriptors from one
io_uringinstance to another, it must manually unregister and re-register them, incurring unnecessary overhead.Add
IORING_REGISTER_CLONE_FILESto allow direct cloning of the file table from a source ring to a destination ring. This includes support for partial offsets and theIORING_REGISTER_DST_REPLACEflag.Signed-off-by harshal24-chavan <[email protected]>---
v2: Dropped unrelated whitespace formatting changes from v1--- include/uapi/linux/io_uring.h | 12 +++ io_uring/register.c | 6 ++ io_uring/rsrc.c | 160 ++++++++++++++++++++++++++++++++++ io_uring/rsrc.h | 1 + 4 files changed, 179 insertions(+)include/uapi/linux/io_uring.h +12
-0@@ -723,6 +723,9 @@ enum io_uring_register_op {/* register bpf filtering programs */ IORING_REGISTER_BPF_FILTER =37 , +/* clone file descriptors from another ring*/ +IORING_REGISTER_CLONE_FILES =38 , +/* this goes last */ IORING_REGISTER_LAST,@@ -854,6 +857,15 @@ struct io_uring_clone_buffers {__u32 pad[3 ]; }; +struct io_uring_clone_files { +__u32 src_fd; +__u32 flags; +__u32 src_off; +__u32 dst_off; +__u32 nr; +__u32 pad[3 ]; +}; +struct io_uring_buf {__u64 addr;__u32 len;io_uring/register.c +6
-0@@ -924,6 +924,12 @@ static int __io_uring_register(struct io_ring_ctx *ctx, unsigned opcode,break ; ret =io_register_clone_buffers (ctx, arg);break ; +case IORING_REGISTER_CLONE_FILES : + ret = -EINVAL ; +if (!arg || nr_args !=1 ) +break ; + ret =io_register_clone_files (ctx, arg); +break ;case IORING_REGISTER_ZCRX_IFQ : ret = -EINVAL ;if (!arg || nr_args !=1 )io_uring/rsrc.c +87
-0@@ -1303,6 +1303,166 @@ int io_register_clone_buffers(struct io_ring_ctx *ctx, void __user *arg)return ret; } + +static int io_clone_files (struct io_ring_ctx *ctx,struct io_ring_ctx *src_ctx, +struct io_uring_clone_files *arg) +{ +struct io_file_table new_file_table; +int i, off, nr; +unsigned int src_nr; + +lockdep_assert_held (&ctx->uring_lock); +lockdep_assert_held (&src_ctx->uring_lock); + +/* if offsets are given, must have nr specified too */ +if (!arg->nr && (arg->dst_off || arg->src_off)) +return -EINVAL ; +/* not allowed unless REPLACE is set */ +if (ctx->file_table.data.nr && + !(arg->flags &IORING_REGISTER_DST_REPLACE )) +return -EBUSY ; + + src_nr = src_ctx->file_table.data.nr; +if (!src_nr) +return -ENXIO ; +if (!arg->nr) + arg->nr = src_nr; +else if (arg->nr > src_nr) +return -EINVAL ; +else if (arg->nr >IORING_MAX_FIXED_FILES ) +return -EINVAL ; +if (check_add_overflow (arg->nr, arg->src_off, &off) || off > src_nr) +return -EOVERFLOW ; +if (check_add_overflow (arg->nr, arg->dst_off, &src_nr)) +return -EOVERFLOW ; +if (src_nr >IORING_MAX_FIXED_FILES ) +return -EINVAL ; +/* Allocate file tables memory {data + bitmap} into new_file_table */ +memset (&new_file_table,0 ,sizeof (new_file_table)); +if (!io_alloc_file_tables (ctx, &new_file_table, +max (src_nr, ctx->file_table.data.nr))) +return -ENOMEM ; + +/* Copy original dst nodes from before the cloned range */ +for (i =0 ; i <min (arg->dst_off, ctx->file_table.data.nr); i++) { +struct io_rsrc_node *node = ctx->file_table.data.nodes[i]; + +if (node) { + new_file_table.data.nodes[i] = node; + node->refs++; +io_file_bitmap_set (&new_file_table, i); + } + } + + off = arg->dst_off; + i = arg->src_off; + nr = arg->nr; +while (nr--) { +struct io_rsrc_node *dst_node, *src_node; + + src_node =io_rsrc_node_lookup (&src_ctx->file_table.data, i); +if (!src_node) { + dst_node =NULL ; + }else { + dst_node =io_rsrc_node_alloc (ctx,IORING_RSRC_FILE ); +if (!dst_node) { +io_free_file_tables (ctx, &new_file_table); +return -ENOMEM ; + } + +struct file *file =io_slot_file (src_node); + +get_file (file); +io_fixed_file_set (dst_node, file); + } + new_file_table.data.nodes[off] = dst_node; +if (dst_node) +io_file_bitmap_set (&new_file_table, off); + + i++; + off++; + } + +/* Copy original dst nodes from after the cloned range */ +for (i = src_nr; i < ctx->file_table.data.nr; i++) { +struct io_rsrc_node *node = ctx->file_table.data.nodes[i]; + +if (node) { + new_file_table.data.nodes[i] = node; + node->refs++;
io_uring/rsrc.c int io_register_clone_buffers(struct io_ring_ctx *ctx, void __user *arg) · line 1402
Quoted 10 lines
io_uring/rsrc.c +10
-0+io_file_bitmap_set (&new_file_table, i); + } + } + +/* +* If asked for replace, put the old table. new_file_table.data->nodes[] holds both +* old and new nodes at this point. +*/ +if (arg->flags &IORING_REGISTER_DST_REPLACE ) +io_free_file_tables (ctx, &ctx->file_table);
IIUC, the IORING_REGISTER_DST_REPLACE exists for backward compatibility, since originally the buffer cloning would fail if existing elements were already there. It is kind of superflous in a new operation but I suppose it is here to mirror the semantics of io_clone_buffers, which is ok, but then...
This free should at least be gated on ctx->file_table->data.nr. We are always replacing the ->file_table if it was initialized, so it is a bit more logical to check the table directly.
Quoted 20 lines
io_uring/rsrc.c +20
-0+ +/* +* ctx->file_table must be empty now - either the contents are being +* replaced and we just freed the table, or the contents are being +* copied to a ring that does not have buffers yet (checked at function +* entry). +*/ +WARN_ON_ONCE (ctx->file_table.data.nr); + ctx->file_table = new_file_table; +io_file_table_set_alloc_range (ctx,0 , ctx->file_table.data.nr); +return 0 ; +} + +int io_register_clone_files (struct io_ring_ctx *ctx,void __user *arg) +{ +struct io_uring_clone_files clone_arg; +struct io_ring_ctx *src_ctx; +bool registered_src; +struct file *file; +int ret;
io_uring/rsrc.c int io_register_clone_buffers(struct io_ring_ctx *ctx, void __user *arg) · line 1432
Quoted 10 lines
io_uring/rsrc.c +10
-0+ +if (copy_from_user (&clone_arg, arg,sizeof (clone_arg))) +return -EFAULT ; +if (clone_arg.flags & + ~(IORING_REGISTER_SRC_REGISTERED |IORING_REGISTER_DST_REPLACE )) +return -EINVAL ; +/* not allowed unless REPLACE is set */ +if (!(clone_arg.flags &IORING_REGISTER_DST_REPLACE ) && + ctx->file_table.data.nr) +return -EBUSY ;
This check is duplicated in io_clone_files.
io_uring/rsrc.c int io_register_clone_buffers(struct io_ring_ctx *ctx, void __user *arg) · line 1443
Quoted 11 lines
io_uring/rsrc.c +11
-0+if (memchr_inv (clone_arg.pad,0 ,sizeof (clone_arg.pad))) +return -EINVAL ; + + registered_src = (clone_arg.flags &IORING_REGISTER_SRC_REGISTERED ) != +0 ; + file =io_uring_ctx_get_file (clone_arg.src_fd, registered_src); +if (IS_ERR (file)) +return PTR_ERR (file); + + src_ctx = file->private_data; +if (src_ctx != ctx) {
Shouldn't we just fail if ctx == src_ctx ?
io_uring/rsrc.c int io_register_clone_buffers(struct io_ring_ctx *ctx, void __user *arg) · line 1451
Quoted 8 lines
io_uring/rsrc.c +8
-0+mutex_unlock (&ctx->uring_lock); +lock_two_rings (ctx, src_ctx); + +/* Prevent cross-process hijacking */ +if (src_ctx->submitter_task && + src_ctx->submitter_task != current) { + ret = -EEXIST ; + goto out;
Is limiting the feature to the submitter_task necessary to safely copy the table even if the lock is held? The use-case for this feature would be setting up a single ring with its file table and then replicating it on other threads, on the common model of one-ring per thread. This check limits it.
There's no cross-process hijacking, IIUC, because io_uring_ctx_get_file sees rings belonging to the process. But a thread could make use of this feature to clone the process_table of another thread which is useful.
Quoted 4 lines
io_uring/rsrc.c +4
-0+ + out: +if (src_ctx != ctx) +mutex_unlock (&src_ctx->uring_lock);
Signature · 1 line
Gabriel Krisman Bertazi
Quoted 9 lines · Gabriel Krisman Bertazi
IIUC, the
IORING_REGISTER_DST_REPLACEexists for backward compatibility, since originally the buffer cloning would fail if existing elements were already there. It is kind of superflous in a new operation but I suppose it is here to mirror the semantics ofio_clone_buffers, which is ok, but then...This free should at least be gated on
ctx->file_table->data.nr. We are always replacing the ->file_table if it was initialized, so it is a bit more logical to check the table directly.
Agreed. In v3, I have gated the io_free_file_tables() call directly on ctx->file_table.data.nr as suggested.
Quoted 5 lines
io_uring/rsrc.c int io_register_clone_buffers(struct io_ring_ctx *ctx, void __user *arg) · line 1432
Quoted 4 lines
io_uring/rsrc.c +4
-0+/* not allowed unless REPLACE is set */ +if (!(clone_arg.flags &IORING_REGISTER_DST_REPLACE ) && + ctx->file_table.data.nr) +return -EBUSY ;This check is duplicated in
io_clone_files.
I have removed this duplicate check from the parent function in v3.
Quoted 3 lines
io_uring/rsrc.c int io_register_clone_buffers(struct io_ring_ctx *ctx, void __user *arg) · line 1443
Quoted 2 lines
io_uring/rsrc.c +2
-0+ src_ctx = file->private_data; +if (src_ctx != ctx) {Shouldn't we just fail if ctx ==
src_ctx?
Yes, I have updated this to explicitly fail with -EINVAL
if ctx == src_ctx to prevent self-cloning in v3. Thanks for catching this.
Quoted 10 lines
io_uring/rsrc.c int io_register_clone_buffers(struct io_ring_ctx *ctx, void __user *arg) · line 1451
Quoted 5 lines
io_uring/rsrc.c +5
-0+/* Prevent cross-process hijacking */ +if (src_ctx->submitter_task && + src_ctx->submitter_task != current) { + ret = -EEXIST ; + goto out;Is limiting the feature to the
submitter_tasknecessary to safely copy the table even if the lock is held? The use-case for this feature would be setting up a single ring with its file table and then replicating it on other threads, on the common model of one-ring per thread. This check limits it.
I have entirely removed the submitter_task check in v3.