regions, param pre-mapping and reg waits extension
v2 cover regions, param pre-mapping and reg waits extension
A bit late but first we need a better and more generic API for ring/memory/region registration (see Patch 4), and it changes the API extending registered waits to be a generic parameter passing mechanism. That will be useful in the future to implement a more flexible rings creation, especially when we want to share same huge page / mapping. Patch 6 uses it for registered wait arguments, and it can also be used to optimise parameter passing for normal io_uring requests.
A dirty liburing branch with tests:
https://github.com/isilence/liburing/tree/io-uring-region-test
v2: cleaned up namings and commit messages
moved all EXT_ARG_REG related bits Patch 5 -> 6 added alignment checks (Patch 6)
Pavel Begunkov (6):
io_uring: fortify io_pin_pages with a warning
io_uring: disable ENTER_EXT_ARG_REG for IOPOLL
io_uring: temporarily disable registered waits
io_uring: introduce concept of memory regions
io_uring: add memory region registration
io_uring: restore back registered wait arguments
include/linux/io_uring_types.h | 20 +++---- include/uapi/linux/io_uring.h | 28 +++++++++- io_uring/io_uring.c | 27 +++++----- io_uring/memmap.c | 67 +++++++++++++++++++++++ io_uring/memmap.h | 14 +++++ io_uring/register.c | 97 ++++++++++++---------------------- io_uring/register.h | 1 - 7 files changed, 164 insertions(+), 90 deletions(-)
Signature · 1 line
2.46.0
PATCH v2 1/6 io_uring: fortify io_pin_pages with a warning
We're a bit too frivolous with types of nr_pages arguments, converting it to long and back to int, passing an unsigned int pointer as an int pointer and so on. Shouldn't cause any problem but should be carefully reviewed, but until then let's add a WARN_ON_ONCE check to be more confident callers don't pass poorely checked arguents.
--- io_uring/memmap.c | 2 ++ 1 file changed, 2 insertions(+)
io_uring/memmap.c +2 -0
@@ -140,6 +140,8 @@ struct page **io_pin_pages(unsigned long uaddr, unsigned long len, int *npages) nr_pages = end - start;if (WARN_ON_ONCE (!nr_pages))return ERR_PTR (-EINVAL ); +if (WARN_ON_ONCE (nr_pages >INT_MAX )) +return ERR_PTR (-EOVERFLOW ); pages =kvmalloc_array (nr_pages,sizeof (struct page *),GFP_KERNEL );if (!pages)
Signature · 1 line
2.46.0
PATCH v2 2/6 io_uring: disable ENTER_EXT_ARG_REG for IOPOLL
In reply to Pavel BegunkovIOPOLL doesn't use the extended arguments, no need for it to support IORING_ENTER_EXT_ARG_REG. Let's disable it for IOPOLL, if anything it leaves more space for future extensions.
--- io_uring/io_uring.c | 8 ++------ 1 file changed, 2 insertions(+), 6 deletions(-)
io_uring/io_uring.c +2 -6
@@ -3214,12 +3214,8 @@ static int io_validate_ext_arg(struct io_ring_ctx *ctx, unsigned flags,if (!(flags &IORING_ENTER_EXT_ARG ))return 0 ; - -if (flags &IORING_ENTER_EXT_ARG_REG ) { -if (argsz !=sizeof (struct io_uring_reg_wait )) -return -EINVAL ; -return PTR_ERR (io_get_ext_arg_reg (ctx, argp)); - } +if (flags &IORING_ENTER_EXT_ARG_REG ) +return -EINVAL ;if (argsz !=sizeof (arg))return -EINVAL ;if (copy_from_user (&arg, argp,sizeof (arg)))
Signature · 1 line
2.46.0
PATCH v2 3/6 io_uring: temporarily disable registered waits
In reply to Pavel BegunkovDisable wait argument registration as it'll be replaced with a more generic feature. We'll still need IORING_ENTER_EXT_ARG_REG parsing in a few commits so leave it be.
--- include/linux/io_uring_types.h | 10 ----- include/uapi/linux/io_uring.h | 3 -- io_uring/io_uring.c | 10 ----- io_uring/register.c | 82 ---------------------------------- io_uring/register.h | 1 - 5 files changed, 106 deletions(-)
include/linux/io_uring_types.h +0 -10
@@ -330,14 +330,6 @@ struct io_ring_ctx {atomic_t cq_wait_nr;atomic_t cq_timeouts;struct wait_queue_head cq_wait; - -/* -* If registered with IORING_REGISTER_CQWAIT_REG, a single -* page holds N entries, mapped in cq_wait_arg. cq_wait_index -* is the maximum allowable index. -*/ -struct io_uring_reg_wait *cq_wait_arg; -unsigned char cq_wait_index; }____cacheline_aligned_in_smp ;/* timeouts */
@@ -431,8 +423,6 @@ struct io_ring_ctx {unsigned short n_sqe_pages;struct page **ring_pages;struct page **sqe_pages; - -struct page **cq_wait_page; };struct io_tw_state {
include/uapi/linux/io_uring.h +0 -3
@@ -627,9 +627,6 @@ enum io_uring_register_op {/* resize CQ ring */ IORING_REGISTER_RESIZE_RINGS =33 , -/* register fixed io_uring_reg_wait arguments */ -IORING_REGISTER_CQWAIT_REG =34 , -/* this goes last */ IORING_REGISTER_LAST,
io_uring/io_uring.c +0 -10
@@ -2709,7 +2709,6 @@ static __cold void io_ring_ctx_free(struct io_ring_ctx *ctx)io_alloc_cache_free (&ctx->msg_cache, io_msg_cache_free);io_futex_cache_free (ctx);io_destroy_buffers (ctx); -io_unregister_cqwait_reg (ctx);mutex_unlock (&ctx->uring_lock);if (ctx->sq_creds)put_cred (ctx->sq_creds);
@@ -3195,15 +3194,6 @@ void __io_uring_cancel(bool cancel_all)static struct io_uring_reg_wait *io_get_ext_arg_reg (struct io_ring_ctx *ctx,const struct io_uring_getevents_arg __user *uarg) { -struct io_uring_reg_wait *arg =READ_ONCE (ctx->cq_wait_arg); - -if (arg) { -unsigned int index = (unsigned int ) (uintptr_t ) uarg; - -if (index <= ctx->cq_wait_index) -return arg + index; - } -return ERR_PTR (-EFAULT ); }
io_uring/register.c +0 -82
94 lines not on this page · raw message
io_uring/register.h +0 -1
@@ -5,6 +5,5 @@int io_eventfd_unregister (struct io_ring_ctx *ctx);int io_unregister_personality (struct io_ring_ctx *ctx,unsigned id);struct file *io_uring_register_get_file (unsigned int fd,bool registered); -void io_unregister_cqwait_reg (struct io_ring_ctx *ctx);#endif
Signature · 1 line
2.46.0
PATCH v2 4/6 io_uring: introduce concept of memory regions
In reply to Pavel BegunkovWe've got a good number of mappings we share with the userspace, that includes the main rings, provided buffer rings, upcoming rings for zerocopy rx and more. All of them duplicate user argument parsing and some internal details as well (page pinnning, huge page optimisations, mmap'ing, etc.)
Introduce a notion of regions. For userspace for now it's just a new structure called struct io_uring_region_desc which is supposed to parameterise all such mapping / queue creations. A region either represents a user provided chunk of memory, in which case the user_addr field should point to it, or a request for the kernel to allocate the memory, in which case the user would need to mmap it after using the offset returned in the mmap_offset field. With a uniform userspace API we can avoid additional boiler plate code and apply future optimisation to all of them at once.
Internally, there is a new structure struct io_mapped_region holding all relevant runtime information and some helpers to work with it. This patch limits it to user provided regions.
--- include/linux/io_uring_types.h | 6 ++++ include/uapi/linux/io_uring.h | 14 ++++++++ io_uring/memmap.c | 65 ++++++++++++++++++++++++++++++++++ io_uring/memmap.h | 14 ++++++++ 4 files changed, 99 insertions(+)
include/linux/io_uring_types.h +6 -0
@@ -75,6 +75,12 @@ struct io_hash_table {unsigned hash_bits; }; +struct io_mapped_region { +struct page **pages; +void *vmap_ptr; +size_t nr_pages; +}; +/* * Arbitrary limit, can be raised if need be */
include/uapi/linux/io_uring.h +14 -0
@@ -647,6 +647,20 @@ struct io_uring_files_update {__aligned_u64 /* __s32 * */ fds; }; +enum { +/* initialise with user provided memory pointed by user_addr */ +IORING_MEM_REGION_TYPE_USER =1 , +}; + +struct io_uring_region_desc { +__u64 user_addr; +__u64 size; +__u32 flags; +__u32 id; +__u64 mmap_offset; +__u64 __resv[4 ]; +}; +/* * Register a fully sparse file space, rather than pass in an array of all * -1 file descriptors.
io_uring/memmap.c +65 -0
77 lines not on this page · raw message
io_uring/memmap.h +14 -0
@@ -22,4 +22,18 @@ unsigned long io_uring_get_unmapped_area(struct file *file, unsigned long addr, unsigned long flags);int io_uring_mmap (struct file *file,struct vm_area_struct *vma); +void io_free_region (struct io_ring_ctx *ctx,struct io_mapped_region *mr); +int io_create_region (struct io_ring_ctx *ctx,struct io_mapped_region *mr, +struct io_uring_region_desc *reg); + +static inline void *io_region_get_ptr (struct io_mapped_region *mr) +{ +return mr->vmap_ptr; +} + +static inline bool io_region_is_set (struct io_mapped_region *mr) +{ +return !!mr->nr_pages; +} +#endif
Signature · 1 line
2.46.0
PATCH v2 5/6 io_uring: add memory region registration
In reply to Pavel BegunkovRegions will serve multiple purposes. First, with it we can decouple ring/etc. object creation from registration / mapping of the memory they will be placed in. We already have hacks that allow to put both SQ and CQ into the same huge page, in the future we should be able to:
region = create_region(io_ring);create_pbuf_ring(io_uring, region, offset=0);create_pbuf_ring(io_uring, region, offset=N);
The second use case is efficiently passing parameters. The following patch enables back on top of regions IORING_ENTER_EXT_ARG_REG, which optimises wait arguments. It'll also be useful for request arguments replacing iovecs, msghdr, etc. pointers. Eventually it would also be handy for BPF as well if it comes to fruition.
--- include/linux/io_uring_types.h | 3 +++ include/uapi/linux/io_uring.h | 8 ++++++++ io_uring/io_uring.c | 1 + io_uring/register.c | 37 ++++++++++++++++++++++++++++++++++ 4 files changed, 49 insertions(+)
include/linux/io_uring_types.h +3 -0
@@ -429,6 +429,9 @@ struct io_ring_ctx {unsigned short n_sqe_pages;struct page **ring_pages;struct page **sqe_pages; + +/* used for optimised request parameter and wait argument passing */ +struct io_mapped_region param_region; };struct io_tw_state {
include/uapi/linux/io_uring.h +8 -0
@@ -627,6 +627,8 @@ enum io_uring_register_op {/* resize CQ ring */ IORING_REGISTER_RESIZE_RINGS =33 , +IORING_REGISTER_MEM_REGION =34 , +/* this goes last */ IORING_REGISTER_LAST,
@@ -661,6 +663,12 @@ struct io_uring_region_desc {__u64 __resv[4 ]; }; +struct io_uring_mem_region_reg { +__u64 region_uptr;/* struct io_uring_region_desc * */ +__u64 flags; +__u64 __resv[2 ]; +}; +/* * Register a fully sparse file space, rather than pass in an array of all * -1 file descriptors.
io_uring/io_uring.c +1 -0
@@ -2709,6 +2709,7 @@ static __cold void io_ring_ctx_free(struct io_ring_ctx *ctx)io_alloc_cache_free (&ctx->msg_cache, io_msg_cache_free);io_futex_cache_free (ctx);io_destroy_buffers (ctx); +io_free_region (ctx, &ctx->param_region);mutex_unlock (&ctx->uring_lock);if (ctx->sq_creds)put_cred (ctx->sq_creds);
io_uring/register.c +37 -0
@@ -570,6 +570,37 @@ static int io_register_resize_rings(struct io_ring_ctx *ctx, void __user *arg)return ret; } +static int io_register_mem_region (struct io_ring_ctx *ctx,void __user *uarg) +{ +struct io_uring_mem_region_reg __user *reg_uptr = uarg; +struct io_uring_mem_region_reg reg; +struct io_uring_region_desc __user *rd_uptr; +struct io_uring_region_desc rd; +int ret; + +if (io_region_is_set (&ctx->param_region)) +return -EBUSY ; +if (copy_from_user (®, reg_uptr,sizeof (reg))) +return -EFAULT ; + rd_uptr =u64_to_user_ptr (reg.region_uptr); +if (copy_from_user (&rd, rd_uptr,sizeof (rd))) +return -EFAULT ; + +if (memchr_inv (®.__resv,0 ,sizeof (reg.__resv))) +return -EINVAL ; +if (reg.flags) +return -EINVAL ; + + ret =io_create_region (ctx, &ctx->param_region, &rd); +if (ret) +return ret; +if (copy_to_user (rd_uptr, &rd,sizeof (rd))) { +io_free_region (ctx, &ctx->param_region); +return -EFAULT ; + } +return 0 ; +} + static int __io_uring_register(struct io_ring_ctx *ctx, unsigned opcode, void __user *arg, unsigned nr_args) __releases(ctx->uring_lock)
@@ -764,6 +795,12 @@ static int __io_uring_register(struct io_ring_ctx *ctx, unsigned opcode,break ; ret =io_register_resize_rings (ctx, arg);break ; +case IORING_REGISTER_MEM_REGION : + ret = -EINVAL ; +if (!arg || nr_args !=1 ) +break ; + ret =io_register_mem_region (ctx, arg); +break ;default : ret = -EINVAL ;break ;
Signature · 1 line
2.46.0
PATCH v2 6/6 io_uring: restore back registered wait arguments
In reply to Pavel BegunkovNow we've got a more generic region registration API, place IORING_ENTER_EXT_ARG_REG and re-enable it.
First, the user has to register a region with the IORING_MEM_REGION_REG_WAIT_ARG flag set. It can only be done for a ring in a disabled state, aka IORING_SETUP_R_DISABLED, to avoid races with already running waiters.
The other API difference is that we're now passing byte offsets instead of indexes. The user _must_ align all offsets / pointers to the native word size, failing to do so might but not necessarily has to lead to a failure usually returned as -EFAULT. liburing will be hiding this details from users.
--- include/linux/io_uring_types.h | 3 +++ include/uapi/linux/io_uring.h | 5 +++++ io_uring/io_uring.c | 14 +++++++++++++- io_uring/register.c | 16 +++++++++++++++- 4 files changed, 36 insertions(+), 2 deletions(-)
include/linux/io_uring_types.h +3 -0
@@ -324,6 +324,9 @@ struct io_ring_ctx {unsigned cq_entries;struct io_ev_fd __rcu *io_ev_fd;unsigned cq_extra; + +void *cq_wait_arg; +size_t cq_wait_size; }____cacheline_aligned_in_smp ;/*
include/uapi/linux/io_uring.h +5 -0
@@ -663,6 +663,11 @@ struct io_uring_region_desc {__u64 __resv[4 ]; }; +enum { +/* expose the region as registered wait arguments */ +IORING_MEM_REGION_REG_WAIT_ARG =1 , +}; +struct io_uring_mem_region_reg {__u64 region_uptr;/* struct io_uring_region_desc * */ __u64 flags;
io_uring/io_uring.c +13 -1
@@ -3195,7 +3195,19 @@ void __io_uring_cancel(bool cancel_all)static struct io_uring_reg_wait *io_get_ext_arg_reg (struct io_ring_ctx *ctx,const struct io_uring_getevents_arg __user *uarg) { -return ERR_PTR (-EFAULT ); +unsigned long size =sizeof (struct io_uring_reg_wait ); +unsigned long offset = (uintptr_t )uarg; +unsigned long end; + +if (unlikely (offset %sizeof (long ))) +return ERR_PTR (-EFAULT ); + +/* also protects from NULL ->cq_wait_arg as the size would be 0 */ +if (unlikely (check_add_overflow (offset, size, &end) || + end >= ctx->cq_wait_size)) +return ERR_PTR (-EFAULT ); + +return ctx->cq_wait_arg + offset; } static int io_validate_ext_arg(struct io_ring_ctx *ctx, unsigned flags,
io_uring/register.c +15 -1
@@ -588,7 +588,16 @@ static int io_register_mem_region(struct io_ring_ctx *ctx, void __user *uarg)if (memchr_inv (®.__resv,0 ,sizeof (reg.__resv)))return -EINVAL ; -if (reg.flags) +if (reg.flags & ~IORING_MEM_REGION_REG_WAIT_ARG ) +return -EINVAL ; + +/* +* This ensures there are no waiters. Waiters are unlocked and it's +* hard to synchronise with them, especially if we need to initialise +* the region. +*/ +if ((reg.flags &IORING_MEM_REGION_REG_WAIT_ARG ) && + !(ctx->flags &IORING_SETUP_R_DISABLED ))return -EINVAL ; ret =io_create_region (ctx, &ctx->param_region, &rd);
@@ -598,6 +607,11 @@ static int io_register_mem_region(struct io_ring_ctx *ctx, void __user *uarg)io_free_region (ctx, &ctx->param_region);return -EFAULT ; } + +if (reg.flags &IORING_MEM_REGION_REG_WAIT_ARG ) { + ctx->cq_wait_arg =io_region_get_ptr (&ctx->param_region); + ctx->cq_wait_size = rd.size; + }return 0 ; }
Signature · 1 line
2.46.0