regions, param pre-mapping and reg waits extension
cover regions, param pre-mapping and reg waits extension
A bit late but first we need a better and more generic API for ring/memory/region registration (see Patch 4), and it changes the API extending registered waits to be a generic parameter passing mechanism. That will be useful not only for waiting but also request arguments (msghdr, iovec, etc), upcomig rw with meta attrobitutes (PI), and for BPF proposal as well.
I covered region registration with tests, but for reg waits it only enables the basic test. Need to enable and run the rest of them before merged. Dirty branch:
https://github.com/isilence/liburing/tree/io-uring-region-test
Pavel Begunkov (6):
io_uring: fortify io_pin_pages with a warning
io_uring: disable ENTER_EXT_ARG_REG for IOPOLL
io_uring: temporarily disable registered waits
io_uring: introduce memory regions
io_uring: add parameter region registration
io_uring: enable IORING_ENTER_EXT_ARG_REG back
include/linux/io_uring_types.h | 20 ++++---- include/uapi/linux/io_uring.h | 27 ++++++++++- io_uring/io_uring.c | 26 +++++----- io_uring/memmap.c | 67 +++++++++++++++++++++++++ io_uring/memmap.h | 14 ++++++ io_uring/register.c | 89 +++++++++++++--------------------- io_uring/register.h | 1 - 7 files changed, 161 insertions(+), 83 deletions(-)
Signature · 1 line
2.46.0
PATCH 1/6 io_uring: fortify io_pin_pages with a warning
We're a bit too frivolous with types of nr_pages arguments, converting it to long and back to int, passing an unsigned int pointer as an int pointer and so on. Shouldn't cause any problem but should be carefully reviewed, but until then let's add a WARN_ON_ONCE check to be more confident callers don't pass poorely checked arguents.
--- io_uring/memmap.c | 2 ++ 1 file changed, 2 insertions(+)
io_uring/memmap.c +2 -0
@@ -140,6 +140,8 @@ struct page **io_pin_pages(unsigned long uaddr, unsigned long len, int *npages) nr_pages = end - start;if (WARN_ON_ONCE (!nr_pages))return ERR_PTR (-EINVAL ); +if (WARN_ON_ONCE (nr_pages >INT_MAX )) +return ERR_PTR (-EOVERFLOW ); pages =kvmalloc_array (nr_pages,sizeof (struct page *),GFP_KERNEL );if (!pages)
Signature · 1 line
2.46.0
PATCH 2/6 io_uring: disable ENTER_EXT_ARG_REG for IOPOLL
In reply to Pavel BegunkovIOPOLL doesn't use the extended arguments, no need for it to support IORING_ENTER_EXT_ARG_REG. Let's disable it for IOPOLL, if anything it leaves more space for future extensions.
--- io_uring/io_uring.c | 8 ++------ 1 file changed, 2 insertions(+), 6 deletions(-)
io_uring/io_uring.c +2 -6
@@ -3214,12 +3214,8 @@ static int io_validate_ext_arg(struct io_ring_ctx *ctx, unsigned flags,if (!(flags &IORING_ENTER_EXT_ARG ))return 0 ; - -if (flags &IORING_ENTER_EXT_ARG_REG ) { -if (argsz !=sizeof (struct io_uring_reg_wait )) -return -EINVAL ; -return PTR_ERR (io_get_ext_arg_reg (ctx, argp)); - } +if (flags &IORING_ENTER_EXT_ARG_REG ) +return -EINVAL ;if (argsz !=sizeof (arg))return -EINVAL ;if (copy_from_user (&arg, argp,sizeof (arg)))
Signature · 1 line
2.46.0
PATCH 3/6 io_uring: temporarily disable registered waits
In reply to Pavel BegunkovDisable wait argument registration as it'll be replaced with a more generic feature. We'll still need IORING_ENTER_EXT_ARG_REG parsing in a few commits so leave it be.
--- include/linux/io_uring_types.h | 10 ----- include/uapi/linux/io_uring.h | 3 -- io_uring/io_uring.c | 10 ----- io_uring/register.c | 82 ---------------------------------- io_uring/register.h | 1 - 5 files changed, 106 deletions(-)
include/linux/io_uring_types.h +0 -10
@@ -330,14 +330,6 @@ struct io_ring_ctx {atomic_t cq_wait_nr;atomic_t cq_timeouts;struct wait_queue_head cq_wait; - -/* -* If registered with IORING_REGISTER_CQWAIT_REG, a single -* page holds N entries, mapped in cq_wait_arg. cq_wait_index -* is the maximum allowable index. -*/ -struct io_uring_reg_wait *cq_wait_arg; -unsigned char cq_wait_index; }____cacheline_aligned_in_smp ;/* timeouts */
@@ -431,8 +423,6 @@ struct io_ring_ctx {unsigned short n_sqe_pages;struct page **ring_pages;struct page **sqe_pages; - -struct page **cq_wait_page; };struct io_tw_state {
include/uapi/linux/io_uring.h +0 -3
@@ -627,9 +627,6 @@ enum io_uring_register_op {/* resize CQ ring */ IORING_REGISTER_RESIZE_RINGS =33 , -/* register fixed io_uring_reg_wait arguments */ -IORING_REGISTER_CQWAIT_REG =34 , -/* this goes last */ IORING_REGISTER_LAST,
io_uring/io_uring.c +0 -10
@@ -2709,7 +2709,6 @@ static __cold void io_ring_ctx_free(struct io_ring_ctx *ctx)io_alloc_cache_free (&ctx->msg_cache, io_msg_cache_free);io_futex_cache_free (ctx);io_destroy_buffers (ctx); -io_unregister_cqwait_reg (ctx);mutex_unlock (&ctx->uring_lock);if (ctx->sq_creds)put_cred (ctx->sq_creds);
@@ -3195,15 +3194,6 @@ void __io_uring_cancel(bool cancel_all)static struct io_uring_reg_wait *io_get_ext_arg_reg (struct io_ring_ctx *ctx,const struct io_uring_getevents_arg __user *uarg) { -struct io_uring_reg_wait *arg =READ_ONCE (ctx->cq_wait_arg); - -if (arg) { -unsigned int index = (unsigned int ) (uintptr_t ) uarg; - -if (index <= ctx->cq_wait_index) -return arg + index; - } -return ERR_PTR (-EFAULT ); }
io_uring/register.c +0 -82
94 lines not on this page · raw message
io_uring/register.h +0 -1
@@ -5,6 +5,5 @@int io_eventfd_unregister (struct io_ring_ctx *ctx);int io_unregister_personality (struct io_ring_ctx *ctx,unsigned id);struct file *io_uring_register_get_file (unsigned int fd,bool registered); -void io_unregister_cqwait_reg (struct io_ring_ctx *ctx);#endif
Signature · 1 line
2.46.0
PATCH 4/6 io_uring: introduce memory regions
In reply to Pavel BegunkovWe've got a good number of mappings we share with the userspace, that includes the main rings, provided buffer rings and at least a couple more types. And all of them duplicate some of the code for page pinning, mmap'ing and attempts to optimise it with huge pages.
Introduce a notion of regions. For userspace it's just a new structure called struct io_uring_region_desc which supposed to parameterise all such mapping / queues creations. It either represents a user provided memory, in which case the user_addr field should point to it, or a request to the kernel to creating the memory, in which case the user is supposed to mmap it after using the offset returned in the mmap_offset field. With uniform userspace API we can avoid additional boiler plate code and when we'd be adding some optimisation it'll be applied to all mapping types.
Internally, there is a new structure struct io_mapped_region holding all relevant runtime information and some helpers to work with it. This patch limits it to user provided regions, which will be extended as a follow up work.
--- include/linux/io_uring_types.h | 6 ++++ include/uapi/linux/io_uring.h | 13 +++++++ io_uring/memmap.c | 65 ++++++++++++++++++++++++++++++++++ io_uring/memmap.h | 14 ++++++++ 4 files changed, 98 insertions(+)
include/linux/io_uring_types.h +6 -0
@@ -75,6 +75,12 @@ struct io_hash_table {unsigned hash_bits; }; +struct io_mapped_region { +struct page **pages; +void *vmap_ptr; +size_t nr_pages; +}; +/* * Arbitrary limit, can be raised if need be */
include/uapi/linux/io_uring.h +13 -0
@@ -647,6 +647,19 @@ struct io_uring_files_update {__aligned_u64 /* __s32 * */ fds; }; +enum { +/* initialise with user memory pointed by user_addr */ +IORING_REGION_USER_MEM =1 , +}; + +struct io_uring_region_desc { +__u64 user_addr; +__u64 size; +__u64 flags; +__u64 mmap_offset; +__u64 __resv[4 ]; +}; +/* * Register a fully sparse file space, rather than pass in an array of all * -1 file descriptors.
io_uring/memmap.c +65 -0
77 lines not on this page · raw message
io_uring/memmap.h +14 -0
@@ -22,4 +22,18 @@ unsigned long io_uring_get_unmapped_area(struct file *file, unsigned long addr, unsigned long flags);int io_uring_mmap (struct file *file,struct vm_area_struct *vma); +void io_free_region (struct io_ring_ctx *ctx,struct io_mapped_region *mr); +int io_create_region (struct io_ring_ctx *ctx,struct io_mapped_region *mr, +struct io_uring_region_desc *reg); + +static inline void *io_region_get_ptr (struct io_mapped_region *mr) +{ +return mr->vmap_ptr; +} + +static inline bool io_region_is_set (struct io_mapped_region *mr) +{ +return !!mr->nr_pages; +} +#endif
Signature · 1 line
2.46.0
PATCH 5/6 io_uring: add parameter region registration
In reply to Pavel BegunkovAllow the user to pre-register a region for passing various paramteres.
To use it for passing wait loop arguments, which is wired in the following commit, the region has to be registered with the IORING_PARAM_REGION_WAIT_ARG flag set. The flag also requires the context to be currently disabled, i.e. IORING_SETUP_R_DISABLED, to avoid races with otherwise potentially running waiters.
This will also be useful in the future for various request / SQE arguments like iovec, the meta read/write API, and also for BPF.
--- include/linux/io_uring_types.h | 6 ++++ include/uapi/linux/io_uring.h | 13 ++++++++ io_uring/io_uring.c | 1 + io_uring/register.c | 59 ++++++++++++++++++++++++++++++++++ 4 files changed, 79 insertions(+)
include/linux/io_uring_types.h +6 -0
@@ -324,6 +324,9 @@ struct io_ring_ctx {unsigned cq_entries;struct io_ev_fd __rcu *io_ev_fd;unsigned cq_extra; + +void *cq_wait_arg; +size_t cq_wait_size; }____cacheline_aligned_in_smp ;/*
@@ -429,6 +432,9 @@ struct io_ring_ctx {unsigned short n_sqe_pages;struct page **ring_pages;struct page **sqe_pages; + +/* used for optimised request parameter and wait argument passing */ +struct io_mapped_region param_region; };struct io_tw_state {
include/uapi/linux/io_uring.h +13 -0
@@ -627,6 +627,8 @@ enum io_uring_register_op {/* resize CQ ring */ IORING_REGISTER_RESIZE_RINGS =33 , +IORING_REGISTER_PARAM_REGION =34 , +/* this goes last */ IORING_REGISTER_LAST,
@@ -660,6 +662,17 @@ struct io_uring_region_desc {__u64 __resv[4 ]; }; +enum { +/* expose the region as registered wait arguments */ +IORING_PARAM_REGION_WAIT_ARG =1 , +}; + +struct io_uring_param_region_reg { +__u64 region_uptr;/* struct io_uring_region_desc * */ +__u64 flags; +__u64 __resv[2 ]; +}; +/* * Register a fully sparse file space, rather than pass in an array of all * -1 file descriptors.
io_uring/io_uring.c +1 -0
@@ -2709,6 +2709,7 @@ static __cold void io_ring_ctx_free(struct io_ring_ctx *ctx)io_alloc_cache_free (&ctx->msg_cache, io_msg_cache_free);io_futex_cache_free (ctx);io_destroy_buffers (ctx); +io_free_region (ctx, &ctx->param_region);mutex_unlock (&ctx->uring_lock);if (ctx->sq_creds)put_cred (ctx->sq_creds);
io_uring/register.c +59 -0
71 lines not on this page · raw message
Signature · 1 line
2.46.0
PATCH 6/6 io_uring: enable IORING_ENTER_EXT_ARG_REG back
In reply to Pavel BegunkovWe can reenable IORING_ENTER_EXT_ARG_REG, which is now backed by parameter regions.
--- io_uring/io_uring.c | 11 ++++++++++- 1 file changed, 10 insertions(+), 1 deletion(-)
io_uring/io_uring.c +10 -1
@@ -3195,7 +3195,16 @@ void __io_uring_cancel(bool cancel_all)static struct io_uring_reg_wait *io_get_ext_arg_reg (struct io_ring_ctx *ctx,const struct io_uring_getevents_arg __user *uarg) { -return ERR_PTR (-EFAULT ); +unsigned long size =sizeof (struct io_uring_reg_wait ); +unsigned long offset = (uintptr_t )uarg; +unsigned long end; + +/* also protects from NULL ->cq_wait_arg as the size would be 0 */ +if (unlikely (check_add_overflow (offset, size, &end) || + end >= ctx->cq_wait_size)) +return ERR_PTR (-EFAULT ); + +return ctx->cq_wait_arg + offset; } static int io_validate_ext_arg(struct io_ring_ctx *ctx, unsigned flags,
Signature · 1 line
2.46.0
Jens Axboe
maintainerRe: [PATCH 4/6] io_uring: introduce memory regions
In reply to Pavel Begunkovio_uring/memmap.c void *__io_uaddr_map(struct page ***pages, unsigned short *npages, · line 208
Quoted 11 lines · Pavel Begunkov
io_uring/memmap.c +11
-0+void io_free_region (struct io_ring_ctx *ctx,struct io_mapped_region *mr) +{ +if (mr->pages) +unpin_user_pages (mr->pages, mr->nr_pages); +if (mr->vmap_ptr) +vunmap (mr->vmap_ptr); +if (mr->nr_pages && ctx->user) +__io_unaccount_mem (ctx->user, mr->nr_pages); + +memset (mr,0 ,sizeof (*mr)); +}
This is missing a kvfree(mr->pages);
Signature · 1 line
Jens Axboe
Re: [PATCH 4/6] io_uring: introduce memory regions
Quoted 14 lines · Jens Axboe
io_uring/memmap.c void *__io_uaddr_map(struct page ***pages, unsigned short *npages, · line 208
Quoted 11 lines · Pavel Begunkov
io_uring/memmap.c +11
-0+void io_free_region (struct io_ring_ctx *ctx,struct io_mapped_region *mr) +{ +if (mr->pages) +unpin_user_pages (mr->pages, mr->nr_pages); +if (mr->vmap_ptr) +vunmap (mr->vmap_ptr); +if (mr->nr_pages && ctx->user) +__io_unaccount_mem (ctx->user, mr->nr_pages); + +memset (mr,0 ,sizeof (*mr)); +}This is missing a
kvfree(mr->pages);
Indeed, I'll fix it.
FWIW, this is v1 instead of v2 (which also has the same problem).
Signature · 1 line
Pavel Begunkov