Skip to contentSkip to conversation

regions, param pre-mapping and reg waits extension

PATCH Superseded Pavel Begunkov 2 participants 9 messages Nov 14, 2024 – Nov 15, 2024 io_uring

v1 of 3 Superseded Latest is v3 →

lore
Share & tools
lore
#0 lore raw

cover regions, param pre-mapping and reg waits extension

A bit late but first we need a better and more generic API for ring/memory/region registration (see Patch 4), and it changes the API extending registered waits to be a generic parameter passing mechanism. That will be useful not only for waiting but also request arguments (msghdr, iovec, etc), upcomig rw with meta attrobitutes (PI), and for BPF proposal as well.

I covered region registration with tests, but for reg waits it only enables the basic test. Need to enable and run the rest of them before merged. Dirty branch:

https://github.com/isilence/liburing/tree/io-uring-region-test

Pavel Begunkov (6):
io_uring: fortify io_pin_pages with a warning
io_uring: disable ENTER_EXT_ARG_REG for IOPOLL
io_uring: temporarily disable registered waits
io_uring: introduce memory regions
io_uring: add parameter region registration
io_uring: enable IORING_ENTER_EXT_ARG_REG back

 include/linux/io_uring_types.h | 20 ++++----
 include/uapi/linux/io_uring.h  | 27 ++++++++++-
 io_uring/io_uring.c            | 26 +++++-----
 io_uring/memmap.c              | 67 +++++++++++++++++++++++++
 io_uring/memmap.h              | 14 ++++++
 io_uring/register.c            | 89 +++++++++++++---------------------
 io_uring/register.h            |  1 -
 7 files changed, 161 insertions(+), 83 deletions(-)
Signature · 1 line
2.46.0
#1 lore raw

PATCH 1/6 io_uring: fortify io_pin_pages with a warning

We're a bit too frivolous with types of nr_pages arguments, converting it to long and back to int, passing an unsigned int pointer as an int pointer and so on. Shouldn't cause any problem but should be carefully reviewed, but until then let's add a WARN_ON_ONCE check to be more confident callers don't pass poorely checked arguents.

---
 io_uring/memmap.c | 2 ++
 1 file changed, 2 insertions(+)
io_uring/memmap.c +2 -0
index 85c66fa54956..6ab59c60dfd0 100644
@@ -140,6 +140,8 @@ struct page **io_pin_pages(unsigned long uaddr, unsigned long len, int *npages)
 	nr_pages = end - start;
 	if (WARN_ON_ONCE(!nr_pages))
 		return ERR_PTR(-EINVAL);
+	if (WARN_ON_ONCE(nr_pages > INT_MAX))
+		return ERR_PTR(-EOVERFLOW);
 
 	pages = kvmalloc_array(nr_pages, sizeof(struct page *), GFP_KERNEL);
 	if (!pages)
Signature · 1 line
2.46.0
#2 lore raw

PATCH 2/6 io_uring: disable ENTER_EXT_ARG_REG for IOPOLL

In reply to Pavel Begunkov

IOPOLL doesn't use the extended arguments, no need for it to support IORING_ENTER_EXT_ARG_REG. Let's disable it for IOPOLL, if anything it leaves more space for future extensions.

---
 io_uring/io_uring.c | 8 ++------
 1 file changed, 2 insertions(+), 6 deletions(-)
io_uring/io_uring.c +2 -6
index bd71782057de..464a70bde7e6 100644
@@ -3214,12 +3214,8 @@ static int io_validate_ext_arg(struct io_ring_ctx *ctx, unsigned flags,
 
 	if (!(flags & IORING_ENTER_EXT_ARG))
 		return 0;
-
-	if (flags & IORING_ENTER_EXT_ARG_REG) {
-		if (argsz != sizeof(struct io_uring_reg_wait))
-			return -EINVAL;
-		return PTR_ERR(io_get_ext_arg_reg(ctx, argp));
-	}
+	if (flags & IORING_ENTER_EXT_ARG_REG)
+		return -EINVAL;
 	if (argsz != sizeof(arg))
 		return -EINVAL;
 	if (copy_from_user(&arg, argp, sizeof(arg)))
Signature · 1 line
2.46.0
#3 lore raw

PATCH 3/6 io_uring: temporarily disable registered waits

In reply to Pavel Begunkov

Disable wait argument registration as it'll be replaced with a more generic feature. We'll still need IORING_ENTER_EXT_ARG_REG parsing in a few commits so leave it be.

---
 include/linux/io_uring_types.h | 10 -----
 include/uapi/linux/io_uring.h  |  3 --
 io_uring/io_uring.c            | 10 -----
 io_uring/register.c            | 82 ----------------------------------
 io_uring/register.h            |  1 -
 5 files changed, 106 deletions(-)
include/linux/io_uring_types.h +0 -10
index 072e65e93105..52a5da99a205 100644
@@ -330,14 +330,6 @@ struct io_ring_ctx {
 		atomic_t		cq_wait_nr;
 		atomic_t		cq_timeouts;
 		struct wait_queue_head	cq_wait;
-
-		/*
-		 * If registered with IORING_REGISTER_CQWAIT_REG, a single
-		 * page holds N entries, mapped in cq_wait_arg. cq_wait_index
-		 * is the maximum allowable index.
-		 */
-		struct io_uring_reg_wait	*cq_wait_arg;
-		unsigned char			cq_wait_index;
 	} ____cacheline_aligned_in_smp;
 
 	/* timeouts */
@@ -431,8 +423,6 @@ struct io_ring_ctx {
 	unsigned short			n_sqe_pages;
 	struct page			**ring_pages;
 	struct page			**sqe_pages;
-
-	struct page			**cq_wait_page;
 };
 
 struct io_tw_state {
include/uapi/linux/io_uring.h +0 -3
index 5d08435b95a8..132f5db3d4e8 100644
@@ -627,9 +627,6 @@ enum io_uring_register_op {
 	/* resize CQ ring */
 	IORING_REGISTER_RESIZE_RINGS		= 33,
 
-	/* register fixed io_uring_reg_wait arguments */
-	IORING_REGISTER_CQWAIT_REG		= 34,
-
 	/* this goes last */
 	IORING_REGISTER_LAST,
 
io_uring/io_uring.c +0 -10
index 464a70bde7e6..286b7bb73978 100644
@@ -2709,7 +2709,6 @@ static __cold void io_ring_ctx_free(struct io_ring_ctx *ctx)
 	io_alloc_cache_free(&ctx->msg_cache, io_msg_cache_free);
 	io_futex_cache_free(ctx);
 	io_destroy_buffers(ctx);
-	io_unregister_cqwait_reg(ctx);
 	mutex_unlock(&ctx->uring_lock);
 	if (ctx->sq_creds)
 		put_cred(ctx->sq_creds);
@@ -3195,15 +3194,6 @@ void __io_uring_cancel(bool cancel_all)
 static struct io_uring_reg_wait *io_get_ext_arg_reg(struct io_ring_ctx *ctx,
 			const struct io_uring_getevents_arg __user *uarg)
 {
-	struct io_uring_reg_wait *arg = READ_ONCE(ctx->cq_wait_arg);
-
-	if (arg) {
-		unsigned int index = (unsigned int) (uintptr_t) uarg;
-
-		if (index <= ctx->cq_wait_index)
-			return arg + index;
-	}
-
 	return ERR_PTR(-EFAULT);
 }
 
io_uring/register.c +0 -82

94 lines not on this page · raw message

io_uring/register.h +0 -1
index 3e935e8fa4b2..a5f39d5ef9e0 100644
@@ -5,6 +5,5 @@
 int io_eventfd_unregister(struct io_ring_ctx *ctx);
 int io_unregister_personality(struct io_ring_ctx *ctx, unsigned id);
 struct file *io_uring_register_get_file(unsigned int fd, bool registered);
-void io_unregister_cqwait_reg(struct io_ring_ctx *ctx);
 
 #endif
Signature · 1 line
2.46.0
#4 lore raw

PATCH 4/6 io_uring: introduce memory regions

In reply to Pavel Begunkov

We've got a good number of mappings we share with the userspace, that includes the main rings, provided buffer rings and at least a couple more types. And all of them duplicate some of the code for page pinning, mmap'ing and attempts to optimise it with huge pages.

Introduce a notion of regions. For userspace it's just a new structure called struct io_uring_region_desc which supposed to parameterise all such mapping / queues creations. It either represents a user provided memory, in which case the user_addr field should point to it, or a request to the kernel to creating the memory, in which case the user is supposed to mmap it after using the offset returned in the mmap_offset field. With uniform userspace API we can avoid additional boiler plate code and when we'd be adding some optimisation it'll be applied to all mapping types.

Internally, there is a new structure struct io_mapped_region holding all relevant runtime information and some helpers to work with it. This patch limits it to user provided regions, which will be extended as a follow up work.

---
 include/linux/io_uring_types.h |  6 ++++
 include/uapi/linux/io_uring.h  | 13 +++++++
 io_uring/memmap.c              | 65 ++++++++++++++++++++++++++++++++++
 io_uring/memmap.h              | 14 ++++++++
 4 files changed, 98 insertions(+)
include/linux/io_uring_types.h +6 -0
index 52a5da99a205..1d3a37234ace 100644
@@ -75,6 +75,12 @@ struct io_hash_table {
 	unsigned		hash_bits;
 };
 
+struct io_mapped_region {
+	struct page		**pages;
+	void			*vmap_ptr;
+	size_t			nr_pages;
+};
+
 /*
  * Arbitrary limit, can be raised if need be
  */
include/uapi/linux/io_uring.h +13 -0
index 132f5db3d4e8..7ceeccbbf4cb 100644
@@ -647,6 +647,19 @@ struct io_uring_files_update {
 	__aligned_u64 /* __s32 * */ fds;
 };
 
+enum {
+	/* initialise with user memory pointed by user_addr */
+	IORING_REGION_USER_MEM			= 1,
+};
+
+struct io_uring_region_desc {
+	__u64 user_addr;
+	__u64 size;
+	__u64 flags;
+	__u64 mmap_offset;
+	__u64 __resv[4];
+};
+
 /*
  * Register a fully sparse file space, rather than pass in an array of all
  * -1 file descriptors.
io_uring/memmap.c +65 -0

77 lines not on this page · raw message

io_uring/memmap.h +14 -0
index 5cec5b7ac49a..f361a635b6c7 100644
@@ -22,4 +22,18 @@ unsigned long io_uring_get_unmapped_area(struct file *file, unsigned long addr,
 					 unsigned long flags);
 int io_uring_mmap(struct file *file, struct vm_area_struct *vma);
 
+void io_free_region(struct io_ring_ctx *ctx, struct io_mapped_region *mr);
+int io_create_region(struct io_ring_ctx *ctx, struct io_mapped_region *mr,
+		     struct io_uring_region_desc *reg);
+
+static inline void *io_region_get_ptr(struct io_mapped_region *mr)
+{
+	return mr->vmap_ptr;
+}
+
+static inline bool io_region_is_set(struct io_mapped_region *mr)
+{
+	return !!mr->nr_pages;
+}
+
 #endif
Signature · 1 line
2.46.0
maintainer
#5 lore raw

Re: [PATCH 4/6] io_uring: introduce memory regions

io_uring/memmap.c void *__io_uaddr_map(struct page ***pages, unsigned short *npages, · line 208

Quoted 11 lines · Pavel Begunkov
io_uring/memmap.c +11 -0
+void io_free_region(struct io_ring_ctx *ctx, struct io_mapped_region *mr)
+{
+	if (mr->pages)
+		unpin_user_pages(mr->pages, mr->nr_pages);
+	if (mr->vmap_ptr)
+		vunmap(mr->vmap_ptr);
+	if (mr->nr_pages && ctx->user)
+		__io_unaccount_mem(ctx->user, mr->nr_pages);
+
+	memset(mr, 0, sizeof(*mr));
+}

This is missing a kvfree(mr->pages);

Signature · 1 line
Jens Axboe
#6 lore raw

Re: [PATCH 4/6] io_uring: introduce memory regions

Quoted 14 lines · Jens Axboe

io_uring/memmap.c void *__io_uaddr_map(struct page ***pages, unsigned short *npages, · line 208

Quoted 11 lines · Pavel Begunkov
io_uring/memmap.c +11 -0
+void io_free_region(struct io_ring_ctx *ctx, struct io_mapped_region *mr)
+{
+	if (mr->pages)
+		unpin_user_pages(mr->pages, mr->nr_pages);
+	if (mr->vmap_ptr)
+		vunmap(mr->vmap_ptr);
+	if (mr->nr_pages && ctx->user)
+		__io_unaccount_mem(ctx->user, mr->nr_pages);
+
+	memset(mr, 0, sizeof(*mr));
+}

This is missing a kvfree(mr->pages);

Indeed, I'll fix it.

FWIW, this is v1 instead of v2 (which also has the same problem).

Signature · 1 line
Pavel Begunkov
#7 lore raw

PATCH 5/6 io_uring: add parameter region registration

In reply to Pavel Begunkov

Allow the user to pre-register a region for passing various paramteres.

To use it for passing wait loop arguments, which is wired in the following commit, the region has to be registered with the IORING_PARAM_REGION_WAIT_ARG flag set. The flag also requires the context to be currently disabled, i.e. IORING_SETUP_R_DISABLED, to avoid races with otherwise potentially running waiters.

This will also be useful in the future for various request / SQE arguments like iovec, the meta read/write API, and also for BPF.

---
 include/linux/io_uring_types.h |  6 ++++
 include/uapi/linux/io_uring.h  | 13 ++++++++
 io_uring/io_uring.c            |  1 +
 io_uring/register.c            | 59 ++++++++++++++++++++++++++++++++++
 4 files changed, 79 insertions(+)
include/linux/io_uring_types.h +6 -0
index 1d3a37234ace..aa5f5ea98076 100644
@@ -324,6 +324,9 @@ struct io_ring_ctx {
 		unsigned		cq_entries;
 		struct io_ev_fd	__rcu	*io_ev_fd;
 		unsigned		cq_extra;
+
+		void			*cq_wait_arg;
+		size_t			cq_wait_size;
 	} ____cacheline_aligned_in_smp;
 
 	/*
@@ -429,6 +432,9 @@ struct io_ring_ctx {
 	unsigned short			n_sqe_pages;
 	struct page			**ring_pages;
 	struct page			**sqe_pages;
+
+	/* used for optimised request parameter and wait argument passing  */
+	struct io_mapped_region		param_region;
 };
 
 struct io_tw_state {
include/uapi/linux/io_uring.h +13 -0
index 7ceeccbbf4cb..49b94029c137 100644
@@ -627,6 +627,8 @@ enum io_uring_register_op {
 	/* resize CQ ring */
 	IORING_REGISTER_RESIZE_RINGS		= 33,
 
+	IORING_REGISTER_PARAM_REGION		= 34,
+
 	/* this goes last */
 	IORING_REGISTER_LAST,
 
@@ -660,6 +662,17 @@ struct io_uring_region_desc {
 	__u64 __resv[4];
 };
 
+enum {
+	/* expose the region as registered wait arguments */
+	IORING_PARAM_REGION_WAIT_ARG			= 1,
+};
+
+struct io_uring_param_region_reg {
+	__u64 region_uptr; /* struct io_uring_region_desc * */
+	__u64 flags;
+	__u64 __resv[2];
+};
+
 /*
  * Register a fully sparse file space, rather than pass in an array of all
  * -1 file descriptors.
io_uring/io_uring.c +1 -0
index 286b7bb73978..c640b8a4ceee 100644
@@ -2709,6 +2709,7 @@ static __cold void io_ring_ctx_free(struct io_ring_ctx *ctx)
 	io_alloc_cache_free(&ctx->msg_cache, io_msg_cache_free);
 	io_futex_cache_free(ctx);
 	io_destroy_buffers(ctx);
+	io_free_region(ctx, &ctx->param_region);
 	mutex_unlock(&ctx->uring_lock);
 	if (ctx->sq_creds)
 		put_cred(ctx->sq_creds);
io_uring/register.c +59 -0

71 lines not on this page · raw message

Signature · 1 line
2.46.0
#8 lore raw

PATCH 6/6 io_uring: enable IORING_ENTER_EXT_ARG_REG back

In reply to Pavel Begunkov

We can reenable IORING_ENTER_EXT_ARG_REG, which is now backed by parameter regions.

---
 io_uring/io_uring.c | 11 ++++++++++-
 1 file changed, 10 insertions(+), 1 deletion(-)
io_uring/io_uring.c +10 -1
index c640b8a4ceee..8b4b866b7761 100644
@@ -3195,7 +3195,16 @@ void __io_uring_cancel(bool cancel_all)
 static struct io_uring_reg_wait *io_get_ext_arg_reg(struct io_ring_ctx *ctx,
 			const struct io_uring_getevents_arg __user *uarg)
 {
-	return ERR_PTR(-EFAULT);
+	unsigned long size = sizeof(struct io_uring_reg_wait);
+	unsigned long offset = (uintptr_t)uarg;
+	unsigned long end;
+
+	/* also protects from NULL ->cq_wait_arg as the size would be 0 */
+	if (unlikely(check_add_overflow(offset, size, &end) ||
+		     end >= ctx->cq_wait_size))
+		return ERR_PTR(-EFAULT);
+
+	return ctx->cq_wait_arg + offset;
 }
 
 static int io_validate_ext_arg(struct io_ring_ctx *ctx, unsigned flags,
Signature · 1 line
2.46.0