Skip to contentSkip to conversation

further ring init cleanups

PATCH Applied Pavel Begunkov 2 participants 10 messages Nov 12, 2025 – Nov 13, 2025 io_uring

lore
Share & tools
lore
#0 lore raw

cover further ring init cleanups

There are several goals for this patch set. It deduplicates ring size and offset calculations between ring setup and resizing. It moves most of verification earlier before any allocations, which usually means simpler error handling. And it keeps the logic localised instead of spreading it across the file.

Pavel Begunkov (7):
io_uring: refactor rings_size nosqarray handling
io_uring: use size_add helpers for ring offsets
io_uring: convert params to pointer in ring reisze
io_uring: introduce struct io_ctx_config
io_uring: keep ring laoyut in a structure
io_uring: pre-calculate scq layout
io_uring: move cq/sq user offset init around

 io_uring/io_uring.c | 137 ++++++++++++++++++++++++--------------------
 io_uring/io_uring.h |  19 +++++-
 io_uring/register.c |  65 +++++++++------------
 3 files changed, 119 insertions(+), 102 deletions(-)
Signature · 1 line
2.49.0
#1 lore raw

PATCH 1/7 io_uring: refactor rings_size nosqarray handling

A preparation patch inversing the IORING_SETUP_NO_SQARRAY check, this way there is only one successful return path from the function, which will be helpful later.

---
 io_uring/io_uring.c | 24 ++++++++++++------------
 1 file changed, 12 insertions(+), 12 deletions(-)
io_uring/io_uring.c +12 -12
index 7e069d56b8a1..b37beb1cfefe 100644
@@ -2761,7 +2761,9 @@ unsigned long rings_size(unsigned int flags, unsigned int sq_entries,
 			 unsigned int cq_entries, size_t *sq_offset)
 {
 	struct io_rings *rings;
-	size_t off, sq_array_size;
+	size_t off;
+
+	*sq_offset = SIZE_MAX;
 
 	off = struct_size(rings, cqes, cq_entries);
 	if (off == SIZE_MAX)
@@ -2785,19 +2787,17 @@ unsigned long rings_size(unsigned int flags, unsigned int sq_entries,
 		return SIZE_MAX;
 #endif
 
-	if (flags & IORING_SETUP_NO_SQARRAY) {
-		*sq_offset = SIZE_MAX;
-		return off;
-	}
-
-	*sq_offset = off;
+	if (!(flags & IORING_SETUP_NO_SQARRAY)) {
+		size_t sq_array_size;
 
-	sq_array_size = array_size(sizeof(u32), sq_entries);
-	if (sq_array_size == SIZE_MAX)
-		return SIZE_MAX;
+		*sq_offset = off;
 
-	if (check_add_overflow(off, sq_array_size, &off))
-		return SIZE_MAX;
+		sq_array_size = array_size(sizeof(u32), sq_entries);
+		if (sq_array_size == SIZE_MAX)
+			return SIZE_MAX;
+		if (check_add_overflow(off, sq_array_size, &off))
+			return SIZE_MAX;
+	}
 
 	return off;
 }
Signature · 1 line
2.49.0
#2 lore raw

PATCH 2/7 io_uring: use size_add helpers for ring offsets

In reply to Pavel Begunkov

Use size_add / size_mul set of functions for rings_size() calculations. It's more consistent with struct_size(), and errors are preserved across a series of calculations, so intermediate result checks can be omitted.

---
 io_uring/io_uring.c | 18 ++++++++----------
 1 file changed, 8 insertions(+), 10 deletions(-)
io_uring/io_uring.c +8 -10
index b37beb1cfefe..57ebba8ba46c 100644
@@ -2765,13 +2765,6 @@ unsigned long rings_size(unsigned int flags, unsigned int sq_entries,
 
 	*sq_offset = SIZE_MAX;
 
-	off = struct_size(rings, cqes, cq_entries);
-	if (off == SIZE_MAX)
-		return SIZE_MAX;
-	if (flags & IORING_SETUP_CQE32) {
-		if (check_shl_overflow(off, 1, &off))
-			return SIZE_MAX;
-	}
 	if (flags & IORING_SETUP_CQE_MIXED) {
 		if (cq_entries < 2)
 			return SIZE_MAX;
@@ -2781,6 +2774,12 @@ unsigned long rings_size(unsigned int flags, unsigned int sq_entries,
 			return SIZE_MAX;
 	}
 
+	off = struct_size(rings, cqes, cq_entries);
+	if (flags & IORING_SETUP_CQE32)
+		off = size_mul(off, 2);
+	if (off == SIZE_MAX)
+		return SIZE_MAX;
+
 #ifdef CONFIG_SMP
 	off = ALIGN(off, SMP_CACHE_BYTES);
 	if (off == 0)
@@ -2793,9 +2792,8 @@ unsigned long rings_size(unsigned int flags, unsigned int sq_entries,
 		*sq_offset = off;
 
 		sq_array_size = array_size(sizeof(u32), sq_entries);
-		if (sq_array_size == SIZE_MAX)
-			return SIZE_MAX;
-		if (check_add_overflow(off, sq_array_size, &off))
+		off = size_add(off, sq_array_size);
+		if (off == SIZE_MAX)
 			return SIZE_MAX;
 	}
 
Signature · 1 line
2.49.0
#3 lore raw

PATCH 3/7 io_uring: convert params to pointer in ring reisze

In reply to Pavel Begunkov

The parameters in io_register_resize_rings() will be moved into another structure in a later patch. In preparation to that, convert the params variable it to a pointer, but still store the data on stack.

---
 io_uring/register.c | 48 ++++++++++++++++++++++-----------------------
 1 file changed, 24 insertions(+), 24 deletions(-)
io_uring/register.c +24 -24
index afb924ceb9b6..f6b7b1c1be48 100644
@@ -402,33 +402,33 @@ static int io_register_resize_rings(struct io_ring_ctx *ctx, void __user *arg)
 	struct io_ring_ctx_rings o = { }, n = { }, *to_free = NULL;
 	size_t size, sq_array_offset;
 	unsigned i, tail, old_head;
-	struct io_uring_params p;
+	struct io_uring_params __p, *p = &__p;
 	int ret;
 
 	/* limited to DEFER_TASKRUN for now */
 	if (!(ctx->flags & IORING_SETUP_DEFER_TASKRUN))
 		return -EINVAL;
-	if (copy_from_user(&p, arg, sizeof(p)))
+	if (copy_from_user(p, arg, sizeof(*p)))
 		return -EFAULT;
-	if (p.flags & ~RESIZE_FLAGS)
+	if (p->flags & ~RESIZE_FLAGS)
 		return -EINVAL;
 
 	/* properties that are always inherited */
-	p.flags |= (ctx->flags & COPY_FLAGS);
+	p->flags |= (ctx->flags & COPY_FLAGS);
 
-	ret = io_uring_fill_params(&p);
+	ret = io_uring_fill_params(p);
 	if (unlikely(ret))
 		return ret;
 
-	size = rings_size(p.flags, p.sq_entries, p.cq_entries,
+	size = rings_size(p->flags, p->sq_entries, p->cq_entries,
 				&sq_array_offset);
 	if (size == SIZE_MAX)
 		return -EOVERFLOW;
 
 	memset(&rd, 0, sizeof(rd));
 	rd.size = PAGE_ALIGN(size);
-	if (p.flags & IORING_SETUP_NO_MMAP) {
-		rd.user_addr = p.cq_off.user_addr;
+	if (p->flags & IORING_SETUP_NO_MMAP) {
+		rd.user_addr = p->cq_off.user_addr;
 		rd.flags |= IORING_MEM_REGION_TYPE_USER;
 	}
 	ret = io_create_region(ctx, &n.ring_region, &rd, IORING_OFF_CQ_RING);
@@ -445,20 +445,20 @@ static int io_register_resize_rings(struct io_ring_ctx *ctx, void __user *arg)
 	 * intent... Use read/write once helpers from here on to indicate the
 	 * shared nature of it.
 	 */
-	WRITE_ONCE(n.rings->sq_ring_mask, p.sq_entries - 1);
-	WRITE_ONCE(n.rings->cq_ring_mask, p.cq_entries - 1);
-	WRITE_ONCE(n.rings->sq_ring_entries, p.sq_entries);
-	WRITE_ONCE(n.rings->cq_ring_entries, p.cq_entries);
+	WRITE_ONCE(n.rings->sq_ring_mask, p->sq_entries - 1);
+	WRITE_ONCE(n.rings->cq_ring_mask, p->cq_entries - 1);
+	WRITE_ONCE(n.rings->sq_ring_entries, p->sq_entries);
+	WRITE_ONCE(n.rings->cq_ring_entries, p->cq_entries);
 
-	if (copy_to_user(arg, &p, sizeof(p))) {
+	if (copy_to_user(arg, p, sizeof(*p))) {
 		io_register_free_rings(ctx, &n);
 		return -EFAULT;
 	}
 
-	if (p.flags & IORING_SETUP_SQE128)
-		size = array_size(2 * sizeof(struct io_uring_sqe), p.sq_entries);
+	if (p->flags & IORING_SETUP_SQE128)
+		size = array_size(2 * sizeof(struct io_uring_sqe), p->sq_entries);
 	else
-		size = array_size(sizeof(struct io_uring_sqe), p.sq_entries);
+		size = array_size(sizeof(struct io_uring_sqe), p->sq_entries);
 	if (size == SIZE_MAX) {
 		io_register_free_rings(ctx, &n);
 		return -EOVERFLOW;
@@ -466,8 +466,8 @@ static int io_register_resize_rings(struct io_ring_ctx *ctx, void __user *arg)
 
 	memset(&rd, 0, sizeof(rd));
 	rd.size = PAGE_ALIGN(size);
-	if (p.flags & IORING_SETUP_NO_MMAP) {
-		rd.user_addr = p.sq_off.user_addr;
+	if (p->flags & IORING_SETUP_NO_MMAP) {
+		rd.user_addr = p->sq_off.user_addr;
 		rd.flags |= IORING_MEM_REGION_TYPE_USER;
 	}
 	ret = io_create_region(ctx, &n.sq_region, &rd, IORING_OFF_SQES);
@@ -508,11 +508,11 @@ static int io_register_resize_rings(struct io_ring_ctx *ctx, void __user *arg)
 	 */
 	tail = READ_ONCE(o.rings->sq.tail);
 	old_head = READ_ONCE(o.rings->sq.head);
-	if (tail - old_head > p.sq_entries)
+	if (tail - old_head > p->sq_entries)
 		goto overflow;
 	for (i = old_head; i < tail; i++) {
 		unsigned src_head = i & (ctx->sq_entries - 1);
-		unsigned dst_head = i & (p.sq_entries - 1);
+		unsigned dst_head = i & (p->sq_entries - 1);
 
 		n.sq_sqes[dst_head] = o.sq_sqes[src_head];
 	}
@@ -521,7 +521,7 @@ static int io_register_resize_rings(struct io_ring_ctx *ctx, void __user *arg)
 
 	tail = READ_ONCE(o.rings->cq.tail);
 	old_head = READ_ONCE(o.rings->cq.head);
-	if (tail - old_head > p.cq_entries) {
+	if (tail - old_head > p->cq_entries) {
 overflow:
 		/* restore old rings, and return -EOVERFLOW via cleanup path */
 		ctx->rings = o.rings;
@@ -532,7 +532,7 @@ static int io_register_resize_rings(struct io_ring_ctx *ctx, void __user *arg)
 	}
 	for (i = old_head; i < tail; i++) {
 		unsigned src_head = i & (ctx->cq_entries - 1);
-		unsigned dst_head = i & (p.cq_entries - 1);
+		unsigned dst_head = i & (p->cq_entries - 1);
 
 		n.rings->cqes[dst_head] = o.rings->cqes[src_head];
 	}
@@ -550,8 +550,8 @@ static int io_register_resize_rings(struct io_ring_ctx *ctx, void __user *arg)
 	if (!(ctx->flags & IORING_SETUP_NO_SQARRAY))
 		ctx->sq_array = (u32 *)((char *)n.rings + sq_array_offset);
 
-	ctx->sq_entries = p.sq_entries;
-	ctx->cq_entries = p.cq_entries;
+	ctx->sq_entries = p->sq_entries;
+	ctx->cq_entries = p->cq_entries;
 
 	ctx->rings = n.rings;
 	ctx->sq_sqes = n.sq_sqes;
Signature · 1 line
2.49.0
#4 lore raw

PATCH 4/7 io_uring: introduce struct io_ctx_config

In reply to Pavel Begunkov

There will be more information needed during ctx setup, and instead of passing a handful of pointers around, wrap them all into a new structure. Add a helper for encapsulating all configuration checks and preparation, that's also reused for ring resizing.

Note, it indirectly adds a io_uring_sanitise_params() check to ring resizing, which is a good thing.

---
 io_uring/io_uring.c | 41 ++++++++++++++++++++++++++++-------------
 io_uring/io_uring.h |  8 +++++++-
 io_uring/register.c |  7 +++++--
 3 files changed, 40 insertions(+), 16 deletions(-)
io_uring/io_uring.c +28 -13
index 57ebba8ba46c..f039e293582a 100644
@@ -3480,7 +3480,7 @@ static int io_uring_sanitise_params(struct io_uring_params *p)
 	return 0;
 }
 
-int io_uring_fill_params(struct io_uring_params *p)
+static int io_uring_fill_params(struct io_uring_params *p)
 {
 	unsigned entries = p->sq_entries;
 
@@ -3545,12 +3545,9 @@ int io_uring_fill_params(struct io_uring_params *p)
 	return 0;
 }
 
-static __cold int io_uring_create(struct io_uring_params *p,
-				  struct io_uring_params __user *params)
+int io_prepare_config(struct io_ctx_config *config)
 {
-	struct io_ring_ctx *ctx;
-	struct io_uring_task *tctx;
-	struct file *file;
+	struct io_uring_params *p = &config->p;
 	int ret;
 
 	ret = io_uring_sanitise_params(p);
@@ -3558,7 +3555,22 @@ static __cold int io_uring_create(struct io_uring_params *p,
 		return ret;
 
 	ret = io_uring_fill_params(p);
-	if (unlikely(ret))
+	if (ret)
+		return ret;
+
+	return 0;
+}
+
+static __cold int io_uring_create(struct io_ctx_config *config)
+{
+	struct io_uring_params *p = &config->p;
+	struct io_ring_ctx *ctx;
+	struct io_uring_task *tctx;
+	struct file *file;
+	int ret;
+
+	ret = io_prepare_config(config);
+	if (ret)
 		return ret;
 
 	ctx = io_ring_ctx_alloc(p);
@@ -3631,7 +3643,7 @@ static __cold int io_uring_create(struct io_uring_params *p,
 
 	p->features = IORING_FEAT_FLAGS;
 
-	if (copy_to_user(params, p, sizeof(*p))) {
+	if (copy_to_user(config->uptr, p, sizeof(*p))) {
 		ret = -EFAULT;
 		goto err;
 	}
@@ -3684,16 +3696,19 @@ static __cold int io_uring_create(struct io_uring_params *p,
  */
 static long io_uring_setup(u32 entries, struct io_uring_params __user *params)
 {
-	struct io_uring_params p;
+	struct io_ctx_config config;
+
+	memset(&config, 0, sizeof(config));
 
-	if (copy_from_user(&p, params, sizeof(p)))
+	if (copy_from_user(&config.p, params, sizeof(config.p)))
 		return -EFAULT;
 
-	if (!mem_is_zero(&p.resv, sizeof(p.resv)))
+	if (!mem_is_zero(&config.p.resv, sizeof(config.p.resv)))
 		return -EINVAL;
 
-	p.sq_entries = entries;
-	return io_uring_create(&p, params);
+	config.p.sq_entries = entries;
+	config.uptr = params;
+	return io_uring_create(&config);
 }
 
 static inline int io_uring_allowed(void)
io_uring/io_uring.h +7 -1
index b2251446497a..d8bc44acb9fa 100644
@@ -17,6 +17,11 @@
 #include <trace/events/io_uring.h>
 #endif
 
+struct io_ctx_config {
+	struct io_uring_params p;
+	struct io_uring_params __user *uptr;
+};
+
 #define IORING_FEAT_FLAGS (IORING_FEAT_SINGLE_MMAP |\
 			IORING_FEAT_NODROP |\
 			IORING_FEAT_SUBMIT_STABLE |\
@@ -136,7 +141,8 @@ static inline bool io_should_wake(struct io_wait_queue *iowq)
 
 unsigned long rings_size(unsigned int flags, unsigned int sq_entries,
 			 unsigned int cq_entries, size_t *sq_offset);
-int io_uring_fill_params(struct io_uring_params *p);
+int io_prepare_config(struct io_ctx_config *config);
+
 bool io_cqe_cache_refill(struct io_ring_ctx *ctx, bool overflow, bool cqe32);
 int io_run_task_work_sig(struct io_ring_ctx *ctx);
 int io_run_local_work(struct io_ring_ctx *ctx, int min_events, int max_events);
io_uring/register.c +5 -2
index f6b7b1c1be48..13385ac0f85a 100644
@@ -398,13 +398,16 @@ static void io_register_free_rings(struct io_ring_ctx *ctx,
 
 static int io_register_resize_rings(struct io_ring_ctx *ctx, void __user *arg)
 {
+	struct io_ctx_config config;
 	struct io_uring_region_desc rd;
 	struct io_ring_ctx_rings o = { }, n = { }, *to_free = NULL;
 	size_t size, sq_array_offset;
 	unsigned i, tail, old_head;
-	struct io_uring_params __p, *p = &__p;
+	struct io_uring_params *p = &config.p;
 	int ret;
 
+	memset(&config, 0, sizeof(config));
+
 	/* limited to DEFER_TASKRUN for now */
 	if (!(ctx->flags & IORING_SETUP_DEFER_TASKRUN))
 		return -EINVAL;
@@ -416,7 +419,7 @@ static int io_register_resize_rings(struct io_ring_ctx *ctx, void __user *arg)
 	/* properties that are always inherited */
 	p->flags |= (ctx->flags & COPY_FLAGS);
 
-	ret = io_uring_fill_params(p);
+	ret = io_prepare_config(&config);
 	if (unlikely(ret))
 		return ret;
 
Signature · 1 line
2.49.0
#5 lore raw

PATCH 5/7 io_uring: keep ring laoyut in a structure

In reply to Pavel Begunkov

Add a structure keeping SQ/CQ sizes and offsets. For now it only records data previously returned from rings_size and the SQ size.

---
 io_uring/io_uring.c | 54 +++++++++++++++++++++++----------------------
 io_uring/io_uring.h | 12 ++++++++--
 io_uring/register.c | 24 ++++++--------------
 3 files changed, 45 insertions(+), 45 deletions(-)
io_uring/io_uring.c +28 -26
index f039e293582a..1dfd0a8a7270 100644
@@ -2757,47 +2757,57 @@ static void io_rings_free(struct io_ring_ctx *ctx)
 	ctx->sq_sqes = NULL;
 }
 
-unsigned long rings_size(unsigned int flags, unsigned int sq_entries,
-			 unsigned int cq_entries, size_t *sq_offset)
+int rings_size(unsigned int flags, unsigned int sq_entries,
+		unsigned int cq_entries, struct io_rings_layout *rl)
 {
 	struct io_rings *rings;
+	size_t sqe_size;
 	size_t off;
 
-	*sq_offset = SIZE_MAX;
-
 	if (flags & IORING_SETUP_CQE_MIXED) {
 		if (cq_entries < 2)
-			return SIZE_MAX;
+			return -EOVERFLOW;
 	}
 	if (flags & IORING_SETUP_SQE_MIXED) {
 		if (sq_entries < 2)
-			return SIZE_MAX;
+			return -EOVERFLOW;
 	}
 
+	rl->sq_array_offset = SIZE_MAX;
+
+	sqe_size = sizeof(struct io_uring_sqe);
+	if (flags & IORING_SETUP_SQE128)
+		sqe_size *= 2;
+
+	rl->sq_size = array_size(sqe_size, sq_entries);
+	if (rl->sq_size == SIZE_MAX)
+		return -EOVERFLOW;
+
 	off = struct_size(rings, cqes, cq_entries);
 	if (flags & IORING_SETUP_CQE32)
 		off = size_mul(off, 2);
 	if (off == SIZE_MAX)
-		return SIZE_MAX;
+		return -EOVERFLOW;
 
 #ifdef CONFIG_SMP
 	off = ALIGN(off, SMP_CACHE_BYTES);
 	if (off == 0)
-		return SIZE_MAX;
+		return -EOVERFLOW;
 #endif
 
 	if (!(flags & IORING_SETUP_NO_SQARRAY)) {
 		size_t sq_array_size;
 
-		*sq_offset = off;
+		rl->sq_array_offset = off;
 
 		sq_array_size = array_size(sizeof(u32), sq_entries);
 		off = size_add(off, sq_array_size);
 		if (off == SIZE_MAX)
-			return SIZE_MAX;
+			return -EOVERFLOW;
 	}
 
-	return off;
+	rl->rings_size = off;
+	return 0;
 }
 
 static __cold void __io_req_caches_free(struct io_ring_ctx *ctx)
@@ -3346,28 +3356,20 @@ static __cold int io_allocate_scq_urings(struct io_ring_ctx *ctx,
 					 struct io_uring_params *p)
 {
 	struct io_uring_region_desc rd;
+	struct io_rings_layout __rl, *rl = &__rl;
 	struct io_rings *rings;
-	size_t sq_array_offset;
-	size_t sq_size, cq_size, sqe_size;
 	int ret;
 
 	/* make sure these are sane, as we already accounted them */
 	ctx->sq_entries = p->sq_entries;
 	ctx->cq_entries = p->cq_entries;
 
-	sqe_size = sizeof(struct io_uring_sqe);
-	if (p->flags & IORING_SETUP_SQE128)
-		sqe_size *= 2;
-	sq_size = array_size(sqe_size, p->sq_entries);
-	if (sq_size == SIZE_MAX)
-		return -EOVERFLOW;
-	cq_size = rings_size(ctx->flags, p->sq_entries, p->cq_entries,
-			  &sq_array_offset);
-	if (cq_size == SIZE_MAX)
-		return -EOVERFLOW;
+	ret = rings_size(ctx->flags, p->sq_entries, p->cq_entries, rl);
+	if (ret)
+		return ret;
 
 	memset(&rd, 0, sizeof(rd));
-	rd.size = PAGE_ALIGN(cq_size);
+	rd.size = PAGE_ALIGN(rl->rings_size);
 	if (ctx->flags & IORING_SETUP_NO_MMAP) {
 		rd.user_addr = p->cq_off.user_addr;
 		rd.flags |= IORING_MEM_REGION_TYPE_USER;
@@ -3378,10 +3380,10 @@ static __cold int io_allocate_scq_urings(struct io_ring_ctx *ctx,
 	ctx->rings = rings = io_region_get_ptr(&ctx->ring_region);
 
 	if (!(ctx->flags & IORING_SETUP_NO_SQARRAY))
-		ctx->sq_array = (u32 *)((char *)rings + sq_array_offset);
+		ctx->sq_array = (u32 *)((char *)rings + rl->sq_array_offset);
 
 	memset(&rd, 0, sizeof(rd));
-	rd.size = PAGE_ALIGN(sq_size);
+	rd.size = PAGE_ALIGN(rl->sq_size);
 	if (ctx->flags & IORING_SETUP_NO_MMAP) {
 		rd.user_addr = p->sq_off.user_addr;
 		rd.flags |= IORING_MEM_REGION_TYPE_USER;
io_uring/io_uring.h +10 -2
index d8bc44acb9fa..5e544c2d27c8 100644
@@ -17,6 +17,14 @@
 #include <trace/events/io_uring.h>
 #endif
 
+struct io_rings_layout {
+	/* size of CQ + headers + SQ offset array */
+	size_t rings_size;
+	size_t sq_size;
+
+	size_t sq_array_offset;
+};
+
 struct io_ctx_config {
 	struct io_uring_params p;
 	struct io_uring_params __user *uptr;
@@ -139,8 +147,8 @@ static inline bool io_should_wake(struct io_wait_queue *iowq)
 #define IORING_MAX_ENTRIES	32768
 #define IORING_MAX_CQ_ENTRIES	(2 * IORING_MAX_ENTRIES)
 
-unsigned long rings_size(unsigned int flags, unsigned int sq_entries,
-			 unsigned int cq_entries, size_t *sq_offset);
+int rings_size(unsigned int flags, unsigned int sq_entries,
+		unsigned int cq_entries, struct io_rings_layout *rl);
 int io_prepare_config(struct io_ctx_config *config);
 
 bool io_cqe_cache_refill(struct io_ring_ctx *ctx, bool overflow, bool cqe32);
io_uring/register.c +7 -17
index 13385ac0f85a..98693021edbe 100644
@@ -401,9 +401,9 @@ static int io_register_resize_rings(struct io_ring_ctx *ctx, void __user *arg)
 	struct io_ctx_config config;
 	struct io_uring_region_desc rd;
 	struct io_ring_ctx_rings o = { }, n = { }, *to_free = NULL;
-	size_t size, sq_array_offset;
 	unsigned i, tail, old_head;
 	struct io_uring_params *p = &config.p;
+	struct io_rings_layout __rl, *rl = &__rl;
 	int ret;
 
 	memset(&config, 0, sizeof(config));
@@ -423,13 +423,12 @@ static int io_register_resize_rings(struct io_ring_ctx *ctx, void __user *arg)
 	if (unlikely(ret))
 		return ret;
 
-	size = rings_size(p->flags, p->sq_entries, p->cq_entries,
-				&sq_array_offset);
-	if (size == SIZE_MAX)
-		return -EOVERFLOW;
+	ret = rings_size(p->flags, p->sq_entries, p->cq_entries, rl);
+	if (ret)
+		return ret;
 
 	memset(&rd, 0, sizeof(rd));
-	rd.size = PAGE_ALIGN(size);
+	rd.size = PAGE_ALIGN(rl->rings_size);
 	if (p->flags & IORING_SETUP_NO_MMAP) {
 		rd.user_addr = p->cq_off.user_addr;
 		rd.flags |= IORING_MEM_REGION_TYPE_USER;
@@ -458,17 +457,8 @@ static int io_register_resize_rings(struct io_ring_ctx *ctx, void __user *arg)
 		return -EFAULT;
 	}
 
-	if (p->flags & IORING_SETUP_SQE128)
-		size = array_size(2 * sizeof(struct io_uring_sqe), p->sq_entries);
-	else
-		size = array_size(sizeof(struct io_uring_sqe), p->sq_entries);
-	if (size == SIZE_MAX) {
-		io_register_free_rings(ctx, &n);
-		return -EOVERFLOW;
-	}
-
 	memset(&rd, 0, sizeof(rd));
-	rd.size = PAGE_ALIGN(size);
+	rd.size = PAGE_ALIGN(rl->sq_size);
 	if (p->flags & IORING_SETUP_NO_MMAP) {
 		rd.user_addr = p->sq_off.user_addr;
 		rd.flags |= IORING_MEM_REGION_TYPE_USER;
@@ -551,7 +541,7 @@ static int io_register_resize_rings(struct io_ring_ctx *ctx, void __user *arg)
 
 	/* all done, store old pointers and assign new ones */
 	if (!(ctx->flags & IORING_SETUP_NO_SQARRAY))
-		ctx->sq_array = (u32 *)((char *)n.rings + sq_array_offset);
+		ctx->sq_array = (u32 *)((char *)n.rings + rl->sq_array_offset);
 
 	ctx->sq_entries = p->sq_entries;
 	ctx->cq_entries = p->cq_entries;
Signature · 1 line
2.49.0
#6 lore raw

PATCH 6/7 io_uring: pre-calculate scq layout

In reply to Pavel Begunkov

Move ring layouts calculations into io_prepare_config(), so that more misconfiguration checking can be done earlier before creating a ctx. It also deduplicates some code with ring resizing. And as a bonus, now it initialises params->sq_off.array, which is closer to all other user offset init, and also applies it to ring resizing, which was previously missing it.

---
 io_uring/io_uring.c | 27 ++++++++++++++-------------
 io_uring/io_uring.h |  3 +--
 io_uring/register.c |  4 ----
 3 files changed, 15 insertions(+), 19 deletions(-)
io_uring/io_uring.c +14 -13
index 1dfd0a8a7270..d286118dcd9d 100644
@@ -2757,8 +2757,8 @@ static void io_rings_free(struct io_ring_ctx *ctx)
 	ctx->sq_sqes = NULL;
 }
 
-int rings_size(unsigned int flags, unsigned int sq_entries,
-		unsigned int cq_entries, struct io_rings_layout *rl)
+static int rings_size(unsigned int flags, unsigned int sq_entries,
+		      unsigned int cq_entries, struct io_rings_layout *rl)
 {
 	struct io_rings *rings;
 	size_t sqe_size;
@@ -3353,10 +3353,11 @@ bool io_is_uring_fops(struct file *file)
 }
 
 static __cold int io_allocate_scq_urings(struct io_ring_ctx *ctx,
-					 struct io_uring_params *p)
+					 struct io_ctx_config *config)
 {
+	struct io_uring_params *p = &config->p;
+	struct io_rings_layout *rl = &config->layout;
 	struct io_uring_region_desc rd;
-	struct io_rings_layout __rl, *rl = &__rl;
 	struct io_rings *rings;
 	int ret;
 
@@ -3364,10 +3365,6 @@ static __cold int io_allocate_scq_urings(struct io_ring_ctx *ctx,
 	ctx->sq_entries = p->sq_entries;
 	ctx->cq_entries = p->cq_entries;
 
-	ret = rings_size(ctx->flags, p->sq_entries, p->cq_entries, rl);
-	if (ret)
-		return ret;
-
 	memset(&rd, 0, sizeof(rd));
 	rd.size = PAGE_ALIGN(rl->rings_size);
 	if (ctx->flags & IORING_SETUP_NO_MMAP) {
@@ -3378,7 +3375,6 @@ static __cold int io_allocate_scq_urings(struct io_ring_ctx *ctx,
 	if (ret)
 		return ret;
 	ctx->rings = rings = io_region_get_ptr(&ctx->ring_region);
-
 	if (!(ctx->flags & IORING_SETUP_NO_SQARRAY))
 		ctx->sq_array = (u32 *)((char *)rings + rl->sq_array_offset);
 
@@ -3560,6 +3556,14 @@ int io_prepare_config(struct io_ctx_config *config)
 	if (ret)
 		return ret;
 
+	ret = rings_size(p->flags, p->sq_entries, p->cq_entries,
+			 &config->layout);
+	if (ret)
+		return ret;
+
+	if (!(p->flags & IORING_SETUP_NO_SQARRAY))
+		p->sq_off.array = config->layout.sq_array_offset;
+
 	return 0;
 }
 
@@ -3632,13 +3636,10 @@ static __cold int io_uring_create(struct io_ctx_config *config)
 	mmgrab(current->mm);
 	ctx->mm_account = current->mm;
 
-	ret = io_allocate_scq_urings(ctx, p);
+	ret = io_allocate_scq_urings(ctx, config);
 	if (ret)
 		goto err;
 
-	if (!(p->flags & IORING_SETUP_NO_SQARRAY))
-		p->sq_off.array = (char *)ctx->sq_array - (char *)ctx->rings;
-
 	ret = io_sq_offload_create(ctx, p);
 	if (ret)
 		goto err;
io_uring/io_uring.h +1 -2
index 5e544c2d27c8..a790c16854d3 100644
@@ -27,6 +27,7 @@ struct io_rings_layout {
 
 struct io_ctx_config {
 	struct io_uring_params p;
+	struct io_rings_layout layout;
 	struct io_uring_params __user *uptr;
 };
 
@@ -147,8 +148,6 @@ static inline bool io_should_wake(struct io_wait_queue *iowq)
 #define IORING_MAX_ENTRIES	32768
 #define IORING_MAX_CQ_ENTRIES	(2 * IORING_MAX_ENTRIES)
 
-int rings_size(unsigned int flags, unsigned int sq_entries,
-		unsigned int cq_entries, struct io_rings_layout *rl);
 int io_prepare_config(struct io_ctx_config *config);
 
 bool io_cqe_cache_refill(struct io_ring_ctx *ctx, bool overflow, bool cqe32);
io_uring/register.c +0 -4
index 98693021edbe..110e978b872d 100644
@@ -423,10 +423,6 @@ static int io_register_resize_rings(struct io_ring_ctx *ctx, void __user *arg)
 	if (unlikely(ret))
 		return ret;
 
-	ret = rings_size(p->flags, p->sq_entries, p->cq_entries, rl);
-	if (ret)
-		return ret;
-
 	memset(&rd, 0, sizeof(rd));
 	rd.size = PAGE_ALIGN(rl->rings_size);
 	if (p->flags & IORING_SETUP_NO_MMAP) {
Signature · 1 line
2.49.0
#7 lore raw

PATCH 7/7 io_uring: move cq/sq user offset init around

In reply to Pavel Begunkov

Move user SQ/CQ offset initialisation at the end of io_prepare_config() where it already calculated all information to set it properly.

---
 io_uring/io_uring.c | 41 ++++++++++++++++++++---------------------
 1 file changed, 20 insertions(+), 21 deletions(-)
io_uring/io_uring.c +20 -21
index d286118dcd9d..c1c923d19cc3 100644
@@ -3519,27 +3519,6 @@ static int io_uring_fill_params(struct io_uring_params *p)
 		p->cq_entries = 2 * p->sq_entries;
 	}
 
-	p->sq_off.head = offsetof(struct io_rings, sq.head);
-	p->sq_off.tail = offsetof(struct io_rings, sq.tail);
-	p->sq_off.ring_mask = offsetof(struct io_rings, sq_ring_mask);
-	p->sq_off.ring_entries = offsetof(struct io_rings, sq_ring_entries);
-	p->sq_off.flags = offsetof(struct io_rings, sq_flags);
-	p->sq_off.dropped = offsetof(struct io_rings, sq_dropped);
-	p->sq_off.resv1 = 0;
-	if (!(p->flags & IORING_SETUP_NO_MMAP))
-		p->sq_off.user_addr = 0;
-
-	p->cq_off.head = offsetof(struct io_rings, cq.head);
-	p->cq_off.tail = offsetof(struct io_rings, cq.tail);
-	p->cq_off.ring_mask = offsetof(struct io_rings, cq_ring_mask);
-	p->cq_off.ring_entries = offsetof(struct io_rings, cq_ring_entries);
-	p->cq_off.overflow = offsetof(struct io_rings, cq_overflow);
-	p->cq_off.cqes = offsetof(struct io_rings, cqes);
-	p->cq_off.flags = offsetof(struct io_rings, cq_flags);
-	p->cq_off.resv1 = 0;
-	if (!(p->flags & IORING_SETUP_NO_MMAP))
-		p->cq_off.user_addr = 0;
-
 	return 0;
 }
 
@@ -3561,6 +3540,26 @@ int io_prepare_config(struct io_ctx_config *config)
 	if (ret)
 		return ret;
 
+	p->sq_off.head = offsetof(struct io_rings, sq.head);
+	p->sq_off.tail = offsetof(struct io_rings, sq.tail);
+	p->sq_off.ring_mask = offsetof(struct io_rings, sq_ring_mask);
+	p->sq_off.ring_entries = offsetof(struct io_rings, sq_ring_entries);
+	p->sq_off.flags = offsetof(struct io_rings, sq_flags);
+	p->sq_off.dropped = offsetof(struct io_rings, sq_dropped);
+	p->sq_off.resv1 = 0;
+	if (!(p->flags & IORING_SETUP_NO_MMAP))
+		p->sq_off.user_addr = 0;
+
+	p->cq_off.head = offsetof(struct io_rings, cq.head);
+	p->cq_off.tail = offsetof(struct io_rings, cq.tail);
+	p->cq_off.ring_mask = offsetof(struct io_rings, cq_ring_mask);
+	p->cq_off.ring_entries = offsetof(struct io_rings, cq_ring_entries);
+	p->cq_off.overflow = offsetof(struct io_rings, cq_overflow);
+	p->cq_off.cqes = offsetof(struct io_rings, cqes);
+	p->cq_off.flags = offsetof(struct io_rings, cq_flags);
+	p->cq_off.resv1 = 0;
+	if (!(p->flags & IORING_SETUP_NO_MMAP))
+		p->cq_off.user_addr = 0;
 	if (!(p->flags & IORING_SETUP_NO_SQARRAY))
 		p->sq_off.array = config->layout.sq_array_offset;
 
Signature · 1 line
2.49.0
maintainer
#8 lore raw
In reply to Pavel Begunkov
Quoted 15 lines · Pavel Begunkov

There are several goals for this patch set. It deduplicates ring size and offset calculations between ring setup and resizing. It moves most of verification earlier before any allocations, which usually means simpler error handling. And it keeps the logic localised instead of spreading it across the file.

Pavel Begunkov (7):
io_uring: refactor rings_size nosqarray handling
io_uring: use size_add helpers for ring offsets
io_uring: convert params to pointer in ring reisze
io_uring: introduce struct io_ctx_config
io_uring: keep ring laoyut in a structure
io_uring: pre-calculate scq layout
io_uring: move cq/sq user offset init around

Quoted 4 lines
 io_uring/io_uring.c | 137 ++++++++++++++++++++++++--------------------
 io_uring/io_uring.h |  19 +++++-
 io_uring/register.c |  65 +++++++++------------
 3 files changed, 119 insertions(+), 102 deletions(-)

All looks pretty straightforward to me. I'm going to let it simmer for a day or so to give other folks a chance to take a look too.

Signature · 1 line
Jens Axboe
maintainer Applied
#9 lore raw
In reply to Pavel Begunkov
Quoted 15 lines · Pavel Begunkov

There are several goals for this patch set. It deduplicates ring size and offset calculations between ring setup and resizing. It moves most of verification earlier before any allocations, which usually means simpler error handling. And it keeps the logic localised instead of spreading it across the file.

Pavel Begunkov (7):
io_uring: refactor rings_size nosqarray handling
io_uring: use size_add helpers for ring offsets
io_uring: convert params to pointer in ring reisze
io_uring: introduce struct io_ctx_config
io_uring: keep ring laoyut in a structure
io_uring: pre-calculate scq layout
io_uring: move cq/sq user offset init around

Quoted 1 line

[...]

Applied, thanks!

[1/7] io_uring: refactor rings_size nosqarray handling

      commit: e279bb4b4c4d012808fb21ff41183a2e76c26679
[2/7] io_uring: use size_add helpers for ring offsets
      commit: 94cd832916521d8d51b25b40691354c24831c655
[3/7] io_uring: convert params to pointer in ring reisze
      commit: 929dbbb699110c9377da721ed7b44a660bb4ee01
[4/7] io_uring: introduce struct io_ctx_config
      commit: 0f4b537363cb66c78e97bb58c26986af62856356
[5/7] io_uring: keep ring laoyut in a structure
      commit: 001b76b7e755767d847e9aebf1fd6e525f1e58c8
[6/7] io_uring: pre-calculate scq layout
      commit: eb76ff6a6829a9a54a385804cc9dbe4460f156d6
[7/7] io_uring: move cq/sq user offset init around
      commit: d741c6255524f0691aea53381219fadcd2b38408

Best regards,

Signature · 1 line
Jens Axboe