Skip to contentSkip to conversation

Introduce ring flexible placement

RFC New Pavel Begunkov 1 participant 17 messages Nov 6, 2025 io_uring

lore
Share & tools
lore
#0 lore raw

cover Introduce ring flexible placement

This patchset allows the user to put rings and ring headers into the memory/parameter region and specify offsets where they should lie.

It solves a problem where the headers are placed before ring entries, which 1. usually padded wasting memory and using more cache lines than a smarter placement would need. And 2. it grows the region size to a non pow2. It's also handy to have a way to put everything into a single region.

It's implemented for SQ and CQ, but planned to be supported by zcrx as well. There is a bunch of cleanups (1-13), it'd make sense to merge some of them separately. It also adds an ability to register a memory region during setup and not via a registration (Patch 15). And the placement handling is in Patch 16.

Pavel Begunkov (16):
io_uring: add helper calculating region byte size
io_uring: pass sq entires in the params struct
io_uring: use mem_is_zero to check ring params
io_uring: move flags check to io_uring_sanitise_params
io_uring: introduce struct io_ctx_config
io_uring: split out config init helper
io_uring: add structure keeping ring offsets
io_uring: pre-calculate scq offsets
io_uring: inroduce helper for setting user offset
io_uring: separate cqe array from headers io_uring/region: introduce io_region_slice
io_uring: convert pointer init to io_region_slice
io_uring: refactor rings_size()
io_uring: extract io_create_mem_region
io_uring: allow creating mem region at setup
io_uring: introduce SCQ placement

 include/linux/io_uring_types.h |  17 +-
 include/uapi/linux/io_uring.h  |  21 +-
 io_uring/fdinfo.c              |   2 +-
 io_uring/io_uring.c            | 341 +++++++++++++++++++++++----------
 io_uring/io_uring.h            |  34 +++-
 io_uring/memmap.c              |  16 +-
 io_uring/memmap.h              |   7 +
 io_uring/register.c            |  75 ++++----
 8 files changed, 360 insertions(+), 153 deletions(-)
Signature · 1 line
2.49.0
#1 lore raw

RFC 1/16 io_uring: add helper calculating region byte size

There has been type related issues with region size calculation, add an utility helper function that returns the size and handles type conversions right.

---
 io_uring/memmap.c | 4 ++--
 io_uring/memmap.h | 5 +++++
 2 files changed, 7 insertions(+), 2 deletions(-)
io_uring/memmap.c +2 -2
index ce8118434c5a..b329cee8d6e8 100644
@@ -135,7 +135,7 @@ static int io_region_pin_pages(struct io_ring_ctx *ctx,
 				struct io_mapped_region *mr,
 				struct io_uring_region_desc *reg)
 {
-	unsigned long size = (size_t)mr->nr_pages << PAGE_SHIFT;
+	size_t size = io_region_size(mr);
 	struct page **pages;
 	int nr_pages;
 
@@ -156,7 +156,7 @@ static int io_region_allocate_pages(struct io_ring_ctx *ctx,
 				    unsigned long mmap_offset)
 {
 	gfp_t gfp = GFP_KERNEL_ACCOUNT | __GFP_ZERO | __GFP_NOWARN;
-	size_t size = (size_t) mr->nr_pages << PAGE_SHIFT;
+	size_t size = io_region_size(mr);
 	unsigned long nr_allocated;
 	struct page **pages;
 
io_uring/memmap.h +5 -0
index f9e94458c01f..d4b8b6363a7d 100644
@@ -43,4 +43,9 @@ static inline void io_region_publish(struct io_ring_ctx *ctx,
 	*dst_region = *src_region;
 }
 
+static inline size_t io_region_size(struct io_mapped_region *mr)
+{
+	return (size_t)mr->nr_pages << PAGE_SHIFT;
+}
+
 #endif
Signature · 1 line
2.49.0
#2 lore raw

RFC 2/16 io_uring: pass sq entires in the params struct

In reply to Pavel Begunkov

There is no need to pass the user requested number of SQ entries separately from the main parameter structure io_uring_params. Initialise it at the beginning and stop passing it in favour of struct io_uring_params::sq_entries.

---
 io_uring/io_uring.c | 11 +++++++----
 io_uring/io_uring.h |  2 +-
 io_uring/register.c |  2 +-
 3 files changed, 9 insertions(+), 6 deletions(-)
io_uring/io_uring.c +7 -4
index f9f8ffcdad07..eae1ad3cd02e 100644
@@ -3484,8 +3484,10 @@ static int io_uring_sanitise_params(struct io_uring_params *p)
 	return 0;
 }
 
-int io_uring_fill_params(unsigned entries, struct io_uring_params *p)
+int io_uring_fill_params(struct io_uring_params *p)
 {
+	unsigned entries = p->sq_entries;
+
 	if (!entries)
 		return -EINVAL;
 	if (entries > IORING_MAX_ENTRIES) {
@@ -3547,7 +3549,7 @@ int io_uring_fill_params(unsigned entries, struct io_uring_params *p)
 	return 0;
 }
 
-static __cold int io_uring_create(unsigned entries, struct io_uring_params *p,
+static __cold int io_uring_create(struct io_uring_params *p,
 				  struct io_uring_params __user *params)
 {
 	struct io_ring_ctx *ctx;
@@ -3559,7 +3561,7 @@ static __cold int io_uring_create(unsigned entries, struct io_uring_params *p,
 	if (ret)
 		return ret;
 
-	ret = io_uring_fill_params(entries, p);
+	ret = io_uring_fill_params(p);
 	if (unlikely(ret))
 		return ret;
 
@@ -3698,7 +3700,8 @@ static long io_uring_setup(u32 entries, struct io_uring_params __user *params)
 
 	if (p.flags & ~IORING_SETUP_FLAGS)
 		return -EINVAL;
-	return io_uring_create(entries, &p, params);
+	p.sq_entries = entries;
+	return io_uring_create(&p, params);
 }
 
 static inline int io_uring_allowed(void)
io_uring/io_uring.h +1 -1
index 23c268ab1c8f..b2251446497a 100644
@@ -136,7 +136,7 @@ static inline bool io_should_wake(struct io_wait_queue *iowq)
 
 unsigned long rings_size(unsigned int flags, unsigned int sq_entries,
 			 unsigned int cq_entries, size_t *sq_offset);
-int io_uring_fill_params(unsigned entries, struct io_uring_params *p);
+int io_uring_fill_params(struct io_uring_params *p);
 bool io_cqe_cache_refill(struct io_ring_ctx *ctx, bool overflow, bool cqe32);
 int io_run_task_work_sig(struct io_ring_ctx *ctx);
 int io_run_local_work(struct io_ring_ctx *ctx, int min_events, int max_events);
io_uring/register.c +1 -1
index 1a3e05be6e7b..0d70696468f6 100644
@@ -416,7 +416,7 @@ static int io_register_resize_rings(struct io_ring_ctx *ctx, void __user *arg)
 	/* properties that are always inherited */
 	p.flags |= (ctx->flags & COPY_FLAGS);
 
-	ret = io_uring_fill_params(p.sq_entries, &p);
+	ret = io_uring_fill_params(&p);
 	if (unlikely(ret))
 		return ret;
 
Signature · 1 line
2.49.0
#3 lore raw

RFC 3/16 io_uring: use mem_is_zero to check ring params

In reply to Pavel Begunkov

mem_is_zero() does the job without hand rolled loops, use that to verify reserved fields of ring params.

---
 io_uring/io_uring.c | 8 +++-----
 1 file changed, 3 insertions(+), 5 deletions(-)
io_uring/io_uring.c +3 -5
index eae1ad3cd02e..dec37cf7c62c 100644
@@ -3689,14 +3689,12 @@ static __cold int io_uring_create(struct io_uring_params *p,
 static long io_uring_setup(u32 entries, struct io_uring_params __user *params)
 {
 	struct io_uring_params p;
-	int i;
 
 	if (copy_from_user(&p, params, sizeof(p)))
 		return -EFAULT;
-	for (i = 0; i < ARRAY_SIZE(p.resv); i++) {
-		if (p.resv[i])
-			return -EINVAL;
-	}
+
+	if (!mem_is_zero(&p.resv, sizeof(p.resv)))
+		return -EINVAL;
 
 	if (p.flags & ~IORING_SETUP_FLAGS)
 		return -EINVAL;
Signature · 1 line
2.49.0
#4 lore raw

RFC 4/16 io_uring: move flags check to io_uring_sanitise_params

In reply to Pavel Begunkov

io_uring_sanitise_params() sanitises most of the setup flags invariants, move the IORING_SETUP_FLAGS check from io_uring_setup() into it.

---
 io_uring/io_uring.c | 5 +++--
 1 file changed, 3 insertions(+), 2 deletions(-)
io_uring/io_uring.c +3 -2
index dec37cf7c62c..ef1b75c5a4d2 100644
@@ -3435,6 +3435,9 @@ static int io_uring_sanitise_params(struct io_uring_params *p)
 {
 	unsigned flags = p->flags;
 
+	if (flags & ~IORING_SETUP_FLAGS)
+		return -EINVAL;
+
 	/* There is no way to mmap rings without a real fd */
 	if ((flags & IORING_SETUP_REGISTERED_FD_ONLY) &&
 	    !(flags & IORING_SETUP_NO_MMAP))
@@ -3696,8 +3699,6 @@ static long io_uring_setup(u32 entries, struct io_uring_params __user *params)
 	if (!mem_is_zero(&p.resv, sizeof(p.resv)))
 		return -EINVAL;
 
-	if (p.flags & ~IORING_SETUP_FLAGS)
-		return -EINVAL;
 	p.sq_entries = entries;
 	return io_uring_create(&p, params);
 }
Signature · 1 line
2.49.0
#5 lore raw

RFC 5/16 io_uring: introduce struct io_ctx_config

In reply to Pavel Begunkov

There will be more information needed during ctx setup, so instead of passing a handful of pointers around, wrap them all into a new structure.

---
 io_uring/io_uring.c | 19 +++++++++++--------
 io_uring/io_uring.h |  5 +++++
 2 files changed, 16 insertions(+), 8 deletions(-)
io_uring/io_uring.c +11 -8
index ef1b75c5a4d2..142811c7a4f5 100644
@@ -3552,9 +3552,9 @@ int io_uring_fill_params(struct io_uring_params *p)
 	return 0;
 }
 
-static __cold int io_uring_create(struct io_uring_params *p,
-				  struct io_uring_params __user *params)
+static __cold int io_uring_create(struct io_ctx_config *config)
 {
+	struct io_uring_params *p = &config->p;
 	struct io_ring_ctx *ctx;
 	struct io_uring_task *tctx;
 	struct file *file;
@@ -3638,7 +3638,7 @@ static __cold int io_uring_create(struct io_uring_params *p,
 
 	p->features = IORING_FEAT_FLAGS;
 
-	if (copy_to_user(params, p, sizeof(*p))) {
+	if (copy_to_user(config->uptr, p, sizeof(*p))) {
 		ret = -EFAULT;
 		goto err;
 	}
@@ -3691,16 +3691,19 @@ static __cold int io_uring_create(struct io_uring_params *p,
  */
 static long io_uring_setup(u32 entries, struct io_uring_params __user *params)
 {
-	struct io_uring_params p;
+	struct io_ctx_config config;
 
-	if (copy_from_user(&p, params, sizeof(p)))
+	memset(&config, 0, sizeof(config));
+
+	if (copy_from_user(&config.p, params, sizeof(config.p)))
 		return -EFAULT;
 
-	if (!mem_is_zero(&p.resv, sizeof(p.resv)))
+	if (!mem_is_zero(&config.p.resv, sizeof(config.p.resv)))
 		return -EINVAL;
 
-	p.sq_entries = entries;
-	return io_uring_create(&p, params);
+	config.p.sq_entries = entries;
+	config.uptr = params;
+	return io_uring_create(&config);
 }
 
 static inline int io_uring_allowed(void)
io_uring/io_uring.h +5 -0
index b2251446497a..c4d47ad7777c 100644
@@ -17,6 +17,11 @@
 #include <trace/events/io_uring.h>
 #endif
 
+struct io_ctx_config {
+	struct io_uring_params p;
+	struct io_uring_params __user *uptr;
+};
+
 #define IORING_FEAT_FLAGS (IORING_FEAT_SINGLE_MMAP |\
 			IORING_FEAT_NODROP |\
 			IORING_FEAT_SUBMIT_STABLE |\
Signature · 1 line
2.49.0
#6 lore raw

RFC 6/16 io_uring: split out config init helper

In reply to Pavel Begunkov

Separate most of configuration verification / calculation into a function, that will be the first step before doing any actual allocation.

---
 io_uring/io_uring.c | 22 +++++++++++++++++-----
 1 file changed, 17 insertions(+), 5 deletions(-)
io_uring/io_uring.c +17 -5
index 142811c7a4f5..30ba60974f1d 100644
@@ -3552,12 +3552,9 @@ int io_uring_fill_params(struct io_uring_params *p)
 	return 0;
 }
 
-static __cold int io_uring_create(struct io_ctx_config *config)
+static int io_prepare_config(struct io_ctx_config *config)
 {
 	struct io_uring_params *p = &config->p;
-	struct io_ring_ctx *ctx;
-	struct io_uring_task *tctx;
-	struct file *file;
 	int ret;
 
 	ret = io_uring_sanitise_params(p);
@@ -3565,7 +3562,22 @@ static __cold int io_uring_create(struct io_ctx_config *config)
 		return ret;
 
 	ret = io_uring_fill_params(p);
-	if (unlikely(ret))
+	if (ret)
+		return ret;
+
+	return 0;
+}
+
+static __cold int io_uring_create(struct io_ctx_config *config)
+{
+	struct io_uring_params *p = &config->p;
+	struct io_ring_ctx *ctx;
+	struct io_uring_task *tctx;
+	struct file *file;
+	int ret;
+
+	ret = io_prepare_config(config);
+	if (ret)
 		return ret;
 
 	ctx = io_ring_ctx_alloc(p);
Signature · 1 line
2.49.0
#7 lore raw

RFC 7/16 io_uring: add structure keeping ring offsets

In reply to Pavel Begunkov

Add struct io_scq_dim that keeps all offset / size / dimension information about the rings, and let rings_size() initialise it. It improves calculation locality and allows to dedup some code.

---
 io_uring/io_uring.c | 60 ++++++++++++++++++++++++---------------------
 io_uring/io_uring.h | 12 +++++++--
 io_uring/register.c | 19 ++++++--------
 3 files changed, 49 insertions(+), 42 deletions(-)
io_uring/io_uring.c +32 -28
index 30ba60974f1d..8166ea9140f8 100644
@@ -2757,49 +2757,61 @@ static void io_rings_free(struct io_ring_ctx *ctx)
 	ctx->sq_sqes = NULL;
 }
 
-unsigned long rings_size(unsigned int flags, unsigned int sq_entries,
-			 unsigned int cq_entries, size_t *sq_offset)
+int rings_size(unsigned int flags, unsigned int sq_entries,
+	       unsigned int cq_entries, struct io_scq_dim *dims)
 {
 	struct io_rings *rings;
 	size_t off, sq_array_size;
+	size_t sqe_size;
+
+	dims->sq_array_offset = SIZE_MAX;
+
+	sqe_size = sizeof(struct io_uring_sqe);
+	if (flags & IORING_SETUP_SQE128)
+		sqe_size *= 2;
+
+	dims->sq_size = array_size(sqe_size, sq_entries);
+	if (dims->sq_size == SIZE_MAX)
+		return -EOVERFLOW;
 
 	off = struct_size(rings, cqes, cq_entries);
 	if (off == SIZE_MAX)
-		return SIZE_MAX;
+		return -EOVERFLOW;
 	if (flags & IORING_SETUP_CQE32) {
 		if (check_shl_overflow(off, 1, &off))
-			return SIZE_MAX;
+			return -EOVERFLOW;
 	}
 	if (flags & IORING_SETUP_CQE_MIXED) {
 		if (cq_entries < 2)
-			return SIZE_MAX;
+			return -EOVERFLOW;
 	}
 	if (flags & IORING_SETUP_SQE_MIXED) {
 		if (sq_entries < 2)
-			return SIZE_MAX;
+			return -EOVERFLOW;
 	}
 
 #ifdef CONFIG_SMP
 	off = ALIGN(off, SMP_CACHE_BYTES);
 	if (off == 0)
-		return SIZE_MAX;
+		return -EOVERFLOW;
 #endif
 
 	if (flags & IORING_SETUP_NO_SQARRAY) {
-		*sq_offset = SIZE_MAX;
-		return off;
+		dims->cq_comp_size = off;
+		return 0;
 	}
 
-	*sq_offset = off;
+	dims->sq_array_offset = off;
 
 	sq_array_size = array_size(sizeof(u32), sq_entries);
 	if (sq_array_size == SIZE_MAX)
-		return SIZE_MAX;
+		return -EOVERFLOW;
 
 	if (check_add_overflow(off, sq_array_size, &off))
-		return SIZE_MAX;
+		return -EOVERFLOW;
 
-	return off;
+	dims->cq_comp_size = off;
+	return 0;
 }
 
 static __cold void __io_req_caches_free(struct io_ring_ctx *ctx)
@@ -3354,27 +3366,19 @@ static __cold int io_allocate_scq_urings(struct io_ring_ctx *ctx,
 {
 	struct io_uring_region_desc rd;
 	struct io_rings *rings;
-	size_t sq_array_offset;
-	size_t sq_size, cq_size, sqe_size;
+	struct io_scq_dim dims;
 	int ret;
 
 	/* make sure these are sane, as we already accounted them */
 	ctx->sq_entries = p->sq_entries;
 	ctx->cq_entries = p->cq_entries;
 
-	sqe_size = sizeof(struct io_uring_sqe);
-	if (p->flags & IORING_SETUP_SQE128)
-		sqe_size *= 2;
-	sq_size = array_size(sqe_size, p->sq_entries);
-	if (sq_size == SIZE_MAX)
-		return -EOVERFLOW;
-	cq_size = rings_size(ctx->flags, p->sq_entries, p->cq_entries,
-			  &sq_array_offset);
-	if (cq_size == SIZE_MAX)
-		return -EOVERFLOW;
+	ret = rings_size(ctx->flags, p->sq_entries, p->cq_entries, &dims);
+	if (ret)
+		return ret;
 
 	memset(&rd, 0, sizeof(rd));
-	rd.size = PAGE_ALIGN(cq_size);
+	rd.size = PAGE_ALIGN(dims.cq_comp_size);
 	if (ctx->flags & IORING_SETUP_NO_MMAP) {
 		rd.user_addr = p->cq_off.user_addr;
 		rd.flags |= IORING_MEM_REGION_TYPE_USER;
@@ -3385,10 +3389,10 @@ static __cold int io_allocate_scq_urings(struct io_ring_ctx *ctx,
 	ctx->rings = rings = io_region_get_ptr(&ctx->ring_region);
 
 	if (!(ctx->flags & IORING_SETUP_NO_SQARRAY))
-		ctx->sq_array = (u32 *)((char *)rings + sq_array_offset);
+		ctx->sq_array = (u32 *)((char *)rings + dims.sq_array_offset);
 
 	memset(&rd, 0, sizeof(rd));
-	rd.size = PAGE_ALIGN(sq_size);
+	rd.size = PAGE_ALIGN(dims.sq_size);
 	if (ctx->flags & IORING_SETUP_NO_MMAP) {
 		rd.user_addr = p->sq_off.user_addr;
 		rd.flags |= IORING_MEM_REGION_TYPE_USER;
io_uring/io_uring.h +10 -2
index c4d47ad7777c..29464be9733c 100644
@@ -17,6 +17,14 @@
 #include <trace/events/io_uring.h>
 #endif
 
+struct io_scq_dim {
+	size_t sq_array_offset;
+	size_t sq_size;
+
+	/* Compound array mmap'ed together with CQ. */
+	size_t cq_comp_size;
+};
+
 struct io_ctx_config {
 	struct io_uring_params p;
 	struct io_uring_params __user *uptr;
@@ -139,8 +147,8 @@ static inline bool io_should_wake(struct io_wait_queue *iowq)
 #define IORING_MAX_ENTRIES	32768
 #define IORING_MAX_CQ_ENTRIES	(2 * IORING_MAX_ENTRIES)
 
-unsigned long rings_size(unsigned int flags, unsigned int sq_entries,
-			 unsigned int cq_entries, size_t *sq_offset);
+int rings_size(unsigned int flags, unsigned int sq_entries,
+	       unsigned int cq_entries, struct io_scq_dim *dims);
 int io_uring_fill_params(struct io_uring_params *p);
 bool io_cqe_cache_refill(struct io_ring_ctx *ctx, bool overflow, bool cqe32);
 int io_run_task_work_sig(struct io_ring_ctx *ctx);
io_uring/register.c +7 -12
index 0d70696468f6..85814f983dde 100644
@@ -402,6 +402,7 @@ static int io_register_resize_rings(struct io_ring_ctx *ctx, void __user *arg)
 	struct io_ring_ctx_rings o = { }, n = { }, *to_free = NULL;
 	size_t size, sq_array_offset;
 	unsigned i, tail, old_head;
+	struct io_scq_dim dims;
 	struct io_uring_params p;
 	int ret;
 
@@ -419,11 +420,12 @@ static int io_register_resize_rings(struct io_ring_ctx *ctx, void __user *arg)
 	ret = io_uring_fill_params(&p);
 	if (unlikely(ret))
 		return ret;
+	ret = rings_size(p.flags, p.sq_entries, p.cq_entries, &dims);
+	if (ret)
+		return ret;
 
-	size = rings_size(p.flags, p.sq_entries, p.cq_entries,
-				&sq_array_offset);
-	if (size == SIZE_MAX)
-		return -EOVERFLOW;
+	size = dims.cq_comp_size;
+	sq_array_offset = dims.sq_array_offset;
 
 	memset(&rd, 0, sizeof(rd));
 	rd.size = PAGE_ALIGN(size);
@@ -455,14 +457,7 @@ static int io_register_resize_rings(struct io_ring_ctx *ctx, void __user *arg)
 		return -EFAULT;
 	}
 
-	if (p.flags & IORING_SETUP_SQE128)
-		size = array_size(2 * sizeof(struct io_uring_sqe), p.sq_entries);
-	else
-		size = array_size(sizeof(struct io_uring_sqe), p.sq_entries);
-	if (size == SIZE_MAX) {
-		io_register_free_rings(ctx, &n);
-		return -EOVERFLOW;
-	}
+	size = dims.sq_size;
 
 	memset(&rd, 0, sizeof(rd));
 	rd.size = PAGE_ALIGN(size);
Signature · 1 line
2.49.0
#8 lore raw

RFC 8/16 io_uring: pre-calculate scq offsets

In reply to Pavel Begunkov

Move ring size / offset calculations into io_prepare_config(). It simplifies misconfiguration handling, keeps it local, allows to move p->sq_off.array calculation closer to the rest of p->sq_off init.

---
 io_uring/io_uring.c | 28 ++++++++++++++--------------
 io_uring/io_uring.h |  1 +
 2 files changed, 15 insertions(+), 14 deletions(-)
io_uring/io_uring.c +14 -14
index 8166ea9140f8..aeb9555bd258 100644
@@ -3362,23 +3362,20 @@ bool io_is_uring_fops(struct file *file)
 }
 
 static __cold int io_allocate_scq_urings(struct io_ring_ctx *ctx,
-					 struct io_uring_params *p)
+					 struct io_ctx_config *config)
 {
+	struct io_uring_params *p = &config->p;
+	struct io_scq_dim *dims = &config->dims;
 	struct io_uring_region_desc rd;
 	struct io_rings *rings;
-	struct io_scq_dim dims;
 	int ret;
 
 	/* make sure these are sane, as we already accounted them */
 	ctx->sq_entries = p->sq_entries;
 	ctx->cq_entries = p->cq_entries;
 
-	ret = rings_size(ctx->flags, p->sq_entries, p->cq_entries, &dims);
-	if (ret)
-		return ret;
-
 	memset(&rd, 0, sizeof(rd));
-	rd.size = PAGE_ALIGN(dims.cq_comp_size);
+	rd.size = PAGE_ALIGN(dims->cq_comp_size);
 	if (ctx->flags & IORING_SETUP_NO_MMAP) {
 		rd.user_addr = p->cq_off.user_addr;
 		rd.flags |= IORING_MEM_REGION_TYPE_USER;
@@ -3387,12 +3384,11 @@ static __cold int io_allocate_scq_urings(struct io_ring_ctx *ctx,
 	if (ret)
 		return ret;
 	ctx->rings = rings = io_region_get_ptr(&ctx->ring_region);
-
 	if (!(ctx->flags & IORING_SETUP_NO_SQARRAY))
-		ctx->sq_array = (u32 *)((char *)rings + dims.sq_array_offset);
+		ctx->sq_array = (u32 *)((char *)rings + dims->sq_array_offset);
 
 	memset(&rd, 0, sizeof(rd));
-	rd.size = PAGE_ALIGN(dims.sq_size);
+	rd.size = PAGE_ALIGN(dims->sq_size);
 	if (ctx->flags & IORING_SETUP_NO_MMAP) {
 		rd.user_addr = p->sq_off.user_addr;
 		rd.flags |= IORING_MEM_REGION_TYPE_USER;
@@ -3569,6 +3565,13 @@ static int io_prepare_config(struct io_ctx_config *config)
 	if (ret)
 		return ret;
 
+	ret = rings_size(p->flags, p->sq_entries, p->cq_entries, &config->dims);
+	if (ret)
+		return ret;
+
+	if (!(p->flags & IORING_SETUP_NO_SQARRAY))
+		p->sq_off.array = config->dims.sq_array_offset;
+
 	return 0;
 }
 
@@ -3641,13 +3644,10 @@ static __cold int io_uring_create(struct io_ctx_config *config)
 	mmgrab(current->mm);
 	ctx->mm_account = current->mm;
 
-	ret = io_allocate_scq_urings(ctx, p);
+	ret = io_allocate_scq_urings(ctx, config);
 	if (ret)
 		goto err;
 
-	if (!(p->flags & IORING_SETUP_NO_SQARRAY))
-		p->sq_off.array = (char *)ctx->sq_array - (char *)ctx->rings;
-
 	ret = io_sq_offload_create(ctx, p);
 	if (ret)
 		goto err;
io_uring/io_uring.h +1 -0
index 29464be9733c..d1c2c70720f1 100644
@@ -27,6 +27,7 @@ struct io_scq_dim {
 
 struct io_ctx_config {
 	struct io_uring_params p;
+	struct io_scq_dim dims;
 	struct io_uring_params __user *uptr;
 };
 
Signature · 1 line
2.49.0
#9 lore raw

RFC 9/16 io_uring: inroduce helper for setting user offset

In reply to Pavel Begunkov

struct io_[c,s]qring_offsets may require computation from other steps like in case of sq_off.array. Move the initialisation out of io_uring_fill_params() into a separate function that can be called later.

---
 io_uring/io_uring.c | 52 ++++++++++++++++++++++++---------------------
 io_uring/io_uring.h |  1 +
 io_uring/register.c |  1 +
 3 files changed, 30 insertions(+), 24 deletions(-)
io_uring/io_uring.c +28 -24
index aeb9555bd258..be866a8e94bf 100644
@@ -3487,6 +3487,33 @@ static int io_uring_sanitise_params(struct io_uring_params *p)
 	return 0;
 }
 
+void io_fill_scq_offsets(struct io_uring_params *p, struct io_scq_dim *dims)
+{
+	p->sq_off.head = offsetof(struct io_rings, sq.head);
+	p->sq_off.tail = offsetof(struct io_rings, sq.tail);
+	p->sq_off.ring_mask = offsetof(struct io_rings, sq_ring_mask);
+	p->sq_off.ring_entries = offsetof(struct io_rings, sq_ring_entries);
+	p->sq_off.flags = offsetof(struct io_rings, sq_flags);
+	p->sq_off.dropped = offsetof(struct io_rings, sq_dropped);
+	p->sq_off.resv1 = 0;
+	if (!(p->flags & IORING_SETUP_NO_MMAP))
+		p->sq_off.user_addr = 0;
+
+	p->cq_off.head = offsetof(struct io_rings, cq.head);
+	p->cq_off.tail = offsetof(struct io_rings, cq.tail);
+	p->cq_off.ring_mask = offsetof(struct io_rings, cq_ring_mask);
+	p->cq_off.ring_entries = offsetof(struct io_rings, cq_ring_entries);
+	p->cq_off.overflow = offsetof(struct io_rings, cq_overflow);
+	p->cq_off.cqes = offsetof(struct io_rings, cqes);
+	p->cq_off.flags = offsetof(struct io_rings, cq_flags);
+	p->cq_off.resv1 = 0;
+	if (!(p->flags & IORING_SETUP_NO_MMAP))
+		p->cq_off.user_addr = 0;
+
+	if (!(p->flags & IORING_SETUP_NO_SQARRAY))
+		p->sq_off.array = dims->sq_array_offset;
+}
+
 int io_uring_fill_params(struct io_uring_params *p)
 {
 	unsigned entries = p->sq_entries;
@@ -3528,27 +3555,6 @@ int io_uring_fill_params(struct io_uring_params *p)
 		p->cq_entries = 2 * p->sq_entries;
 	}
 
-	p->sq_off.head = offsetof(struct io_rings, sq.head);
-	p->sq_off.tail = offsetof(struct io_rings, sq.tail);
-	p->sq_off.ring_mask = offsetof(struct io_rings, sq_ring_mask);
-	p->sq_off.ring_entries = offsetof(struct io_rings, sq_ring_entries);
-	p->sq_off.flags = offsetof(struct io_rings, sq_flags);
-	p->sq_off.dropped = offsetof(struct io_rings, sq_dropped);
-	p->sq_off.resv1 = 0;
-	if (!(p->flags & IORING_SETUP_NO_MMAP))
-		p->sq_off.user_addr = 0;
-
-	p->cq_off.head = offsetof(struct io_rings, cq.head);
-	p->cq_off.tail = offsetof(struct io_rings, cq.tail);
-	p->cq_off.ring_mask = offsetof(struct io_rings, cq_ring_mask);
-	p->cq_off.ring_entries = offsetof(struct io_rings, cq_ring_entries);
-	p->cq_off.overflow = offsetof(struct io_rings, cq_overflow);
-	p->cq_off.cqes = offsetof(struct io_rings, cqes);
-	p->cq_off.flags = offsetof(struct io_rings, cq_flags);
-	p->cq_off.resv1 = 0;
-	if (!(p->flags & IORING_SETUP_NO_MMAP))
-		p->cq_off.user_addr = 0;
-
 	return 0;
 }
 
@@ -3569,9 +3575,7 @@ static int io_prepare_config(struct io_ctx_config *config)
 	if (ret)
 		return ret;
 
-	if (!(p->flags & IORING_SETUP_NO_SQARRAY))
-		p->sq_off.array = config->dims.sq_array_offset;
-
+	io_fill_scq_offsets(p, &config->dims);
 	return 0;
 }
 
io_uring/io_uring.h +1 -0
index d1c2c70720f1..f6c4b141a33d 100644
@@ -151,6 +151,7 @@ static inline bool io_should_wake(struct io_wait_queue *iowq)
 int rings_size(unsigned int flags, unsigned int sq_entries,
 	       unsigned int cq_entries, struct io_scq_dim *dims);
 int io_uring_fill_params(struct io_uring_params *p);
+void io_fill_scq_offsets(struct io_uring_params *p, struct io_scq_dim *dims);
 bool io_cqe_cache_refill(struct io_ring_ctx *ctx, bool overflow, bool cqe32);
 int io_run_task_work_sig(struct io_ring_ctx *ctx);
 int io_run_local_work(struct io_ring_ctx *ctx, int min_events, int max_events);
io_uring/register.c +1 -0
index 85814f983dde..da804f925622 100644
@@ -423,6 +423,7 @@ static int io_register_resize_rings(struct io_ring_ctx *ctx, void __user *arg)
 	ret = rings_size(p.flags, p.sq_entries, p.cq_entries, &dims);
 	if (ret)
 		return ret;
+	io_fill_scq_offsets(&p, &dims);
 
 	size = dims.cq_comp_size;
 	sq_array_offset = dims.sq_array_offset;
Signature · 1 line
2.49.0
#10 lore raw

RFC 10/16 io_uring: separate cqe array from headers

In reply to Pavel Begunkov

Keep a pointer to the CQ separate from SCQ headers, it'll be used shortly in next patches. Also, don't overestimate the CQ size for SETUP_CQE32, which not only doubles memory for CQ entries but also the headers as well.

---
 include/linux/io_uring_types.h | 17 +++++++++--------
 io_uring/fdinfo.c              |  2 +-
 io_uring/io_uring.c            | 35 ++++++++++++++++++++++------------
 io_uring/io_uring.h            |  1 +
 io_uring/register.c            |  8 +++++++-
 5 files changed, 41 insertions(+), 22 deletions(-)
include/linux/io_uring_types.h +9 -8
index 92780764d5fa..91ded559a147 100644
@@ -209,14 +209,6 @@ struct io_rings {
 	 * ordered with any other data.
 	 */
 	u32			cq_overflow;
-	/*
-	 * Ring buffer of completion events.
-	 *
-	 * The kernel writes completion events fresh every time they are
-	 * produced, so the application is allowed to modify pending
-	 * entries.
-	 */
-	struct io_uring_cqe	cqes[] ____cacheline_aligned_in_smp;
 };
 
 struct io_restriction {
@@ -274,6 +266,15 @@ struct io_ring_ctx {
 
 		struct task_struct	*submitter_task;
 		struct io_rings		*rings;
+		/*
+		 * Ring buffer of completion events.
+		 *
+		 * The kernel writes completion events fresh every time they are
+		 * produced, so the application is allowed to modify pending
+		 * entries.
+		 */
+		struct io_uring_cqe	*cqes;
+
 		struct percpu_ref	refs;
 
 		clockid_t		clockid;
io_uring/fdinfo.c +1 -1
index ac6e7edc7027..eae13ac9b1a9 100644
@@ -153,7 +153,7 @@ static void __io_uring_show_fdinfo(struct io_ring_ctx *ctx, struct seq_file *m)
 		struct io_uring_cqe *cqe;
 		bool cqe32 = false;
 
-		cqe = &r->cqes[(cq_head & cq_mask)];
+		cqe = &ctx->cqes[(cq_head & cq_mask)];
 		if (cqe->flags & IORING_CQE_F_32 || ctx->flags & IORING_SETUP_CQE32)
 			cqe32 = true;
 		seq_printf(m, "%5u: user_data:%llu, res:%d, flag:%x",
io_uring/io_uring.c +23 -12
index be866a8e94bf..9aef41f6ce23 100644
@@ -745,7 +745,7 @@ static struct io_overflow_cqe *io_alloc_ocqe(struct io_ring_ctx *ctx,
 static bool io_fill_nop_cqe(struct io_ring_ctx *ctx, unsigned int off)
 {
 	if (__io_cqring_events(ctx) < ctx->cq_entries) {
-		struct io_uring_cqe *cqe = &ctx->rings->cqes[off];
+		struct io_uring_cqe *cqe = &ctx->cqes[off];
 
 		cqe->user_data = 0;
 		cqe->res = 0;
@@ -763,7 +763,6 @@ static bool io_fill_nop_cqe(struct io_ring_ctx *ctx, unsigned int off)
  */
 bool io_cqe_cache_refill(struct io_ring_ctx *ctx, bool overflow, bool cqe32)
 {
-	struct io_rings *rings = ctx->rings;
 	unsigned int off = ctx->cached_cq_tail & (ctx->cq_entries - 1);
 	unsigned int free, queued, len;
 
@@ -798,7 +797,7 @@ bool io_cqe_cache_refill(struct io_ring_ctx *ctx, bool overflow, bool cqe32)
 		len <<= 1;
 	}
 
-	ctx->cqe_cached = &rings->cqes[off];
+	ctx->cqe_cached = &ctx->cqes[off];
 	ctx->cqe_sentinel = ctx->cqe_cached + len;
 	return true;
 }
@@ -2760,8 +2759,8 @@ static void io_rings_free(struct io_ring_ctx *ctx)
 int rings_size(unsigned int flags, unsigned int sq_entries,
 	       unsigned int cq_entries, struct io_scq_dim *dims)
 {
-	struct io_rings *rings;
 	size_t off, sq_array_size;
+	size_t cq_size, cqe_size;
 	size_t sqe_size;
 
 	dims->sq_array_offset = SIZE_MAX;
@@ -2769,18 +2768,26 @@ int rings_size(unsigned int flags, unsigned int sq_entries,
 	sqe_size = sizeof(struct io_uring_sqe);
 	if (flags & IORING_SETUP_SQE128)
 		sqe_size *= 2;
+	cqe_size = sizeof(struct io_uring_cqe);
+	if (flags & IORING_SETUP_CQE32)
+		cqe_size *= 2;
 
 	dims->sq_size = array_size(sqe_size, sq_entries);
 	if (dims->sq_size == SIZE_MAX)
 		return -EOVERFLOW;
 
-	off = struct_size(rings, cqes, cq_entries);
+	off = sizeof(struct io_rings);
+	off = L1_CACHE_ALIGN(off);
+	dims->cq_offset = off;
+
+	cq_size = array_size(cqe_size, cq_entries);
+	if (cq_size == SIZE_MAX)
+		return -EOVERFLOW;
+
+	off = size_add(off, cq_size);
 	if (off == SIZE_MAX)
 		return -EOVERFLOW;
-	if (flags & IORING_SETUP_CQE32) {
-		if (check_shl_overflow(off, 1, &off))
-			return -EOVERFLOW;
-	}
+
 	if (flags & IORING_SETUP_CQE_MIXED) {
 		if (cq_entries < 2)
 			return -EOVERFLOW;
@@ -3368,6 +3375,7 @@ static __cold int io_allocate_scq_urings(struct io_ring_ctx *ctx,
 	struct io_scq_dim *dims = &config->dims;
 	struct io_uring_region_desc rd;
 	struct io_rings *rings;
+	void *ptr;
 	int ret;
 
 	/* make sure these are sane, as we already accounted them */
@@ -3383,9 +3391,12 @@ static __cold int io_allocate_scq_urings(struct io_ring_ctx *ctx,
 	ret = io_create_region(ctx, &ctx->ring_region, &rd, IORING_OFF_CQ_RING);
 	if (ret)
 		return ret;
-	ctx->rings = rings = io_region_get_ptr(&ctx->ring_region);
+	ptr = io_region_get_ptr(&ctx->ring_region);
+	ctx->rings = rings = ptr;
+	ctx->cqes = ptr + config->dims.cq_offset;
+
 	if (!(ctx->flags & IORING_SETUP_NO_SQARRAY))
-		ctx->sq_array = (u32 *)((char *)rings + dims->sq_array_offset);
+		ctx->sq_array = ptr + dims->sq_array_offset;
 
 	memset(&rd, 0, sizeof(rd));
 	rd.size = PAGE_ALIGN(dims->sq_size);
@@ -3504,7 +3515,7 @@ void io_fill_scq_offsets(struct io_uring_params *p, struct io_scq_dim *dims)
 	p->cq_off.ring_mask = offsetof(struct io_rings, cq_ring_mask);
 	p->cq_off.ring_entries = offsetof(struct io_rings, cq_ring_entries);
 	p->cq_off.overflow = offsetof(struct io_rings, cq_overflow);
-	p->cq_off.cqes = offsetof(struct io_rings, cqes);
+	p->cq_off.cqes = dims->cq_offset;
 	p->cq_off.flags = offsetof(struct io_rings, cq_flags);
 	p->cq_off.resv1 = 0;
 	if (!(p->flags & IORING_SETUP_NO_MMAP))
io_uring/io_uring.h +1 -0
index f6c4b141a33d..80228c5a843c 100644
@@ -20,6 +20,7 @@
 struct io_scq_dim {
 	size_t sq_array_offset;
 	size_t sq_size;
+	size_t cq_offset;
 
 	/* Compound array mmap'ed together with CQ. */
 	size_t cq_comp_size;
io_uring/register.c +7 -1
index da804f925622..b43a121e2974 100644
@@ -373,6 +373,7 @@ static int io_register_clock(struct io_ring_ctx *ctx,
 struct io_ring_ctx_rings {
 	struct io_rings *rings;
 	struct io_uring_sqe *sq_sqes;
+	struct io_uring_cqe *cqes;
 
 	struct io_mapped_region sq_region;
 	struct io_mapped_region ring_region;
@@ -439,6 +440,7 @@ static int io_register_resize_rings(struct io_ring_ctx *ctx, void __user *arg)
 		return ret;
 
 	n.rings = io_region_get_ptr(&n.ring_region);
+	n.cqes = io_region_get_ptr(&n.ring_region) + dims.cq_offset;
 
 	/*
 	 * At this point n.rings is shared with userspace, just like o.rings
@@ -497,6 +499,8 @@ static int io_register_resize_rings(struct io_ring_ctx *ctx, void __user *arg)
 	ctx->rings = NULL;
 	o.sq_sqes = ctx->sq_sqes;
 	ctx->sq_sqes = NULL;
+	o.cqes = ctx->cqes;
+	ctx->cqes = NULL;
 
 	/*
 	 * Now copy SQ and CQ entries, if any. If either of the destination
@@ -522,6 +526,7 @@ static int io_register_resize_rings(struct io_ring_ctx *ctx, void __user *arg)
 		/* restore old rings, and return -EOVERFLOW via cleanup path */
 		ctx->rings = o.rings;
 		ctx->sq_sqes = o.sq_sqes;
+		ctx->cqes = o.cqes;
 		to_free = &n;
 		ret = -EOVERFLOW;
 		goto out;
@@ -530,7 +535,7 @@ static int io_register_resize_rings(struct io_ring_ctx *ctx, void __user *arg)
 		unsigned src_head = i & (ctx->cq_entries - 1);
 		unsigned dst_head = i & (p.cq_entries - 1);
 
-		n.rings->cqes[dst_head] = o.rings->cqes[src_head];
+		n.cqes[dst_head] = o.cqes[src_head];
 	}
 	WRITE_ONCE(n.rings->cq.head, old_head);
 	WRITE_ONCE(n.rings->cq.tail, tail);
@@ -551,6 +556,7 @@ static int io_register_resize_rings(struct io_ring_ctx *ctx, void __user *arg)
 
 	ctx->rings = n.rings;
 	ctx->sq_sqes = n.sq_sqes;
+	ctx->cqes = n.cqes;
 	swap_old(ctx, o, n, ring_region);
 	swap_old(ctx, o, n, sq_region);
 	to_free = &o;
Signature · 1 line
2.49.0
#11 lore raw

RFC 11/16 io_uring/region: introduce io_region_slice

In reply to Pavel Begunkov

Add a new helper function that returns a sub-slice of a region memory from an {offset,size} pair and performs a bunch of extra sanitisation. It'll be used later for (slow path) ring setup.

---
 io_uring/memmap.c | 12 ++++++++++++
 io_uring/memmap.h |  2 ++
 2 files changed, 14 insertions(+)
io_uring/memmap.c +12 -0
index b329cee8d6e8..83faef350b9d 100644
@@ -232,6 +232,18 @@ int io_create_region(struct io_ring_ctx *ctx, struct io_mapped_region *mr,
 	return ret;
 }
 
+void *io_region_slice(struct io_mapped_region *mr, size_t off, size_t size)
+{
+	if (WARN_ON_ONCE(!size) || !io_region_is_set(mr))
+		return NULL;
+
+	size = size_add(off, size);
+	if (size == SIZE_MAX || size > io_region_size(mr))
+		return NULL;
+
+	return io_region_get_ptr(mr) + off;
+}
+
 static struct io_mapped_region *io_mmap_get_region(struct io_ring_ctx *ctx,
 						   loff_t pgoff)
 {
io_uring/memmap.h +2 -0
index d4b8b6363a7d..fa7a45cdb6dd 100644
@@ -21,6 +21,8 @@ int io_create_region(struct io_ring_ctx *ctx, struct io_mapped_region *mr,
 		     struct io_uring_region_desc *reg,
 		     unsigned long mmap_offset);
 
+void *io_region_slice(struct io_mapped_region *mr, size_t off, size_t size);
+
 static inline void *io_region_get_ptr(struct io_mapped_region *mr)
 {
 	return mr->ptr;
Signature · 1 line
2.49.0
#12 lore raw

RFC 12/16 io_uring: convert pointer init to io_region_slice

In reply to Pavel Begunkov

Use io_region_slice() to initialise ctx ring pointers. The helper performs bound checks and other sanitisation, which is safer and will be especially helpful when ring placement gets more complicated in coming patches. It also extends struct io_scq_dim with all intermediate offsets and sizes to fully describe rings.

---
 io_uring/io_uring.c | 47 ++++++++++++++++++++++++---------------------
 io_uring/io_uring.h |  2 ++
 2 files changed, 27 insertions(+), 22 deletions(-)
io_uring/io_uring.c +25 -22
index 9aef41f6ce23..4f38a0b587fd 100644
@@ -2759,9 +2759,7 @@ static void io_rings_free(struct io_ring_ctx *ctx)
 int rings_size(unsigned int flags, unsigned int sq_entries,
 	       unsigned int cq_entries, struct io_scq_dim *dims)
 {
-	size_t off, sq_array_size;
-	size_t cq_size, cqe_size;
-	size_t sqe_size;
+	size_t cqe_size, off, sqe_size;
 
 	dims->sq_array_offset = SIZE_MAX;
 
@@ -2773,18 +2771,18 @@ int rings_size(unsigned int flags, unsigned int sq_entries,
 		cqe_size *= 2;
 
 	dims->sq_size = array_size(sqe_size, sq_entries);
-	if (dims->sq_size == SIZE_MAX)
+	dims->sq_array_size = array_size(sizeof(u32), sq_entries);
+	dims->cq_size = array_size(cqe_size, cq_entries);
+
+	if (dims->sq_size == SIZE_MAX || dims->cq_size == SIZE_MAX ||
+	    dims->sq_array_size == SIZE_MAX)
 		return -EOVERFLOW;
 
 	off = sizeof(struct io_rings);
 	off = L1_CACHE_ALIGN(off);
 	dims->cq_offset = off;
 
-	cq_size = array_size(cqe_size, cq_entries);
-	if (cq_size == SIZE_MAX)
-		return -EOVERFLOW;
-
-	off = size_add(off, cq_size);
+	off = size_add(off, dims->cq_size);
 	if (off == SIZE_MAX)
 		return -EOVERFLOW;
 
@@ -2809,12 +2807,7 @@ int rings_size(unsigned int flags, unsigned int sq_entries,
 	}
 
 	dims->sq_array_offset = off;
-
-	sq_array_size = array_size(sizeof(u32), sq_entries);
-	if (sq_array_size == SIZE_MAX)
-		return -EOVERFLOW;
-
-	if (check_add_overflow(off, sq_array_size, &off))
+	if (check_add_overflow(off, dims->sq_array_size, &off))
 		return -EOVERFLOW;
 
 	dims->cq_comp_size = off;
@@ -3375,7 +3368,6 @@ static __cold int io_allocate_scq_urings(struct io_ring_ctx *ctx,
 	struct io_scq_dim *dims = &config->dims;
 	struct io_uring_region_desc rd;
 	struct io_rings *rings;
-	void *ptr;
 	int ret;
 
 	/* make sure these are sane, as we already accounted them */
@@ -3391,12 +3383,19 @@ static __cold int io_allocate_scq_urings(struct io_ring_ctx *ctx,
 	ret = io_create_region(ctx, &ctx->ring_region, &rd, IORING_OFF_CQ_RING);
 	if (ret)
 		return ret;
-	ptr = io_region_get_ptr(&ctx->ring_region);
-	ctx->rings = rings = ptr;
-	ctx->cqes = ptr + config->dims.cq_offset;
 
-	if (!(ctx->flags & IORING_SETUP_NO_SQARRAY))
-		ctx->sq_array = ptr + dims->sq_array_offset;
+	ctx->rings = io_region_slice(&ctx->ring_region, 0, sizeof(struct io_rings));
+	ctx->cqes = io_region_slice(&ctx->ring_region, dims->cq_offset, dims->cq_size);
+	if (!ctx->rings || !ctx->cqes)
+		return -EFAULT;
+
+	if (!(ctx->flags & IORING_SETUP_NO_SQARRAY)) {
+		ctx->sq_array = io_region_slice(&ctx->ring_region,
+						dims->sq_array_offset,
+						dims->sq_array_size);
+		if (!ctx->sq_array)
+			return -EFAULT;
+	}
 
 	memset(&rd, 0, sizeof(rd));
 	rd.size = PAGE_ALIGN(dims->sq_size);
@@ -3409,8 +3408,12 @@ static __cold int io_allocate_scq_urings(struct io_ring_ctx *ctx,
 		io_rings_free(ctx);
 		return ret;
 	}
-	ctx->sq_sqes = io_region_get_ptr(&ctx->sq_region);
 
+	ctx->sq_sqes = io_region_slice(&ctx->sq_region, 0, dims->sq_size);
+	if (!ctx->sq_sqes)
+		return -EFAULT;
+
+	rings = ctx->rings;
 	memset(rings, 0, sizeof(*rings));
 	WRITE_ONCE(rings->sq_ring_mask, ctx->sq_entries - 1);
 	WRITE_ONCE(rings->cq_ring_mask, ctx->cq_entries - 1);
io_uring/io_uring.h +2 -0
index 80228c5a843c..ed57ab4161db 100644
@@ -19,8 +19,10 @@
 
 struct io_scq_dim {
 	size_t sq_array_offset;
+	size_t sq_array_size;
 	size_t sq_size;
 	size_t cq_offset;
+	size_t cq_size;
 
 	/* Compound array mmap'ed together with CQ. */
 	size_t cq_comp_size;
Signature · 1 line
2.49.0
#13 lore raw

RFC 13/16 io_uring: refactor rings_size()

In reply to Pavel Begunkov

Refactor rings_size() to facilitate further changes. Move entries checks to the beginning, which localises offset calculation. And instead of special casing IORING_SETUP_NO_SQARRAY, reverse the check and just continue offset calculation. This way it assign cq_comp_size only once at the end.

---
 io_uring/io_uring.c | 31 +++++++++++++++----------------
 1 file changed, 15 insertions(+), 16 deletions(-)
io_uring/io_uring.c +15 -16
index 4f38a0b587fd..ff52d9f110ce 100644
@@ -2761,6 +2761,15 @@ int rings_size(unsigned int flags, unsigned int sq_entries,
 {
 	size_t cqe_size, off, sqe_size;
 
+	if (flags & IORING_SETUP_CQE_MIXED) {
+		if (cq_entries < 2)
+			return -EOVERFLOW;
+	}
+	if (flags & IORING_SETUP_SQE_MIXED) {
+		if (sq_entries < 2)
+			return -EOVERFLOW;
+	}
+
 	dims->sq_array_offset = SIZE_MAX;
 
 	sqe_size = sizeof(struct io_uring_sqe);
@@ -2786,29 +2795,19 @@ int rings_size(unsigned int flags, unsigned int sq_entries,
 	if (off == SIZE_MAX)
 		return -EOVERFLOW;
 
-	if (flags & IORING_SETUP_CQE_MIXED) {
-		if (cq_entries < 2)
-			return -EOVERFLOW;
-	}
-	if (flags & IORING_SETUP_SQE_MIXED) {
-		if (sq_entries < 2)
-			return -EOVERFLOW;
-	}
-
 #ifdef CONFIG_SMP
 	off = ALIGN(off, SMP_CACHE_BYTES);
 	if (off == 0)
 		return -EOVERFLOW;
 #endif
 
-	if (flags & IORING_SETUP_NO_SQARRAY) {
-		dims->cq_comp_size = off;
-		return 0;
-	}
+	if (!(flags & IORING_SETUP_NO_SQARRAY)) {
+		dims->sq_array_offset = off;
 
-	dims->sq_array_offset = off;
-	if (check_add_overflow(off, dims->sq_array_size, &off))
-		return -EOVERFLOW;
+		off = size_add(off, dims->sq_array_size);
+		if (off == SIZE_MAX)
+			return -EOVERFLOW;
+	}
 
 	dims->cq_comp_size = off;
 	return 0;
Signature · 1 line
2.49.0
#14 lore raw

RFC 14/16 io_uring: extract io_create_mem_region

In reply to Pavel Begunkov

Extract a helper function for creating a memory region but that can be used at setup and not only in io_uring_register(). Specifically, it doesn't check IORING_SETUP_R_DISABLED.

---
 io_uring/io_uring.h |  3 +++
 io_uring/register.c | 43 +++++++++++++++++++++++++------------------
 2 files changed, 28 insertions(+), 18 deletions(-)
io_uring/io_uring.h +3 -0
index ed57ab4161db..20f6ca4696c1 100644
@@ -200,6 +200,9 @@ void io_queue_next(struct io_kiocb *req);
 void io_task_refs_refill(struct io_uring_task *tctx);
 bool __io_alloc_req_refill(struct io_ring_ctx *ctx);
 
+int io_create_mem_region(struct io_ring_ctx *ctx,
+			 struct io_uring_mem_region_reg *reg);
+
 void io_activate_pollwq(struct io_ring_ctx *ctx);
 
 static inline void io_lockdep_assert_cq_locked(struct io_ring_ctx *ctx)
io_uring/register.c +25 -18

71 lines not on this page · raw message

Signature · 1 line
2.49.0
#15 lore raw

RFC 15/16 io_uring: allow creating mem region at setup

In reply to Pavel Begunkov

This patch gives users a way to create a memory region at ring setup instead of requiring a registration call. First, it can be much more convenient when the region is used for wait parameters and hence requires IORING_SETUP_R_DISABLED. I'll also need it in the next patch for placing SCQ into it.

---
 include/uapi/linux/io_uring.h |  7 ++++++-
 io_uring/io_uring.c           | 26 ++++++++++++++++++++++++++
 io_uring/io_uring.h           |  1 +
 io_uring/register.c           |  2 ++
 4 files changed, 35 insertions(+), 1 deletion(-)
include/uapi/linux/io_uring.h +6 -1
index 04797a9b76bc..2da052bd4138 100644
@@ -599,11 +599,16 @@ struct io_uring_params {
 	__u32 sq_thread_idle;
 	__u32 features;
 	__u32 wq_fd;
-	__u32 resv[3];
+	__u32 resv;
+	__u64 params_ext; /* pointer to struct io_uring_params_ext */
 	struct io_sqring_offsets sq_off;
 	struct io_cqring_offsets cq_off;
 };
 
+struct io_uring_params_ext {
+	__u64 mem_region; /* pointer to struct io_uring_mem_region_reg */
+};
+
 /*
  * io_uring_params->features flags
  */
io_uring/io_uring.c +26 -0
index ff52d9f110ce..908c432aaaaa 100644
@@ -3574,8 +3574,13 @@ int io_uring_fill_params(struct io_uring_params *p)
 static int io_prepare_config(struct io_ctx_config *config)
 {
 	struct io_uring_params *p = &config->p;
+	struct io_uring_params_ext __user *ext_user;
 	int ret;
 
+	ext_user = u64_to_user_ptr(config->p.params_ext);
+	if (ext_user && copy_from_user(&config->ext, ext_user, sizeof(config->ext)))
+		return -EFAULT;
+
 	ret = io_uring_sanitise_params(p);
 	if (ret)
 		return ret;
@@ -3592,6 +3597,22 @@ static int io_prepare_config(struct io_ctx_config *config)
 	return 0;
 }
 
+static int io_ctx_init_mem_region(struct io_ring_ctx *ctx,
+				  struct io_ctx_config *config)
+{
+	struct io_uring_params_ext *e = &config->ext;
+	struct io_uring_mem_region_reg __user *mr_user;
+	struct io_uring_mem_region_reg mr;
+
+	mr_user = u64_to_user_ptr(e->mem_region);
+	if (!mr_user)
+		return 0;
+
+	if (copy_from_user(&mr, mr_user, sizeof(mr)))
+		return -EFAULT;
+	return io_create_mem_region(ctx, &mr);
+}
+
 static __cold int io_uring_create(struct io_ctx_config *config)
 {
 	struct io_uring_params *p = &config->p;
@@ -3661,10 +3682,15 @@ static __cold int io_uring_create(struct io_ctx_config *config)
 	mmgrab(current->mm);
 	ctx->mm_account = current->mm;
 
+	ret = io_ctx_init_mem_region(ctx, config);
+	if (ret)
+		goto err;
+
 	ret = io_allocate_scq_urings(ctx, config);
 	if (ret)
 		goto err;
 
+
 	ret = io_sq_offload_create(ctx, p);
 	if (ret)
 		goto err;
io_uring/io_uring.h +1 -0
index 20f6ca4696c1..c883017b11d3 100644
@@ -30,6 +30,7 @@ struct io_scq_dim {
 
 struct io_ctx_config {
 	struct io_uring_params p;
+	struct io_uring_params_ext ext;
 	struct io_scq_dim dims;
 	struct io_uring_params __user *uptr;
 };
io_uring/register.c +2 -0
index 425529a30dd9..4affabc416aa 100644
@@ -414,6 +414,8 @@ static int io_register_resize_rings(struct io_ring_ctx *ctx, void __user *arg)
 		return -EFAULT;
 	if (p.flags & ~RESIZE_FLAGS)
 		return -EINVAL;
+	if (p.params_ext)
+		return -EINVAL;
 
 	/* properties that are always inherited */
 	p.flags |= (ctx->flags & COPY_FLAGS);
Signature · 1 line
2.49.0
#16 lore raw

RFC 16/16 io_uring: introduce SCQ placement

In reply to Pavel Begunkov

There is a repeated problem with how io_uring manages rings. Specifically, it creates a new memory region for each ring and places entries together with headers. As the number of entries is always a power of 2, it usually means that it needs to allocate an additional page just for headers, which is wasteful. The headers structure size is also usually small and under the cache line size, however it's padded, which might mean additional cache bouncing.

Introduce a way for the user space to overlap SCQ headers and/or rings onto a pre-registered memory/parameter region. Each of them has a separate flag / offset, and they'll be attempted to be placed at the specified offset in the region. If the user doesn't request placement for SQ and/or CQ, io_uring will create a new memory region for them as before.

The second goal is to be able to put all components into a single region while knowing what's placed where. It's specifically interesting for planned BPF work, as it makes program writing much simpler.

Note: zcrx also have the same issue, but it's left out of this series.

---
 include/uapi/linux/io_uring.h |  14 ++++
 io_uring/io_uring.c           | 143 ++++++++++++++++++++++++----------
 io_uring/io_uring.h           |  10 ++-
 io_uring/register.c           |   4 +-
 4 files changed, 128 insertions(+), 43 deletions(-)
include/uapi/linux/io_uring.h +14 -0
index 2da052bd4138..6574f0c6fc57 100644
@@ -605,8 +605,22 @@ struct io_uring_params {
 	struct io_cqring_offsets cq_off;
 };
 
+enum io_uring_scq_placement_flags {
+	IORING_PLACEMENT_SCQ_HDR		= (1U << 0),
+	IORING_PLACEMENT_SQ			= (1U << 1),
+	IORING_PLACEMENT_CQ			= (1U << 2),
+};
+
+struct io_uring_scq_placement {
+	__u64 flags;
+	__u64 scq_hdr_off;
+	__u64 sq_off;
+	__u64 cq_off;
+};
+
 struct io_uring_params_ext {
 	__u64 mem_region; /* pointer to struct io_uring_mem_region_reg */
+	struct io_uring_scq_placement placement;
 };
 
 /*
io_uring/io_uring.c +104 -39

209 lines not on this page · raw message

io_uring/io_uring.h +8 -2
index c883017b11d3..307710464cc4 100644
@@ -25,7 +25,8 @@ struct io_scq_dim {
 	size_t cq_size;
 
 	/* Compound array mmap'ed together with CQ. */
-	size_t cq_comp_size;
+	size_t rings_mr_size;
+	size_t sq_mr_size;
 };
 
 struct io_ctx_config {
@@ -35,6 +36,10 @@ struct io_ctx_config {
 	struct io_uring_params __user *uptr;
 };
 
+#define IORING_PLACEMENT_MASK (IORING_PLACEMENT_SCQ_HDR |\
+				IORING_PLACEMENT_SQ |\
+				IORING_PLACEMENT_CQ)
+
 #define IORING_FEAT_FLAGS (IORING_FEAT_SINGLE_MMAP |\
 			IORING_FEAT_NODROP |\
 			IORING_FEAT_SUBMIT_STABLE |\
@@ -153,7 +158,8 @@ static inline bool io_should_wake(struct io_wait_queue *iowq)
 #define IORING_MAX_CQ_ENTRIES	(2 * IORING_MAX_ENTRIES)
 
 int rings_size(unsigned int flags, unsigned int sq_entries,
-	       unsigned int cq_entries, struct io_scq_dim *dims);
+	       unsigned int cq_entries, struct io_scq_dim *dims,
+	       unsigned placement_flags);
 int io_uring_fill_params(struct io_uring_params *p);
 void io_fill_scq_offsets(struct io_uring_params *p, struct io_scq_dim *dims);
 bool io_cqe_cache_refill(struct io_ring_ctx *ctx, bool overflow, bool cqe32);
io_uring/register.c +2 -2
index 4affabc416aa..bbcb5a79a35f 100644
@@ -423,12 +423,12 @@ static int io_register_resize_rings(struct io_ring_ctx *ctx, void __user *arg)
 	ret = io_uring_fill_params(&p);
 	if (unlikely(ret))
 		return ret;
-	ret = rings_size(p.flags, p.sq_entries, p.cq_entries, &dims);
+	ret = rings_size(p.flags, p.sq_entries, p.cq_entries, &dims, 0);
 	if (ret)
 		return ret;
 	io_fill_scq_offsets(&p, &dims);
 
-	size = dims.cq_comp_size;
+	size = dims.rings_mr_size;
 	sq_array_offset = dims.sq_array_offset;
 
 	memset(&rd, 0, sizeof(rd));
Signature · 1 line
2.49.0