Introduce ring flexible placement
cover Introduce ring flexible placement
This patchset allows the user to put rings and ring headers into the memory/parameter region and specify offsets where they should lie.
It solves a problem where the headers are placed before ring entries, which 1. usually padded wasting memory and using more cache lines than a smarter placement would need. And 2. it grows the region size to a non pow2. It's also handy to have a way to put everything into a single region.
It's implemented for SQ and CQ, but planned to be supported by zcrx as well. There is a bunch of cleanups (1-13), it'd make sense to merge some of them separately. It also adds an ability to register a memory region during setup and not via a registration (Patch 15). And the placement handling is in Patch 16.
Pavel Begunkov (16):
io_uring: add helper calculating region byte size
io_uring: pass sq entires in the params struct
io_uring: use mem_is_zero to check ring params
io_uring: move flags check to io_uring_sanitise_params
io_uring: introduce struct io_ctx_config
io_uring: split out config init helper
io_uring: add structure keeping ring offsets
io_uring: pre-calculate scq offsets
io_uring: inroduce helper for setting user offset
io_uring: separate cqe array from headers io_uring/region: introduce io_region_slice
io_uring: convert pointer init to io_region_slice
io_uring: refactor rings_size()
io_uring: extract io_create_mem_region
io_uring: allow creating mem region at setup
io_uring: introduce SCQ placement
include/linux/io_uring_types.h | 17 +- include/uapi/linux/io_uring.h | 21 +- io_uring/fdinfo.c | 2 +- io_uring/io_uring.c | 341 +++++++++++++++++++++++---------- io_uring/io_uring.h | 34 +++- io_uring/memmap.c | 16 +- io_uring/memmap.h | 7 + io_uring/register.c | 75 ++++---- 8 files changed, 360 insertions(+), 153 deletions(-)
Signature · 1 line
2.49.0
RFC 1/16 io_uring: add helper calculating region byte size
There has been type related issues with region size calculation, add an utility helper function that returns the size and handles type conversions right.
--- io_uring/memmap.c | 4 ++-- io_uring/memmap.h | 5 +++++ 2 files changed, 7 insertions(+), 2 deletions(-)
io_uring/memmap.c +2 -2
@@ -135,7 +135,7 @@ static int io_region_pin_pages(struct io_ring_ctx *ctx,struct io_mapped_region *mr, struct io_uring_region_desc *reg) { - unsigned long size = (size_t)mr->nr_pages << PAGE_SHIFT; + size_t size = io_region_size(mr);struct page **pages;int nr_pages;
@@ -156,7 +156,7 @@ static int io_region_allocate_pages(struct io_ring_ctx *ctx, unsigned long mmap_offset) {gfp_t gfp =GFP_KERNEL_ACCOUNT | __GFP_ZERO | __GFP_NOWARN; -size_t size = (size_t ) mr->nr_pages <<PAGE_SHIFT ; +size_t size =io_region_size (mr);unsigned long nr_allocated;struct page **pages;
io_uring/memmap.h +5 -0
@@ -43,4 +43,9 @@ static inline void io_region_publish(struct io_ring_ctx *ctx, *dst_region = *src_region; } +static inline size_t io_region_size (struct io_mapped_region *mr) +{ +return (size_t )mr->nr_pages <<PAGE_SHIFT ; +} +#endif
Signature · 1 line
2.49.0
RFC 2/16 io_uring: pass sq entires in the params struct
In reply to Pavel BegunkovThere is no need to pass the user requested number of SQ entries separately from the main parameter structure io_uring_params. Initialise it at the beginning and stop passing it in favour of struct io_uring_params::sq_entries.
--- io_uring/io_uring.c | 11 +++++++---- io_uring/io_uring.h | 2 +- io_uring/register.c | 2 +- 3 files changed, 9 insertions(+), 6 deletions(-)
io_uring/io_uring.c +7 -4
@@ -3484,8 +3484,10 @@ static int io_uring_sanitise_params(struct io_uring_params *p)return 0 ; } -int io_uring_fill_params (unsigned entries,struct io_uring_params *p) +int io_uring_fill_params (struct io_uring_params *p) { +unsigned entries = p->sq_entries; +if (!entries)return -EINVAL ;if (entries >IORING_MAX_ENTRIES ) {
@@ -3547,7 +3549,7 @@ int io_uring_fill_params(unsigned entries, struct io_uring_params *p)return 0 ; } -static __cold int io_uring_create (unsigned entries,struct io_uring_params *p, +static __cold int io_uring_create (struct io_uring_params *p,struct io_uring_params __user *params) {struct io_ring_ctx *ctx;
@@ -3559,7 +3561,7 @@ static __cold int io_uring_create(unsigned entries, struct io_uring_params *p,if (ret)return ret; - ret =io_uring_fill_params (entries, p); + ret =io_uring_fill_params (p);if (unlikely (ret))return ret;
@@ -3698,7 +3700,8 @@ static long io_uring_setup(u32 entries, struct io_uring_params __user *params)if (p.flags & ~IORING_SETUP_FLAGS )return -EINVAL ; -return io_uring_create (entries, &p, params); + p.sq_entries = entries; +return io_uring_create (&p, params); }static inline int io_uring_allowed (void )
io_uring/io_uring.h +1 -1
@@ -136,7 +136,7 @@ static inline bool io_should_wake(struct io_wait_queue *iowq)unsigned long rings_size (unsigned int flags,unsigned int sq_entries,unsigned int cq_entries,size_t *sq_offset); -int io_uring_fill_params (unsigned entries,struct io_uring_params *p); +int io_uring_fill_params (struct io_uring_params *p);bool io_cqe_cache_refill (struct io_ring_ctx *ctx,bool overflow,bool cqe32);int io_run_task_work_sig (struct io_ring_ctx *ctx);int io_run_local_work (struct io_ring_ctx *ctx,int min_events,int max_events);
io_uring/register.c +1 -1
@@ -416,7 +416,7 @@ static int io_register_resize_rings(struct io_ring_ctx *ctx, void __user *arg)/* properties that are always inherited */ p.flags |= (ctx->flags ©_FLAGS ); - ret =io_uring_fill_params (p.sq_entries, &p); + ret =io_uring_fill_params (&p);if (unlikely (ret))return ret;
Signature · 1 line
2.49.0
RFC 3/16 io_uring: use mem_is_zero to check ring params
In reply to Pavel Begunkovmem_is_zero() does the job without hand rolled loops, use that to verify reserved fields of ring params.
--- io_uring/io_uring.c | 8 +++----- 1 file changed, 3 insertions(+), 5 deletions(-)
io_uring/io_uring.c +3 -5
@@ -3689,14 +3689,12 @@ static __cold int io_uring_create(struct io_uring_params *p,static long io_uring_setup (u32 entries,struct io_uring_params __user *params) {struct io_uring_params p; -int i;if (copy_from_user (&p, params,sizeof (p)))return -EFAULT ; -for (i =0 ; i <ARRAY_SIZE (p.resv); i++) { -if (p.resv[i]) -return -EINVAL ; - } + +if (!mem_is_zero (&p.resv,sizeof (p.resv))) +return -EINVAL ;if (p.flags & ~IORING_SETUP_FLAGS )return -EINVAL ;
Signature · 1 line
2.49.0
RFC 4/16 io_uring: move flags check to io_uring_sanitise_params
In reply to Pavel Begunkovio_uring_sanitise_params() sanitises most of the setup flags invariants, move the IORING_SETUP_FLAGS check from io_uring_setup() into it.
--- io_uring/io_uring.c | 5 +++-- 1 file changed, 3 insertions(+), 2 deletions(-)
io_uring/io_uring.c +3 -2
@@ -3435,6 +3435,9 @@ static int io_uring_sanitise_params(struct io_uring_params *p) {unsigned flags = p->flags; +if (flags & ~IORING_SETUP_FLAGS ) +return -EINVAL ; +/* There is no way to mmap rings without a real fd */ if ((flags &IORING_SETUP_REGISTERED_FD_ONLY ) && !(flags &IORING_SETUP_NO_MMAP ))
@@ -3696,8 +3699,6 @@ static long io_uring_setup(u32 entries, struct io_uring_params __user *params)if (!mem_is_zero (&p.resv,sizeof (p.resv)))return -EINVAL ; -if (p.flags & ~IORING_SETUP_FLAGS ) -return -EINVAL ; p.sq_entries = entries;return io_uring_create (&p, params); }
Signature · 1 line
2.49.0
RFC 5/16 io_uring: introduce struct io_ctx_config
In reply to Pavel BegunkovThere will be more information needed during ctx setup, so instead of passing a handful of pointers around, wrap them all into a new structure.
--- io_uring/io_uring.c | 19 +++++++++++-------- io_uring/io_uring.h | 5 +++++ 2 files changed, 16 insertions(+), 8 deletions(-)
io_uring/io_uring.c +11 -8
@@ -3552,9 +3552,9 @@ int io_uring_fill_params(struct io_uring_params *p)return 0 ; } -static __cold int io_uring_create (struct io_uring_params *p, -struct io_uring_params __user *params) +static __cold int io_uring_create (struct io_ctx_config *config) { +struct io_uring_params *p = &config->p;struct io_ring_ctx *ctx;struct io_uring_task *tctx;struct file *file;
@@ -3638,7 +3638,7 @@ static __cold int io_uring_create(struct io_uring_params *p, p->features =IORING_FEAT_FLAGS ; -if (copy_to_user (params, p,sizeof (*p))) { +if (copy_to_user (config->uptr, p,sizeof (*p))) { ret = -EFAULT ; goto err; }
@@ -3691,16 +3691,19 @@ static __cold int io_uring_create(struct io_uring_params *p,*/ static long io_uring_setup (u32 entries,struct io_uring_params __user *params) { -struct io_uring_params p; +struct io_ctx_config config; -if (copy_from_user (&p, params,sizeof (p))) +memset (&config,0 ,sizeof (config)); + +if (copy_from_user (&config.p, params,sizeof (config.p)))return -EFAULT ; -if (!mem_is_zero (&p.resv,sizeof (p.resv))) +if (!mem_is_zero (&config.p.resv,sizeof (config.p.resv)))return -EINVAL ; - p.sq_entries = entries; -return io_uring_create (&p, params); + config.p.sq_entries = entries; + config.uptr = params; +return io_uring_create (&config); }static inline int io_uring_allowed (void )
io_uring/io_uring.h +5 -0
@@ -17,6 +17,11 @@#include <trace/events/io_uring.h> #endif +struct io_ctx_config { +struct io_uring_params p; +struct io_uring_params __user *uptr; +}; +#define IORING_FEAT_FLAGS (IORING_FEAT_SINGLE_MMAP |\ IORING_FEAT_NODROP |\ IORING_FEAT_SUBMIT_STABLE |\
Signature · 1 line
2.49.0
RFC 6/16 io_uring: split out config init helper
In reply to Pavel BegunkovSeparate most of configuration verification / calculation into a function, that will be the first step before doing any actual allocation.
--- io_uring/io_uring.c | 22 +++++++++++++++++----- 1 file changed, 17 insertions(+), 5 deletions(-)
io_uring/io_uring.c +17 -5
@@ -3552,12 +3552,9 @@ int io_uring_fill_params(struct io_uring_params *p)return 0 ; } -static __cold int io_uring_create (struct io_ctx_config *config) +static int io_prepare_config (struct io_ctx_config *config) {struct io_uring_params *p = &config->p; -struct io_ring_ctx *ctx; -struct io_uring_task *tctx; -struct file *file;int ret; ret =io_uring_sanitise_params (p);
@@ -3565,7 +3562,22 @@ static __cold int io_uring_create(struct io_ctx_config *config)return ret; ret =io_uring_fill_params (p); -if (unlikely (ret)) +if (ret) +return ret; + +return 0 ; +} + +static __cold int io_uring_create (struct io_ctx_config *config) +{ +struct io_uring_params *p = &config->p; +struct io_ring_ctx *ctx; +struct io_uring_task *tctx; +struct file *file; +int ret; + + ret =io_prepare_config (config); +if (ret)return ret; ctx =io_ring_ctx_alloc (p);
Signature · 1 line
2.49.0
RFC 7/16 io_uring: add structure keeping ring offsets
In reply to Pavel BegunkovAdd struct io_scq_dim that keeps all offset / size / dimension information about the rings, and let rings_size() initialise it. It improves calculation locality and allows to dedup some code.
--- io_uring/io_uring.c | 60 ++++++++++++++++++++++++--------------------- io_uring/io_uring.h | 12 +++++++-- io_uring/register.c | 19 ++++++-------- 3 files changed, 49 insertions(+), 42 deletions(-)
io_uring/io_uring.c +32 -28
@@ -2757,49 +2757,61 @@ static void io_rings_free(struct io_ring_ctx *ctx) ctx->sq_sqes =NULL ; } -unsigned long rings_size (unsigned int flags,unsigned int sq_entries, -unsigned int cq_entries,size_t *sq_offset) +int rings_size (unsigned int flags,unsigned int sq_entries, +unsigned int cq_entries,struct io_scq_dim *dims) {struct io_rings *rings;size_t off, sq_array_size; +size_t sqe_size; + + dims->sq_array_offset =SIZE_MAX ; + + sqe_size =sizeof (struct io_uring_sqe ); +if (flags &IORING_SETUP_SQE128 ) + sqe_size *=2 ; + + dims->sq_size =array_size (sqe_size, sq_entries); +if (dims->sq_size ==SIZE_MAX ) +return -EOVERFLOW ; off =struct_size (rings, cqes, cq_entries);if (off ==SIZE_MAX ) -return SIZE_MAX ; +return -EOVERFLOW ;if (flags &IORING_SETUP_CQE32 ) {if (check_shl_overflow (off,1 , &off)) -return SIZE_MAX ; +return -EOVERFLOW ; }if (flags &IORING_SETUP_CQE_MIXED ) {if (cq_entries <2 ) -return SIZE_MAX ; +return -EOVERFLOW ; }if (flags &IORING_SETUP_SQE_MIXED ) {if (sq_entries <2 ) -return SIZE_MAX ; +return -EOVERFLOW ; }#ifdef CONFIG_SMP off =ALIGN (off,SMP_CACHE_BYTES );if (off ==0 ) -return SIZE_MAX ; +return -EOVERFLOW ;#endif if (flags &IORING_SETUP_NO_SQARRAY ) { - *sq_offset =SIZE_MAX ; -return off; + dims->cq_comp_size = off; +return 0 ; } - *sq_offset = off; + dims->sq_array_offset = off; sq_array_size =array_size (sizeof (u32), sq_entries);if (sq_array_size ==SIZE_MAX ) -return SIZE_MAX ; +return -EOVERFLOW ;if (check_add_overflow (off, sq_array_size, &off)) -return SIZE_MAX ; +return -EOVERFLOW ; -return off; + dims->cq_comp_size = off; +return 0 ; }static __cold void __io_req_caches_free (struct io_ring_ctx *ctx)
@@ -3354,27 +3366,19 @@ static __cold int io_allocate_scq_urings(struct io_ring_ctx *ctx, {struct io_uring_region_desc rd;struct io_rings *rings; -size_t sq_array_offset; -size_t sq_size, cq_size, sqe_size; +struct io_scq_dim dims;int ret;/* make sure these are sane, as we already accounted them */ ctx->sq_entries = p->sq_entries; ctx->cq_entries = p->cq_entries; - sqe_size =sizeof (struct io_uring_sqe ); -if (p->flags &IORING_SETUP_SQE128 ) - sqe_size *=2 ; - sq_size =array_size (sqe_size, p->sq_entries); -if (sq_size ==SIZE_MAX ) -return -EOVERFLOW ; - cq_size =rings_size (ctx->flags, p->sq_entries, p->cq_entries, - &sq_array_offset); -if (cq_size ==SIZE_MAX ) -return -EOVERFLOW ; + ret =rings_size (ctx->flags, p->sq_entries, p->cq_entries, &dims); +if (ret) +return ret;memset (&rd,0 ,sizeof (rd)); - rd.size =PAGE_ALIGN (cq_size); + rd.size =PAGE_ALIGN (dims.cq_comp_size);if (ctx->flags &IORING_SETUP_NO_MMAP ) { rd.user_addr = p->cq_off.user_addr; rd.flags |=IORING_MEM_REGION_TYPE_USER ;
@@ -3385,10 +3389,10 @@ static __cold int io_allocate_scq_urings(struct io_ring_ctx *ctx, ctx->rings = rings =io_region_get_ptr (&ctx->ring_region);if (!(ctx->flags &IORING_SETUP_NO_SQARRAY )) - ctx->sq_array = (u32 *)((char *)rings + sq_array_offset); + ctx->sq_array = (u32 *)((char *)rings + dims.sq_array_offset);memset (&rd,0 ,sizeof (rd)); - rd.size =PAGE_ALIGN (sq_size); + rd.size =PAGE_ALIGN (dims.sq_size);if (ctx->flags &IORING_SETUP_NO_MMAP ) { rd.user_addr = p->sq_off.user_addr; rd.flags |=IORING_MEM_REGION_TYPE_USER ;
io_uring/io_uring.h +10 -2
@@ -17,6 +17,14 @@#include <trace/events/io_uring.h> #endif +struct io_scq_dim { +size_t sq_array_offset; +size_t sq_size; + +/* Compound array mmap'ed together with CQ. */ +size_t cq_comp_size; +}; +struct io_ctx_config {struct io_uring_params p;struct io_uring_params __user *uptr;
@@ -139,8 +147,8 @@ static inline bool io_should_wake(struct io_wait_queue *iowq)#define IORING_MAX_ENTRIES 32768#define IORING_MAX_CQ_ENTRIES (2 * IORING_MAX_ENTRIES) -unsigned long rings_size (unsigned int flags,unsigned int sq_entries, -unsigned int cq_entries,size_t *sq_offset); +int rings_size (unsigned int flags,unsigned int sq_entries, +unsigned int cq_entries,struct io_scq_dim *dims);int io_uring_fill_params (struct io_uring_params *p);bool io_cqe_cache_refill (struct io_ring_ctx *ctx,bool overflow,bool cqe32);int io_run_task_work_sig (struct io_ring_ctx *ctx);
io_uring/register.c +7 -12
@@ -402,6 +402,7 @@ static int io_register_resize_rings(struct io_ring_ctx *ctx, void __user *arg)struct io_ring_ctx_rings o = { }, n = { }, *to_free =NULL ;size_t size, sq_array_offset;unsigned i, tail, old_head; +struct io_scq_dim dims;struct io_uring_params p;int ret;
@@ -419,11 +420,12 @@ static int io_register_resize_rings(struct io_ring_ctx *ctx, void __user *arg) ret =io_uring_fill_params (&p);if (unlikely (ret))return ret; + ret =rings_size (p.flags, p.sq_entries, p.cq_entries, &dims); +if (ret) +return ret; - size =rings_size (p.flags, p.sq_entries, p.cq_entries, - &sq_array_offset); -if (size ==SIZE_MAX ) -return -EOVERFLOW ; + size = dims.cq_comp_size; + sq_array_offset = dims.sq_array_offset;memset (&rd,0 ,sizeof (rd)); rd.size =PAGE_ALIGN (size);
@@ -455,14 +457,7 @@ static int io_register_resize_rings(struct io_ring_ctx *ctx, void __user *arg)return -EFAULT ; } -if (p.flags &IORING_SETUP_SQE128 ) - size =array_size (2 *sizeof (struct io_uring_sqe ), p.sq_entries); -else - size =array_size (sizeof (struct io_uring_sqe ), p.sq_entries); -if (size ==SIZE_MAX ) { -io_register_free_rings (ctx, &n); -return -EOVERFLOW ; - } + size = dims.sq_size;memset (&rd,0 ,sizeof (rd)); rd.size =PAGE_ALIGN (size);
Signature · 1 line
2.49.0
RFC 8/16 io_uring: pre-calculate scq offsets
In reply to Pavel BegunkovMove ring size / offset calculations into io_prepare_config(). It simplifies misconfiguration handling, keeps it local, allows to move p->sq_off.array calculation closer to the rest of p->sq_off init.
--- io_uring/io_uring.c | 28 ++++++++++++++-------------- io_uring/io_uring.h | 1 + 2 files changed, 15 insertions(+), 14 deletions(-)
io_uring/io_uring.c +14 -14
@@ -3362,23 +3362,20 @@ bool io_is_uring_fops(struct file *file) }static __cold int io_allocate_scq_urings (struct io_ring_ctx *ctx, -struct io_uring_params *p) +struct io_ctx_config *config) { +struct io_uring_params *p = &config->p; +struct io_scq_dim *dims = &config->dims;struct io_uring_region_desc rd;struct io_rings *rings; -struct io_scq_dim dims;int ret;/* make sure these are sane, as we already accounted them */ ctx->sq_entries = p->sq_entries; ctx->cq_entries = p->cq_entries; - ret =rings_size (ctx->flags, p->sq_entries, p->cq_entries, &dims); -if (ret) -return ret; -memset (&rd,0 ,sizeof (rd)); - rd.size =PAGE_ALIGN (dims.cq_comp_size); + rd.size =PAGE_ALIGN (dims->cq_comp_size);if (ctx->flags &IORING_SETUP_NO_MMAP ) { rd.user_addr = p->cq_off.user_addr; rd.flags |=IORING_MEM_REGION_TYPE_USER ;
@@ -3387,12 +3384,11 @@ static __cold int io_allocate_scq_urings(struct io_ring_ctx *ctx,if (ret)return ret; ctx->rings = rings =io_region_get_ptr (&ctx->ring_region); -if (!(ctx->flags &IORING_SETUP_NO_SQARRAY )) - ctx->sq_array = (u32 *)((char *)rings + dims.sq_array_offset); + ctx->sq_array = (u32 *)((char *)rings + dims->sq_array_offset);memset (&rd,0 ,sizeof (rd)); - rd.size =PAGE_ALIGN (dims.sq_size); + rd.size =PAGE_ALIGN (dims->sq_size);if (ctx->flags &IORING_SETUP_NO_MMAP ) { rd.user_addr = p->sq_off.user_addr; rd.flags |=IORING_MEM_REGION_TYPE_USER ;
@@ -3569,6 +3565,13 @@ static int io_prepare_config(struct io_ctx_config *config)if (ret)return ret; + ret =rings_size (p->flags, p->sq_entries, p->cq_entries, &config->dims); +if (ret) +return ret; + +if (!(p->flags &IORING_SETUP_NO_SQARRAY )) + p->sq_off.array = config->dims.sq_array_offset; +return 0 ; }
@@ -3641,13 +3644,10 @@ static __cold int io_uring_create(struct io_ctx_config *config)mmgrab (current->mm); ctx->mm_account = current->mm; - ret =io_allocate_scq_urings (ctx, p); + ret =io_allocate_scq_urings (ctx, config);if (ret) goto err; -if (!(p->flags &IORING_SETUP_NO_SQARRAY )) - p->sq_off.array = (char *)ctx->sq_array - (char *)ctx->rings; - ret =io_sq_offload_create (ctx, p);if (ret) goto err;
io_uring/io_uring.h +1 -0
@@ -27,6 +27,7 @@ struct io_scq_dim {struct io_ctx_config {struct io_uring_params p; +struct io_scq_dim dims;struct io_uring_params __user *uptr; };
Signature · 1 line
2.49.0
RFC 9/16 io_uring: inroduce helper for setting user offset
In reply to Pavel Begunkovstruct io_[c,s]qring_offsets may require computation from other steps like in case of sq_off.array. Move the initialisation out of io_uring_fill_params() into a separate function that can be called later.
--- io_uring/io_uring.c | 52 ++++++++++++++++++++++++--------------------- io_uring/io_uring.h | 1 + io_uring/register.c | 1 + 3 files changed, 30 insertions(+), 24 deletions(-)
io_uring/io_uring.c +28 -24
@@ -3487,6 +3487,33 @@ static int io_uring_sanitise_params(struct io_uring_params *p)return 0 ; } +void io_fill_scq_offsets (struct io_uring_params *p,struct io_scq_dim *dims) +{ + p->sq_off.head = offsetof(struct io_rings, sq.head); + p->sq_off.tail = offsetof(struct io_rings, sq.tail); + p->sq_off.ring_mask = offsetof(struct io_rings , sq_ring_mask); + p->sq_off.ring_entries = offsetof(struct io_rings , sq_ring_entries); + p->sq_off.flags = offsetof(struct io_rings , sq_flags); + p->sq_off.dropped = offsetof(struct io_rings , sq_dropped); + p->sq_off.resv1 =0 ; +if (!(p->flags &IORING_SETUP_NO_MMAP )) + p->sq_off.user_addr =0 ; + + p->cq_off.head = offsetof(struct io_rings, cq.head); + p->cq_off.tail = offsetof(struct io_rings, cq.tail); + p->cq_off.ring_mask = offsetof(struct io_rings , cq_ring_mask); + p->cq_off.ring_entries = offsetof(struct io_rings , cq_ring_entries); + p->cq_off.overflow = offsetof(struct io_rings , cq_overflow); + p->cq_off.cqes = offsetof(struct io_rings , cqes); + p->cq_off.flags = offsetof(struct io_rings , cq_flags); + p->cq_off.resv1 =0 ; +if (!(p->flags &IORING_SETUP_NO_MMAP )) + p->cq_off.user_addr =0 ; + +if (!(p->flags &IORING_SETUP_NO_SQARRAY )) + p->sq_off.array = dims->sq_array_offset; +} +int io_uring_fill_params (struct io_uring_params *p) {unsigned entries = p->sq_entries;
@@ -3528,27 +3555,6 @@ int io_uring_fill_params(struct io_uring_params *p) p->cq_entries =2 * p->sq_entries; } - p->sq_off.head = offsetof(struct io_rings, sq.head); - p->sq_off.tail = offsetof(struct io_rings, sq.tail); - p->sq_off.ring_mask = offsetof(struct io_rings , sq_ring_mask); - p->sq_off.ring_entries = offsetof(struct io_rings , sq_ring_entries); - p->sq_off.flags = offsetof(struct io_rings , sq_flags); - p->sq_off.dropped = offsetof(struct io_rings , sq_dropped); - p->sq_off.resv1 =0 ; -if (!(p->flags &IORING_SETUP_NO_MMAP )) - p->sq_off.user_addr =0 ; - - p->cq_off.head = offsetof(struct io_rings, cq.head); - p->cq_off.tail = offsetof(struct io_rings, cq.tail); - p->cq_off.ring_mask = offsetof(struct io_rings , cq_ring_mask); - p->cq_off.ring_entries = offsetof(struct io_rings , cq_ring_entries); - p->cq_off.overflow = offsetof(struct io_rings , cq_overflow); - p->cq_off.cqes = offsetof(struct io_rings , cqes); - p->cq_off.flags = offsetof(struct io_rings , cq_flags); - p->cq_off.resv1 =0 ; -if (!(p->flags &IORING_SETUP_NO_MMAP )) - p->cq_off.user_addr =0 ; -return 0 ; }
@@ -3569,9 +3575,7 @@ static int io_prepare_config(struct io_ctx_config *config)if (ret)return ret; -if (!(p->flags &IORING_SETUP_NO_SQARRAY )) - p->sq_off.array = config->dims.sq_array_offset; - +io_fill_scq_offsets (p, &config->dims);return 0 ; }
io_uring/io_uring.h +1 -0
@@ -151,6 +151,7 @@ static inline bool io_should_wake(struct io_wait_queue *iowq)int rings_size (unsigned int flags,unsigned int sq_entries,unsigned int cq_entries,struct io_scq_dim *dims);int io_uring_fill_params (struct io_uring_params *p); +void io_fill_scq_offsets (struct io_uring_params *p,struct io_scq_dim *dims);bool io_cqe_cache_refill (struct io_ring_ctx *ctx,bool overflow,bool cqe32);int io_run_task_work_sig (struct io_ring_ctx *ctx);int io_run_local_work (struct io_ring_ctx *ctx,int min_events,int max_events);
io_uring/register.c +1 -0
@@ -423,6 +423,7 @@ static int io_register_resize_rings(struct io_ring_ctx *ctx, void __user *arg) ret =rings_size (p.flags, p.sq_entries, p.cq_entries, &dims);if (ret)return ret; +io_fill_scq_offsets (&p, &dims); size = dims.cq_comp_size; sq_array_offset = dims.sq_array_offset;
Signature · 1 line
2.49.0
RFC 10/16 io_uring: separate cqe array from headers
In reply to Pavel BegunkovKeep a pointer to the CQ separate from SCQ headers, it'll be used shortly in next patches. Also, don't overestimate the CQ size for SETUP_CQE32, which not only doubles memory for CQ entries but also the headers as well.
--- include/linux/io_uring_types.h | 17 +++++++++-------- io_uring/fdinfo.c | 2 +- io_uring/io_uring.c | 35 ++++++++++++++++++++++------------ io_uring/io_uring.h | 1 + io_uring/register.c | 8 +++++++- 5 files changed, 41 insertions(+), 22 deletions(-)
include/linux/io_uring_types.h +9 -8
@@ -209,14 +209,6 @@ struct io_rings {* ordered with any other data. */ u32 cq_overflow; -/* -* Ring buffer of completion events. -* -* The kernel writes completion events fresh every time they are -* produced, so the application is allowed to modify pending -* entries. -*/ -struct io_uring_cqe cqes[]____cacheline_aligned_in_smp ; };struct io_restriction {
@@ -274,6 +266,15 @@ struct io_ring_ctx {struct task_struct *submitter_task;struct io_rings *rings; +/* +* Ring buffer of completion events. +* +* The kernel writes completion events fresh every time they are +* produced, so the application is allowed to modify pending +* entries. +*/ +struct io_uring_cqe *cqes; +struct percpu_ref refs;clockid_t clockid;
io_uring/fdinfo.c +1 -1
@@ -153,7 +153,7 @@ static void __io_uring_show_fdinfo(struct io_ring_ctx *ctx, struct seq_file *m)struct io_uring_cqe *cqe;bool cqe32 = false; - cqe = &r->cqes[(cq_head & cq_mask)]; + cqe = &ctx->cqes[(cq_head & cq_mask)];if (cqe->flags &IORING_CQE_F_32 || ctx->flags &IORING_SETUP_CQE32 ) cqe32 = true; seq_printf(m, "%5u: user_data:%llu, res:%d, flag:%x",
io_uring/io_uring.c +23 -12
@@ -745,7 +745,7 @@ static struct io_overflow_cqe *io_alloc_ocqe(struct io_ring_ctx *ctx,static bool io_fill_nop_cqe (struct io_ring_ctx *ctx,unsigned int off) {if (__io_cqring_events (ctx) < ctx->cq_entries) { -struct io_uring_cqe *cqe = &ctx->rings->cqes[off]; +struct io_uring_cqe *cqe = &ctx->cqes[off]; cqe->user_data =0 ; cqe->res =0 ;
@@ -763,7 +763,6 @@ static bool io_fill_nop_cqe(struct io_ring_ctx *ctx, unsigned int off)*/ bool io_cqe_cache_refill (struct io_ring_ctx *ctx,bool overflow,bool cqe32) { -struct io_rings *rings = ctx->rings;unsigned int off = ctx->cached_cq_tail & (ctx->cq_entries -1 );unsigned int free, queued, len;
@@ -798,7 +797,7 @@ bool io_cqe_cache_refill(struct io_ring_ctx *ctx, bool overflow, bool cqe32) len <<=1 ; } - ctx->cqe_cached = &rings->cqes[off]; + ctx->cqe_cached = &ctx->cqes[off]; ctx->cqe_sentinel = ctx->cqe_cached + len;return true; }
@@ -2760,8 +2759,8 @@ static void io_rings_free(struct io_ring_ctx *ctx)int rings_size (unsigned int flags,unsigned int sq_entries,unsigned int cq_entries,struct io_scq_dim *dims) { -struct io_rings *rings;size_t off, sq_array_size; +size_t cq_size, cqe_size;size_t sqe_size; dims->sq_array_offset =SIZE_MAX ;
@@ -2769,18 +2768,26 @@ int rings_size(unsigned int flags, unsigned int sq_entries, sqe_size =sizeof (struct io_uring_sqe );if (flags &IORING_SETUP_SQE128 ) sqe_size *=2 ; + cqe_size =sizeof (struct io_uring_cqe ); +if (flags &IORING_SETUP_CQE32 ) + cqe_size *=2 ; dims->sq_size =array_size (sqe_size, sq_entries);if (dims->sq_size ==SIZE_MAX )return -EOVERFLOW ; - off =struct_size (rings, cqes, cq_entries); + off =sizeof (struct io_rings ); + off =L1_CACHE_ALIGN (off); + dims->cq_offset = off; + + cq_size =array_size (cqe_size, cq_entries); +if (cq_size ==SIZE_MAX ) +return -EOVERFLOW ; + + off =size_add (off, cq_size);if (off ==SIZE_MAX )return -EOVERFLOW ; -if (flags &IORING_SETUP_CQE32 ) { -if (check_shl_overflow (off,1 , &off)) -return -EOVERFLOW ; - } +if (flags &IORING_SETUP_CQE_MIXED ) {if (cq_entries <2 )return -EOVERFLOW ;
@@ -3368,6 +3375,7 @@ static __cold int io_allocate_scq_urings(struct io_ring_ctx *ctx,struct io_scq_dim *dims = &config->dims;struct io_uring_region_desc rd;struct io_rings *rings; +void *ptr;int ret;/* make sure these are sane, as we already accounted them */
@@ -3383,9 +3391,12 @@ static __cold int io_allocate_scq_urings(struct io_ring_ctx *ctx, ret =io_create_region (ctx, &ctx->ring_region, &rd,IORING_OFF_CQ_RING );if (ret)return ret; - ctx->rings = rings =io_region_get_ptr (&ctx->ring_region); + ptr =io_region_get_ptr (&ctx->ring_region); + ctx->rings = rings = ptr; + ctx->cqes = ptr + config->dims.cq_offset; +if (!(ctx->flags &IORING_SETUP_NO_SQARRAY )) - ctx->sq_array = (u32 *)((char *)rings + dims->sq_array_offset); + ctx->sq_array = ptr + dims->sq_array_offset;memset (&rd,0 ,sizeof (rd)); rd.size =PAGE_ALIGN (dims->sq_size);
@@ -3504,7 +3515,7 @@ void io_fill_scq_offsets(struct io_uring_params *p, struct io_scq_dim *dims) p->cq_off.ring_mask = offsetof(struct io_rings , cq_ring_mask); p->cq_off.ring_entries = offsetof(struct io_rings , cq_ring_entries); p->cq_off.overflow = offsetof(struct io_rings , cq_overflow); - p->cq_off.cqes = offsetof(struct io_rings , cqes); + p->cq_off.cqes = dims->cq_offset; p->cq_off.flags = offsetof(struct io_rings , cq_flags); p->cq_off.resv1 =0 ;if (!(p->flags &IORING_SETUP_NO_MMAP ))
io_uring/io_uring.h +1 -0
@@ -20,6 +20,7 @@struct io_scq_dim {size_t sq_array_offset;size_t sq_size; +size_t cq_offset;/* Compound array mmap'ed together with CQ. */ size_t cq_comp_size;
io_uring/register.c +7 -1
@@ -373,6 +373,7 @@ static int io_register_clock(struct io_ring_ctx *ctx,struct io_ring_ctx_rings {struct io_rings *rings;struct io_uring_sqe *sq_sqes; +struct io_uring_cqe *cqes;struct io_mapped_region sq_region;struct io_mapped_region ring_region;
@@ -439,6 +440,7 @@ static int io_register_resize_rings(struct io_ring_ctx *ctx, void __user *arg)return ret; n.rings =io_region_get_ptr (&n.ring_region); + n.cqes =io_region_get_ptr (&n.ring_region) + dims.cq_offset;/* * At this point n.rings is shared with userspace, just like o.rings
@@ -497,6 +499,8 @@ static int io_register_resize_rings(struct io_ring_ctx *ctx, void __user *arg) ctx->rings =NULL ; o.sq_sqes = ctx->sq_sqes; ctx->sq_sqes =NULL ; + o.cqes = ctx->cqes; + ctx->cqes =NULL ;/* * Now copy SQ and CQ entries, if any. If either of the destination
@@ -522,6 +526,7 @@ static int io_register_resize_rings(struct io_ring_ctx *ctx, void __user *arg)/* restore old rings, and return -EOVERFLOW via cleanup path */ ctx->rings = o.rings; ctx->sq_sqes = o.sq_sqes; + ctx->cqes = o.cqes; to_free = &n; ret = -EOVERFLOW ; goto out;
@@ -530,7 +535,7 @@ static int io_register_resize_rings(struct io_ring_ctx *ctx, void __user *arg)unsigned src_head = i & (ctx->cq_entries -1 );unsigned dst_head = i & (p.cq_entries -1 ); - n.rings->cqes[dst_head] = o.rings->cqes[src_head]; + n.cqes[dst_head] = o.cqes[src_head]; }WRITE_ONCE (n.rings->cq.head, old_head);WRITE_ONCE (n.rings->cq.tail, tail);
@@ -551,6 +556,7 @@ static int io_register_resize_rings(struct io_ring_ctx *ctx, void __user *arg) ctx->rings = n.rings; ctx->sq_sqes = n.sq_sqes; + ctx->cqes = n.cqes;swap_old (ctx, o, n, ring_region);swap_old (ctx, o, n, sq_region); to_free = &o;
Signature · 1 line
2.49.0
RFC 11/16 io_uring/region: introduce io_region_slice
In reply to Pavel BegunkovAdd a new helper function that returns a sub-slice of a region memory from an {offset,size} pair and performs a bunch of extra sanitisation. It'll be used later for (slow path) ring setup.
--- io_uring/memmap.c | 12 ++++++++++++ io_uring/memmap.h | 2 ++ 2 files changed, 14 insertions(+)
io_uring/memmap.c +12 -0
@@ -232,6 +232,18 @@ int io_create_region(struct io_ring_ctx *ctx, struct io_mapped_region *mr,return ret; } +void *io_region_slice (struct io_mapped_region *mr,size_t off,size_t size) +{ +if (WARN_ON_ONCE (!size) || !io_region_is_set (mr)) +return NULL ; + + size =size_add (off, size); +if (size ==SIZE_MAX || size >io_region_size (mr)) +return NULL ; + +return io_region_get_ptr (mr) + off; +} +static struct io_mapped_region *io_mmap_get_region (struct io_ring_ctx *ctx,loff_t pgoff) {
io_uring/memmap.h +2 -0
@@ -21,6 +21,8 @@ int io_create_region(struct io_ring_ctx *ctx, struct io_mapped_region *mr,struct io_uring_region_desc *reg, unsigned long mmap_offset); +void *io_region_slice (struct io_mapped_region *mr,size_t off,size_t size); +static inline void *io_region_get_ptr (struct io_mapped_region *mr) {return mr->ptr;
Signature · 1 line
2.49.0
RFC 12/16 io_uring: convert pointer init to io_region_slice
In reply to Pavel BegunkovUse io_region_slice() to initialise ctx ring pointers. The helper performs bound checks and other sanitisation, which is safer and will be especially helpful when ring placement gets more complicated in coming patches. It also extends struct io_scq_dim with all intermediate offsets and sizes to fully describe rings.
--- io_uring/io_uring.c | 47 ++++++++++++++++++++++++--------------------- io_uring/io_uring.h | 2 ++ 2 files changed, 27 insertions(+), 22 deletions(-)
io_uring/io_uring.c +25 -22
@@ -2759,9 +2759,7 @@ static void io_rings_free(struct io_ring_ctx *ctx)int rings_size (unsigned int flags,unsigned int sq_entries,unsigned int cq_entries,struct io_scq_dim *dims) { -size_t off, sq_array_size; -size_t cq_size, cqe_size; -size_t sqe_size; +size_t cqe_size, off, sqe_size; dims->sq_array_offset =SIZE_MAX ;
@@ -2773,18 +2771,18 @@ int rings_size(unsigned int flags, unsigned int sq_entries, cqe_size *=2 ; dims->sq_size =array_size (sqe_size, sq_entries); -if (dims->sq_size ==SIZE_MAX ) + dims->sq_array_size =array_size (sizeof (u32), sq_entries); + dims->cq_size =array_size (cqe_size, cq_entries); + +if (dims->sq_size ==SIZE_MAX || dims->cq_size ==SIZE_MAX || + dims->sq_array_size ==SIZE_MAX )return -EOVERFLOW ; off =sizeof (struct io_rings ); off =L1_CACHE_ALIGN (off); dims->cq_offset = off; - cq_size =array_size (cqe_size, cq_entries); -if (cq_size ==SIZE_MAX ) -return -EOVERFLOW ; - - off =size_add (off, cq_size); + off =size_add (off, dims->cq_size);if (off ==SIZE_MAX )return -EOVERFLOW ;
@@ -2809,12 +2807,7 @@ int rings_size(unsigned int flags, unsigned int sq_entries, } dims->sq_array_offset = off; - - sq_array_size =array_size (sizeof (u32), sq_entries); -if (sq_array_size ==SIZE_MAX ) -return -EOVERFLOW ; - -if (check_add_overflow (off, sq_array_size, &off)) +if (check_add_overflow (off, dims->sq_array_size, &off))return -EOVERFLOW ; dims->cq_comp_size = off;
@@ -3375,7 +3368,6 @@ static __cold int io_allocate_scq_urings(struct io_ring_ctx *ctx,struct io_scq_dim *dims = &config->dims;struct io_uring_region_desc rd;struct io_rings *rings; -void *ptr;int ret;/* make sure these are sane, as we already accounted them */
@@ -3391,12 +3383,19 @@ static __cold int io_allocate_scq_urings(struct io_ring_ctx *ctx, ret =io_create_region (ctx, &ctx->ring_region, &rd,IORING_OFF_CQ_RING );if (ret)return ret; - ptr =io_region_get_ptr (&ctx->ring_region); - ctx->rings = rings = ptr; - ctx->cqes = ptr + config->dims.cq_offset; -if (!(ctx->flags &IORING_SETUP_NO_SQARRAY )) - ctx->sq_array = ptr + dims->sq_array_offset; + ctx->rings =io_region_slice (&ctx->ring_region,0 ,sizeof (struct io_rings )); + ctx->cqes =io_region_slice (&ctx->ring_region, dims->cq_offset, dims->cq_size); +if (!ctx->rings || !ctx->cqes) +return -EFAULT ; + +if (!(ctx->flags &IORING_SETUP_NO_SQARRAY )) { + ctx->sq_array =io_region_slice (&ctx->ring_region, + dims->sq_array_offset, + dims->sq_array_size); +if (!ctx->sq_array) +return -EFAULT ; + }memset (&rd,0 ,sizeof (rd)); rd.size =PAGE_ALIGN (dims->sq_size);
@@ -3409,8 +3408,12 @@ static __cold int io_allocate_scq_urings(struct io_ring_ctx *ctx,io_rings_free (ctx);return ret; } - ctx->sq_sqes =io_region_get_ptr (&ctx->sq_region); + ctx->sq_sqes =io_region_slice (&ctx->sq_region,0 , dims->sq_size); +if (!ctx->sq_sqes) +return -EFAULT ; + + rings = ctx->rings;memset (rings,0 ,sizeof (*rings));WRITE_ONCE (rings->sq_ring_mask, ctx->sq_entries -1 );WRITE_ONCE (rings->cq_ring_mask, ctx->cq_entries -1 );
io_uring/io_uring.h +2 -0
@@ -19,8 +19,10 @@struct io_scq_dim {size_t sq_array_offset; +size_t sq_array_size;size_t sq_size;size_t cq_offset; +size_t cq_size;/* Compound array mmap'ed together with CQ. */ size_t cq_comp_size;
Signature · 1 line
2.49.0
RFC 13/16 io_uring: refactor rings_size()
In reply to Pavel BegunkovRefactor rings_size() to facilitate further changes. Move entries checks to the beginning, which localises offset calculation. And instead of special casing IORING_SETUP_NO_SQARRAY, reverse the check and just continue offset calculation. This way it assign cq_comp_size only once at the end.
--- io_uring/io_uring.c | 31 +++++++++++++++---------------- 1 file changed, 15 insertions(+), 16 deletions(-)
io_uring/io_uring.c +15 -16
@@ -2761,6 +2761,15 @@ int rings_size(unsigned int flags, unsigned int sq_entries, {size_t cqe_size, off, sqe_size; +if (flags &IORING_SETUP_CQE_MIXED ) { +if (cq_entries <2 ) +return -EOVERFLOW ; + } +if (flags &IORING_SETUP_SQE_MIXED ) { +if (sq_entries <2 ) +return -EOVERFLOW ; + } + dims->sq_array_offset =SIZE_MAX ; sqe_size =sizeof (struct io_uring_sqe );
@@ -2786,29 +2795,19 @@ int rings_size(unsigned int flags, unsigned int sq_entries,if (off ==SIZE_MAX )return -EOVERFLOW ; -if (flags &IORING_SETUP_CQE_MIXED ) { -if (cq_entries <2 ) -return -EOVERFLOW ; - } -if (flags &IORING_SETUP_SQE_MIXED ) { -if (sq_entries <2 ) -return -EOVERFLOW ; - } -#ifdef CONFIG_SMP off =ALIGN (off,SMP_CACHE_BYTES );if (off ==0 )return -EOVERFLOW ;#endif -if (flags &IORING_SETUP_NO_SQARRAY ) { - dims->cq_comp_size = off; -return 0 ; - } +if (!(flags &IORING_SETUP_NO_SQARRAY )) { + dims->sq_array_offset = off; - dims->sq_array_offset = off; -if (check_add_overflow (off, dims->sq_array_size, &off)) -return -EOVERFLOW ; + off =size_add (off, dims->sq_array_size); +if (off ==SIZE_MAX ) +return -EOVERFLOW ; + } dims->cq_comp_size = off;return 0 ;
Signature · 1 line
2.49.0
RFC 14/16 io_uring: extract io_create_mem_region
In reply to Pavel BegunkovExtract a helper function for creating a memory region but that can be used at setup and not only in io_uring_register(). Specifically, it doesn't check IORING_SETUP_R_DISABLED.
--- io_uring/io_uring.h | 3 +++ io_uring/register.c | 43 +++++++++++++++++++++++++------------------ 2 files changed, 28 insertions(+), 18 deletions(-)
io_uring/io_uring.h +3 -0
@@ -200,6 +200,9 @@ void io_queue_next(struct io_kiocb *req);void io_task_refs_refill (struct io_uring_task *tctx);bool __io_alloc_req_refill (struct io_ring_ctx *ctx); +int io_create_mem_region (struct io_ring_ctx *ctx, +struct io_uring_mem_region_reg *reg); +void io_activate_pollwq (struct io_ring_ctx *ctx);static inline void io_lockdep_assert_cq_locked (struct io_ring_ctx *ctx)
io_uring/register.c +25 -18
71 lines not on this page · raw message
Signature · 1 line
2.49.0
RFC 15/16 io_uring: allow creating mem region at setup
In reply to Pavel BegunkovThis patch gives users a way to create a memory region at ring setup instead of requiring a registration call. First, it can be much more convenient when the region is used for wait parameters and hence requires IORING_SETUP_R_DISABLED. I'll also need it in the next patch for placing SCQ into it.
--- include/uapi/linux/io_uring.h | 7 ++++++- io_uring/io_uring.c | 26 ++++++++++++++++++++++++++ io_uring/io_uring.h | 1 + io_uring/register.c | 2 ++ 4 files changed, 35 insertions(+), 1 deletion(-)
include/uapi/linux/io_uring.h +6 -1
@@ -599,11 +599,16 @@ struct io_uring_params {__u32 sq_thread_idle;__u32 features;__u32 wq_fd; -__u32 resv[3 ]; +__u32 resv; +__u64 params_ext;/* pointer to struct io_uring_params_ext */ struct io_sqring_offsets sq_off;struct io_cqring_offsets cq_off; }; +struct io_uring_params_ext { +__u64 mem_region;/* pointer to struct io_uring_mem_region_reg */ +}; +/* * io_uring_params->features flags */
io_uring/io_uring.c +26 -0
@@ -3574,8 +3574,13 @@ int io_uring_fill_params(struct io_uring_params *p)static int io_prepare_config (struct io_ctx_config *config) {struct io_uring_params *p = &config->p; +struct io_uring_params_ext __user *ext_user;int ret; + ext_user =u64_to_user_ptr (config->p.params_ext); +if (ext_user &©_from_user (&config->ext, ext_user,sizeof (config->ext))) +return -EFAULT ; + ret =io_uring_sanitise_params (p);if (ret)return ret;
@@ -3592,6 +3597,22 @@ static int io_prepare_config(struct io_ctx_config *config)return 0 ; } +static int io_ctx_init_mem_region (struct io_ring_ctx *ctx, +struct io_ctx_config *config) +{ +struct io_uring_params_ext *e = &config->ext; +struct io_uring_mem_region_reg __user *mr_user; +struct io_uring_mem_region_reg mr; + + mr_user =u64_to_user_ptr (e->mem_region); +if (!mr_user) +return 0 ; + +if (copy_from_user (&mr, mr_user,sizeof (mr))) +return -EFAULT ; +return io_create_mem_region (ctx, &mr); +} +static __cold int io_uring_create (struct io_ctx_config *config) {struct io_uring_params *p = &config->p;
@@ -3661,10 +3682,15 @@ static __cold int io_uring_create(struct io_ctx_config *config)mmgrab (current->mm); ctx->mm_account = current->mm; + ret =io_ctx_init_mem_region (ctx, config); +if (ret) + goto err; + ret =io_allocate_scq_urings (ctx, config);if (ret) goto err; + ret =io_sq_offload_create (ctx, p);if (ret) goto err;
io_uring/io_uring.h +1 -0
@@ -30,6 +30,7 @@ struct io_scq_dim {struct io_ctx_config {struct io_uring_params p; +struct io_uring_params_ext ext;struct io_scq_dim dims;struct io_uring_params __user *uptr; };
io_uring/register.c +2 -0
@@ -414,6 +414,8 @@ static int io_register_resize_rings(struct io_ring_ctx *ctx, void __user *arg)return -EFAULT ;if (p.flags & ~RESIZE_FLAGS )return -EINVAL ; +if (p.params_ext) +return -EINVAL ;/* properties that are always inherited */ p.flags |= (ctx->flags ©_FLAGS );
Signature · 1 line
2.49.0
RFC 16/16 io_uring: introduce SCQ placement
In reply to Pavel BegunkovThere is a repeated problem with how io_uring manages rings. Specifically, it creates a new memory region for each ring and places entries together with headers. As the number of entries is always a power of 2, it usually means that it needs to allocate an additional page just for headers, which is wasteful. The headers structure size is also usually small and under the cache line size, however it's padded, which might mean additional cache bouncing.
Introduce a way for the user space to overlap SCQ headers and/or rings onto a pre-registered memory/parameter region. Each of them has a separate flag / offset, and they'll be attempted to be placed at the specified offset in the region. If the user doesn't request placement for SQ and/or CQ, io_uring will create a new memory region for them as before.
The second goal is to be able to put all components into a single region while knowing what's placed where. It's specifically interesting for planned BPF work, as it makes program writing much simpler.
Note: zcrx also have the same issue, but it's left out of this series.
--- include/uapi/linux/io_uring.h | 14 ++++ io_uring/io_uring.c | 143 ++++++++++++++++++++++++---------- io_uring/io_uring.h | 10 ++- io_uring/register.c | 4 +- 4 files changed, 128 insertions(+), 43 deletions(-)
include/uapi/linux/io_uring.h +14 -0
@@ -605,8 +605,22 @@ struct io_uring_params {struct io_cqring_offsets cq_off; }; +enum io_uring_scq_placement_flags { +IORING_PLACEMENT_SCQ_HDR = (1U <<0 ), +IORING_PLACEMENT_SQ = (1U <<1 ), +IORING_PLACEMENT_CQ = (1U <<2 ), +}; + +struct io_uring_scq_placement { +__u64 flags; +__u64 scq_hdr_off; +__u64 sq_off; +__u64 cq_off; +}; +struct io_uring_params_ext {__u64 mem_region;/* pointer to struct io_uring_mem_region_reg */ +struct io_uring_scq_placement placement; };/*
io_uring/io_uring.c +104 -39
209 lines not on this page · raw message
io_uring/io_uring.h +8 -2
@@ -25,7 +25,8 @@ struct io_scq_dim {size_t cq_size;/* Compound array mmap'ed together with CQ. */ -size_t cq_comp_size; +size_t rings_mr_size; +size_t sq_mr_size; };struct io_ctx_config {
@@ -35,6 +36,10 @@ struct io_ctx_config {struct io_uring_params __user *uptr; }; +#define IORING_PLACEMENT_MASK (IORING_PLACEMENT_SCQ_HDR |\ + IORING_PLACEMENT_SQ |\ + IORING_PLACEMENT_CQ) +#define IORING_FEAT_FLAGS (IORING_FEAT_SINGLE_MMAP |\ IORING_FEAT_NODROP |\ IORING_FEAT_SUBMIT_STABLE |\
@@ -153,7 +158,8 @@ static inline bool io_should_wake(struct io_wait_queue *iowq)#define IORING_MAX_CQ_ENTRIES (2 * IORING_MAX_ENTRIES)int rings_size (unsigned int flags,unsigned int sq_entries, -unsigned int cq_entries,struct io_scq_dim *dims); +unsigned int cq_entries,struct io_scq_dim *dims, +unsigned placement_flags);int io_uring_fill_params (struct io_uring_params *p);void io_fill_scq_offsets (struct io_uring_params *p,struct io_scq_dim *dims);bool io_cqe_cache_refill (struct io_ring_ctx *ctx,bool overflow,bool cqe32);
io_uring/register.c +2 -2
@@ -423,12 +423,12 @@ static int io_register_resize_rings(struct io_ring_ctx *ctx, void __user *arg) ret =io_uring_fill_params (&p);if (unlikely (ret))return ret; - ret =rings_size (p.flags, p.sq_entries, p.cq_entries, &dims); + ret =rings_size (p.flags, p.sq_entries, p.cq_entries, &dims,0 );if (ret)return ret;io_fill_scq_offsets (&p, &dims); - size = dims.cq_comp_size; + size = dims.rings_mr_size; sq_array_offset = dims.sq_array_offset;memset (&rd,0 ,sizeof (rd));
Signature · 1 line
2.49.0