further ring init cleanups
cover further ring init cleanups
There are several goals for this patch set. It deduplicates ring size and offset calculations between ring setup and resizing. It moves most of verification earlier before any allocations, which usually means simpler error handling. And it keeps the logic localised instead of spreading it across the file.
Pavel Begunkov (7):
io_uring: refactor rings_size nosqarray handling
io_uring: use size_add helpers for ring offsets
io_uring: convert params to pointer in ring reisze
io_uring: introduce struct io_ctx_config
io_uring: keep ring laoyut in a structure
io_uring: pre-calculate scq layout
io_uring: move cq/sq user offset init around
io_uring/io_uring.c | 137 ++++++++++++++++++++++++-------------------- io_uring/io_uring.h | 19 +++++- io_uring/register.c | 65 +++++++++------------ 3 files changed, 119 insertions(+), 102 deletions(-)
Signature · 1 line
2.49.0
PATCH 1/7 io_uring: refactor rings_size nosqarray handling
A preparation patch inversing the IORING_SETUP_NO_SQARRAY check, this way there is only one successful return path from the function, which will be helpful later.
--- io_uring/io_uring.c | 24 ++++++++++++------------ 1 file changed, 12 insertions(+), 12 deletions(-)
io_uring/io_uring.c +12 -12
@@ -2761,7 +2761,9 @@ unsigned long rings_size(unsigned int flags, unsigned int sq_entries, unsigned int cq_entries, size_t *sq_offset) { struct io_rings *rings; -size_t off, sq_array_size; +size_t off; + + *sq_offset =SIZE_MAX ; off =struct_size (rings, cqes, cq_entries);if (off ==SIZE_MAX )
@@ -2785,19 +2787,17 @@ unsigned long rings_size(unsigned int flags, unsigned int sq_entries,return SIZE_MAX ;#endif -if (flags &IORING_SETUP_NO_SQARRAY ) { - *sq_offset =SIZE_MAX ; -return off; - } - - *sq_offset = off; +if (!(flags &IORING_SETUP_NO_SQARRAY )) { +size_t sq_array_size; - sq_array_size =array_size (sizeof (u32), sq_entries); -if (sq_array_size ==SIZE_MAX ) -return SIZE_MAX ; + *sq_offset = off; -if (check_add_overflow (off, sq_array_size, &off)) -return SIZE_MAX ; + sq_array_size =array_size (sizeof (u32), sq_entries); +if (sq_array_size ==SIZE_MAX ) +return SIZE_MAX ; +if (check_add_overflow (off, sq_array_size, &off)) +return SIZE_MAX ; + }return off; }
Signature · 1 line
2.49.0
PATCH 2/7 io_uring: use size_add helpers for ring offsets
In reply to Pavel BegunkovUse size_add / size_mul set of functions for rings_size() calculations. It's more consistent with struct_size(), and errors are preserved across a series of calculations, so intermediate result checks can be omitted.
--- io_uring/io_uring.c | 18 ++++++++---------- 1 file changed, 8 insertions(+), 10 deletions(-)
io_uring/io_uring.c +8 -10
@@ -2765,13 +2765,6 @@ unsigned long rings_size(unsigned int flags, unsigned int sq_entries, *sq_offset =SIZE_MAX ; - off =struct_size (rings, cqes, cq_entries); -if (off ==SIZE_MAX ) -return SIZE_MAX ; -if (flags &IORING_SETUP_CQE32 ) { -if (check_shl_overflow (off,1 , &off)) -return SIZE_MAX ; - }if (flags &IORING_SETUP_CQE_MIXED ) {if (cq_entries <2 )return SIZE_MAX ;
@@ -2781,6 +2774,12 @@ unsigned long rings_size(unsigned int flags, unsigned int sq_entries,return SIZE_MAX ; } + off =struct_size (rings, cqes, cq_entries); +if (flags &IORING_SETUP_CQE32 ) + off =size_mul (off,2 ); +if (off ==SIZE_MAX ) +return SIZE_MAX ; +#ifdef CONFIG_SMP off =ALIGN (off,SMP_CACHE_BYTES );if (off ==0 )
@@ -2793,9 +2792,8 @@ unsigned long rings_size(unsigned int flags, unsigned int sq_entries, *sq_offset = off; sq_array_size =array_size (sizeof (u32), sq_entries); -if (sq_array_size ==SIZE_MAX ) -return SIZE_MAX ; -if (check_add_overflow (off, sq_array_size, &off)) + off =size_add (off, sq_array_size); +if (off ==SIZE_MAX )return SIZE_MAX ; }
Signature · 1 line
2.49.0
PATCH 3/7 io_uring: convert params to pointer in ring reisze
In reply to Pavel BegunkovThe parameters in io_register_resize_rings() will be moved into another structure in a later patch. In preparation to that, convert the params variable it to a pointer, but still store the data on stack.
--- io_uring/register.c | 48 ++++++++++++++++++++++----------------------- 1 file changed, 24 insertions(+), 24 deletions(-)
io_uring/register.c +24 -24
@@ -402,33 +402,33 @@ static int io_register_resize_rings(struct io_ring_ctx *ctx, void __user *arg)struct io_ring_ctx_rings o = { }, n = { }, *to_free =NULL ;size_t size, sq_array_offset;unsigned i, tail, old_head; -struct io_uring_params p; +struct io_uring_params __p, *p = &__p;int ret;/* limited to DEFER_TASKRUN for now */ if (!(ctx->flags &IORING_SETUP_DEFER_TASKRUN ))return -EINVAL ; -if (copy_from_user (&p, arg,sizeof (p))) +if (copy_from_user (p, arg,sizeof (*p)))return -EFAULT ; -if (p.flags & ~RESIZE_FLAGS ) +if (p->flags & ~RESIZE_FLAGS )return -EINVAL ;/* properties that are always inherited */ - p.flags |= (ctx->flags ©_FLAGS ); + p->flags |= (ctx->flags ©_FLAGS ); - ret =io_uring_fill_params (&p); + ret =io_uring_fill_params (p);if (unlikely (ret))return ret; - size =rings_size (p.flags, p.sq_entries, p.cq_entries, + size =rings_size (p->flags, p->sq_entries, p->cq_entries, &sq_array_offset);if (size ==SIZE_MAX )return -EOVERFLOW ;memset (&rd,0 ,sizeof (rd)); rd.size =PAGE_ALIGN (size); -if (p.flags &IORING_SETUP_NO_MMAP ) { - rd.user_addr = p.cq_off.user_addr; +if (p->flags &IORING_SETUP_NO_MMAP ) { + rd.user_addr = p->cq_off.user_addr; rd.flags |=IORING_MEM_REGION_TYPE_USER ; } ret =io_create_region (ctx, &n.ring_region, &rd,IORING_OFF_CQ_RING );
@@ -445,20 +445,20 @@ static int io_register_resize_rings(struct io_ring_ctx *ctx, void __user *arg)* intent... Use read/write once helpers from here on to indicate the * shared nature of it. */ -WRITE_ONCE (n.rings->sq_ring_mask, p.sq_entries -1 ); -WRITE_ONCE (n.rings->cq_ring_mask, p.cq_entries -1 ); -WRITE_ONCE (n.rings->sq_ring_entries, p.sq_entries); -WRITE_ONCE (n.rings->cq_ring_entries, p.cq_entries); +WRITE_ONCE (n.rings->sq_ring_mask, p->sq_entries -1 ); +WRITE_ONCE (n.rings->cq_ring_mask, p->cq_entries -1 ); +WRITE_ONCE (n.rings->sq_ring_entries, p->sq_entries); +WRITE_ONCE (n.rings->cq_ring_entries, p->cq_entries); -if (copy_to_user (arg, &p,sizeof (p))) { +if (copy_to_user (arg, p,sizeof (*p))) {io_register_free_rings (ctx, &n);return -EFAULT ; } -if (p.flags &IORING_SETUP_SQE128 ) - size =array_size (2 *sizeof (struct io_uring_sqe ), p.sq_entries); +if (p->flags &IORING_SETUP_SQE128 ) + size =array_size (2 *sizeof (struct io_uring_sqe ), p->sq_entries);else - size =array_size (sizeof (struct io_uring_sqe ), p.sq_entries); + size =array_size (sizeof (struct io_uring_sqe ), p->sq_entries);if (size ==SIZE_MAX ) {io_register_free_rings (ctx, &n);return -EOVERFLOW ;
@@ -466,8 +466,8 @@ static int io_register_resize_rings(struct io_ring_ctx *ctx, void __user *arg)memset (&rd,0 ,sizeof (rd)); rd.size =PAGE_ALIGN (size); -if (p.flags &IORING_SETUP_NO_MMAP ) { - rd.user_addr = p.sq_off.user_addr; +if (p->flags &IORING_SETUP_NO_MMAP ) { + rd.user_addr = p->sq_off.user_addr; rd.flags |=IORING_MEM_REGION_TYPE_USER ; } ret =io_create_region (ctx, &n.sq_region, &rd,IORING_OFF_SQES );
@@ -508,11 +508,11 @@ static int io_register_resize_rings(struct io_ring_ctx *ctx, void __user *arg)*/ tail =READ_ONCE (o.rings->sq.tail); old_head =READ_ONCE (o.rings->sq.head); -if (tail - old_head > p.sq_entries) +if (tail - old_head > p->sq_entries) goto overflow;for (i = old_head; i < tail; i++) {unsigned src_head = i & (ctx->sq_entries -1 ); -unsigned dst_head = i & (p.sq_entries -1 ); +unsigned dst_head = i & (p->sq_entries -1 ); n.sq_sqes[dst_head] = o.sq_sqes[src_head]; }
@@ -521,7 +521,7 @@ static int io_register_resize_rings(struct io_ring_ctx *ctx, void __user *arg) tail =READ_ONCE (o.rings->cq.tail); old_head =READ_ONCE (o.rings->cq.head); -if (tail - old_head > p.cq_entries) { +if (tail - old_head > p->cq_entries) { overflow:/* restore old rings, and return -EOVERFLOW via cleanup path */ ctx->rings = o.rings;
@@ -532,7 +532,7 @@ static int io_register_resize_rings(struct io_ring_ctx *ctx, void __user *arg) }for (i = old_head; i < tail; i++) {unsigned src_head = i & (ctx->cq_entries -1 ); -unsigned dst_head = i & (p.cq_entries -1 ); +unsigned dst_head = i & (p->cq_entries -1 ); n.rings->cqes[dst_head] = o.rings->cqes[src_head]; }
@@ -550,8 +550,8 @@ static int io_register_resize_rings(struct io_ring_ctx *ctx, void __user *arg)if (!(ctx->flags &IORING_SETUP_NO_SQARRAY )) ctx->sq_array = (u32 *)((char *)n.rings + sq_array_offset); - ctx->sq_entries = p.sq_entries; - ctx->cq_entries = p.cq_entries; + ctx->sq_entries = p->sq_entries; + ctx->cq_entries = p->cq_entries; ctx->rings = n.rings; ctx->sq_sqes = n.sq_sqes;
Signature · 1 line
2.49.0
PATCH 4/7 io_uring: introduce struct io_ctx_config
In reply to Pavel BegunkovThere will be more information needed during ctx setup, and instead of passing a handful of pointers around, wrap them all into a new structure. Add a helper for encapsulating all configuration checks and preparation, that's also reused for ring resizing.
Note, it indirectly adds a io_uring_sanitise_params() check to ring resizing, which is a good thing.
--- io_uring/io_uring.c | 41 ++++++++++++++++++++++++++++------------- io_uring/io_uring.h | 8 +++++++- io_uring/register.c | 7 +++++-- 3 files changed, 40 insertions(+), 16 deletions(-)
io_uring/io_uring.c +28 -13
@@ -3480,7 +3480,7 @@ static int io_uring_sanitise_params(struct io_uring_params *p)return 0 ; } -int io_uring_fill_params (struct io_uring_params *p) +static int io_uring_fill_params (struct io_uring_params *p) {unsigned entries = p->sq_entries;
@@ -3545,12 +3545,9 @@ int io_uring_fill_params(struct io_uring_params *p)return 0 ; } -static __cold int io_uring_create (struct io_uring_params *p, -struct io_uring_params __user *params) +int io_prepare_config (struct io_ctx_config *config) { -struct io_ring_ctx *ctx; -struct io_uring_task *tctx; -struct file *file; +struct io_uring_params *p = &config->p;int ret; ret =io_uring_sanitise_params (p);
@@ -3558,7 +3555,22 @@ static __cold int io_uring_create(struct io_uring_params *p,return ret; ret =io_uring_fill_params (p); -if (unlikely (ret)) +if (ret) +return ret; + +return 0 ; +} + +static __cold int io_uring_create (struct io_ctx_config *config) +{ +struct io_uring_params *p = &config->p; +struct io_ring_ctx *ctx; +struct io_uring_task *tctx; +struct file *file; +int ret; + + ret =io_prepare_config (config); +if (ret)return ret; ctx =io_ring_ctx_alloc (p);
@@ -3631,7 +3643,7 @@ static __cold int io_uring_create(struct io_uring_params *p, p->features =IORING_FEAT_FLAGS ; -if (copy_to_user (params, p,sizeof (*p))) { +if (copy_to_user (config->uptr, p,sizeof (*p))) { ret = -EFAULT ; goto err; }
@@ -3684,16 +3696,19 @@ static __cold int io_uring_create(struct io_uring_params *p,*/ static long io_uring_setup (u32 entries,struct io_uring_params __user *params) { -struct io_uring_params p; +struct io_ctx_config config; + +memset (&config,0 ,sizeof (config)); -if (copy_from_user (&p, params,sizeof (p))) +if (copy_from_user (&config.p, params,sizeof (config.p)))return -EFAULT ; -if (!mem_is_zero (&p.resv,sizeof (p.resv))) +if (!mem_is_zero (&config.p.resv,sizeof (config.p.resv)))return -EINVAL ; - p.sq_entries = entries; -return io_uring_create (&p, params); + config.p.sq_entries = entries; + config.uptr = params; +return io_uring_create (&config); }static inline int io_uring_allowed (void )
io_uring/io_uring.h +7 -1
@@ -17,6 +17,11 @@#include <trace/events/io_uring.h> #endif +struct io_ctx_config { +struct io_uring_params p; +struct io_uring_params __user *uptr; +}; +#define IORING_FEAT_FLAGS (IORING_FEAT_SINGLE_MMAP |\ IORING_FEAT_NODROP |\ IORING_FEAT_SUBMIT_STABLE |\
@@ -136,7 +141,8 @@ static inline bool io_should_wake(struct io_wait_queue *iowq)unsigned long rings_size (unsigned int flags,unsigned int sq_entries,unsigned int cq_entries,size_t *sq_offset); -int io_uring_fill_params (struct io_uring_params *p); +int io_prepare_config (struct io_ctx_config *config); +bool io_cqe_cache_refill (struct io_ring_ctx *ctx,bool overflow,bool cqe32);int io_run_task_work_sig (struct io_ring_ctx *ctx);int io_run_local_work (struct io_ring_ctx *ctx,int min_events,int max_events);
io_uring/register.c +5 -2
@@ -398,13 +398,16 @@ static void io_register_free_rings(struct io_ring_ctx *ctx,static int io_register_resize_rings (struct io_ring_ctx *ctx,void __user *arg) { +struct io_ctx_config config;struct io_uring_region_desc rd;struct io_ring_ctx_rings o = { }, n = { }, *to_free =NULL ;size_t size, sq_array_offset;unsigned i, tail, old_head; -struct io_uring_params __p, *p = &__p; +struct io_uring_params *p = &config.p;int ret; +memset (&config,0 ,sizeof (config)); +/* limited to DEFER_TASKRUN for now */ if (!(ctx->flags &IORING_SETUP_DEFER_TASKRUN ))return -EINVAL ;
@@ -416,7 +419,7 @@ static int io_register_resize_rings(struct io_ring_ctx *ctx, void __user *arg)/* properties that are always inherited */ p->flags |= (ctx->flags ©_FLAGS ); - ret =io_uring_fill_params (p); + ret =io_prepare_config (&config);if (unlikely (ret))return ret;
Signature · 1 line
2.49.0
PATCH 5/7 io_uring: keep ring laoyut in a structure
In reply to Pavel BegunkovAdd a structure keeping SQ/CQ sizes and offsets. For now it only records data previously returned from rings_size and the SQ size.
--- io_uring/io_uring.c | 54 +++++++++++++++++++++++---------------------- io_uring/io_uring.h | 12 ++++++++-- io_uring/register.c | 24 ++++++-------------- 3 files changed, 45 insertions(+), 45 deletions(-)
io_uring/io_uring.c +28 -26
@@ -2757,47 +2757,57 @@ static void io_rings_free(struct io_ring_ctx *ctx) ctx->sq_sqes =NULL ; } -unsigned long rings_size (unsigned int flags,unsigned int sq_entries, -unsigned int cq_entries,size_t *sq_offset) +int rings_size (unsigned int flags,unsigned int sq_entries, +unsigned int cq_entries,struct io_rings_layout *rl) {struct io_rings *rings; +size_t sqe_size;size_t off; - *sq_offset =SIZE_MAX ; -if (flags &IORING_SETUP_CQE_MIXED ) {if (cq_entries <2 ) -return SIZE_MAX ; +return -EOVERFLOW ; }if (flags &IORING_SETUP_SQE_MIXED ) {if (sq_entries <2 ) -return SIZE_MAX ; +return -EOVERFLOW ; } + rl->sq_array_offset =SIZE_MAX ; + + sqe_size =sizeof (struct io_uring_sqe ); +if (flags &IORING_SETUP_SQE128 ) + sqe_size *=2 ; + + rl->sq_size =array_size (sqe_size, sq_entries); +if (rl->sq_size ==SIZE_MAX ) +return -EOVERFLOW ; + off =struct_size (rings, cqes, cq_entries);if (flags &IORING_SETUP_CQE32 ) off =size_mul (off,2 );if (off ==SIZE_MAX ) -return SIZE_MAX ; +return -EOVERFLOW ;#ifdef CONFIG_SMP off =ALIGN (off,SMP_CACHE_BYTES );if (off ==0 ) -return SIZE_MAX ; +return -EOVERFLOW ;#endif if (!(flags &IORING_SETUP_NO_SQARRAY )) {size_t sq_array_size; - *sq_offset = off; + rl->sq_array_offset = off; sq_array_size =array_size (sizeof (u32), sq_entries); off =size_add (off, sq_array_size);if (off ==SIZE_MAX ) -return SIZE_MAX ; +return -EOVERFLOW ; } -return off; + rl->rings_size = off; +return 0 ; }static __cold void __io_req_caches_free (struct io_ring_ctx *ctx)
@@ -3346,28 +3356,20 @@ static __cold int io_allocate_scq_urings(struct io_ring_ctx *ctx, struct io_uring_params *p) {struct io_uring_region_desc rd; +struct io_rings_layout __rl, *rl = &__rl;struct io_rings *rings; -size_t sq_array_offset; -size_t sq_size, cq_size, sqe_size;int ret;/* make sure these are sane, as we already accounted them */ ctx->sq_entries = p->sq_entries; ctx->cq_entries = p->cq_entries; - sqe_size =sizeof (struct io_uring_sqe ); -if (p->flags &IORING_SETUP_SQE128 ) - sqe_size *=2 ; - sq_size =array_size (sqe_size, p->sq_entries); -if (sq_size ==SIZE_MAX ) -return -EOVERFLOW ; - cq_size =rings_size (ctx->flags, p->sq_entries, p->cq_entries, - &sq_array_offset); -if (cq_size ==SIZE_MAX ) -return -EOVERFLOW ; + ret =rings_size (ctx->flags, p->sq_entries, p->cq_entries, rl); +if (ret) +return ret;memset (&rd,0 ,sizeof (rd)); - rd.size =PAGE_ALIGN (cq_size); + rd.size =PAGE_ALIGN (rl->rings_size);if (ctx->flags &IORING_SETUP_NO_MMAP ) { rd.user_addr = p->cq_off.user_addr; rd.flags |=IORING_MEM_REGION_TYPE_USER ;
@@ -3378,10 +3380,10 @@ static __cold int io_allocate_scq_urings(struct io_ring_ctx *ctx, ctx->rings = rings =io_region_get_ptr (&ctx->ring_region);if (!(ctx->flags &IORING_SETUP_NO_SQARRAY )) - ctx->sq_array = (u32 *)((char *)rings + sq_array_offset); + ctx->sq_array = (u32 *)((char *)rings + rl->sq_array_offset);memset (&rd,0 ,sizeof (rd)); - rd.size =PAGE_ALIGN (sq_size); + rd.size =PAGE_ALIGN (rl->sq_size);if (ctx->flags &IORING_SETUP_NO_MMAP ) { rd.user_addr = p->sq_off.user_addr; rd.flags |=IORING_MEM_REGION_TYPE_USER ;
io_uring/io_uring.h +10 -2
@@ -17,6 +17,14 @@#include <trace/events/io_uring.h> #endif +struct io_rings_layout { +/* size of CQ + headers + SQ offset array */ +size_t rings_size; +size_t sq_size; + +size_t sq_array_offset; +}; +struct io_ctx_config {struct io_uring_params p;struct io_uring_params __user *uptr;
@@ -139,8 +147,8 @@ static inline bool io_should_wake(struct io_wait_queue *iowq)#define IORING_MAX_ENTRIES 32768#define IORING_MAX_CQ_ENTRIES (2 * IORING_MAX_ENTRIES) -unsigned long rings_size (unsigned int flags,unsigned int sq_entries, -unsigned int cq_entries,size_t *sq_offset); +int rings_size (unsigned int flags,unsigned int sq_entries, +unsigned int cq_entries,struct io_rings_layout *rl);int io_prepare_config (struct io_ctx_config *config);bool io_cqe_cache_refill (struct io_ring_ctx *ctx,bool overflow,bool cqe32);
io_uring/register.c +7 -17
@@ -401,9 +401,9 @@ static int io_register_resize_rings(struct io_ring_ctx *ctx, void __user *arg)struct io_ctx_config config;struct io_uring_region_desc rd;struct io_ring_ctx_rings o = { }, n = { }, *to_free =NULL ; -size_t size, sq_array_offset;unsigned i, tail, old_head;struct io_uring_params *p = &config.p; +struct io_rings_layout __rl, *rl = &__rl;int ret;memset (&config,0 ,sizeof (config));
@@ -423,13 +423,12 @@ static int io_register_resize_rings(struct io_ring_ctx *ctx, void __user *arg)if (unlikely (ret))return ret; - size =rings_size (p->flags, p->sq_entries, p->cq_entries, - &sq_array_offset); -if (size ==SIZE_MAX ) -return -EOVERFLOW ; + ret =rings_size (p->flags, p->sq_entries, p->cq_entries, rl); +if (ret) +return ret;memset (&rd,0 ,sizeof (rd)); - rd.size =PAGE_ALIGN (size); + rd.size =PAGE_ALIGN (rl->rings_size);if (p->flags &IORING_SETUP_NO_MMAP ) { rd.user_addr = p->cq_off.user_addr; rd.flags |=IORING_MEM_REGION_TYPE_USER ;
@@ -458,17 +457,8 @@ static int io_register_resize_rings(struct io_ring_ctx *ctx, void __user *arg)return -EFAULT ; } -if (p->flags &IORING_SETUP_SQE128 ) - size =array_size (2 *sizeof (struct io_uring_sqe ), p->sq_entries); -else - size =array_size (sizeof (struct io_uring_sqe ), p->sq_entries); -if (size ==SIZE_MAX ) { -io_register_free_rings (ctx, &n); -return -EOVERFLOW ; - } -memset (&rd,0 ,sizeof (rd)); - rd.size =PAGE_ALIGN (size); + rd.size =PAGE_ALIGN (rl->sq_size);if (p->flags &IORING_SETUP_NO_MMAP ) { rd.user_addr = p->sq_off.user_addr; rd.flags |=IORING_MEM_REGION_TYPE_USER ;
@@ -551,7 +541,7 @@ static int io_register_resize_rings(struct io_ring_ctx *ctx, void __user *arg)/* all done, store old pointers and assign new ones */ if (!(ctx->flags &IORING_SETUP_NO_SQARRAY )) - ctx->sq_array = (u32 *)((char *)n.rings + sq_array_offset); + ctx->sq_array = (u32 *)((char *)n.rings + rl->sq_array_offset); ctx->sq_entries = p->sq_entries; ctx->cq_entries = p->cq_entries;
Signature · 1 line
2.49.0
PATCH 6/7 io_uring: pre-calculate scq layout
In reply to Pavel BegunkovMove ring layouts calculations into io_prepare_config(), so that more misconfiguration checking can be done earlier before creating a ctx. It also deduplicates some code with ring resizing. And as a bonus, now it initialises params->sq_off.array, which is closer to all other user offset init, and also applies it to ring resizing, which was previously missing it.
--- io_uring/io_uring.c | 27 ++++++++++++++------------- io_uring/io_uring.h | 3 +-- io_uring/register.c | 4 ---- 3 files changed, 15 insertions(+), 19 deletions(-)
io_uring/io_uring.c +14 -13
@@ -2757,8 +2757,8 @@ static void io_rings_free(struct io_ring_ctx *ctx) ctx->sq_sqes =NULL ; } -int rings_size (unsigned int flags,unsigned int sq_entries, -unsigned int cq_entries,struct io_rings_layout *rl) +static int rings_size (unsigned int flags,unsigned int sq_entries, +unsigned int cq_entries,struct io_rings_layout *rl) {struct io_rings *rings;size_t sqe_size;
@@ -3353,10 +3353,11 @@ bool io_is_uring_fops(struct file *file) }static __cold int io_allocate_scq_urings (struct io_ring_ctx *ctx, -struct io_uring_params *p) +struct io_ctx_config *config) { +struct io_uring_params *p = &config->p; +struct io_rings_layout *rl = &config->layout;struct io_uring_region_desc rd; -struct io_rings_layout __rl, *rl = &__rl;struct io_rings *rings;int ret;
@@ -3364,10 +3365,6 @@ static __cold int io_allocate_scq_urings(struct io_ring_ctx *ctx, ctx->sq_entries = p->sq_entries; ctx->cq_entries = p->cq_entries; - ret =rings_size (ctx->flags, p->sq_entries, p->cq_entries, rl); -if (ret) -return ret; -memset (&rd,0 ,sizeof (rd)); rd.size =PAGE_ALIGN (rl->rings_size);if (ctx->flags &IORING_SETUP_NO_MMAP ) {
@@ -3378,7 +3375,6 @@ static __cold int io_allocate_scq_urings(struct io_ring_ctx *ctx,if (ret)return ret; ctx->rings = rings =io_region_get_ptr (&ctx->ring_region); -if (!(ctx->flags &IORING_SETUP_NO_SQARRAY )) ctx->sq_array = (u32 *)((char *)rings + rl->sq_array_offset);
@@ -3560,6 +3556,14 @@ int io_prepare_config(struct io_ctx_config *config)if (ret)return ret; + ret =rings_size (p->flags, p->sq_entries, p->cq_entries, + &config->layout); +if (ret) +return ret; + +if (!(p->flags &IORING_SETUP_NO_SQARRAY )) + p->sq_off.array = config->layout.sq_array_offset; +return 0 ; }
@@ -3632,13 +3636,10 @@ static __cold int io_uring_create(struct io_ctx_config *config)mmgrab (current->mm); ctx->mm_account = current->mm; - ret =io_allocate_scq_urings (ctx, p); + ret =io_allocate_scq_urings (ctx, config);if (ret) goto err; -if (!(p->flags &IORING_SETUP_NO_SQARRAY )) - p->sq_off.array = (char *)ctx->sq_array - (char *)ctx->rings; - ret =io_sq_offload_create (ctx, p);if (ret) goto err;
io_uring/io_uring.h +1 -2
@@ -27,6 +27,7 @@ struct io_rings_layout {struct io_ctx_config {struct io_uring_params p; +struct io_rings_layout layout;struct io_uring_params __user *uptr; };
@@ -147,8 +148,6 @@ static inline bool io_should_wake(struct io_wait_queue *iowq)#define IORING_MAX_ENTRIES 32768#define IORING_MAX_CQ_ENTRIES (2 * IORING_MAX_ENTRIES) -int rings_size (unsigned int flags,unsigned int sq_entries, -unsigned int cq_entries,struct io_rings_layout *rl);int io_prepare_config (struct io_ctx_config *config);bool io_cqe_cache_refill (struct io_ring_ctx *ctx,bool overflow,bool cqe32);
io_uring/register.c +0 -4
@@ -423,10 +423,6 @@ static int io_register_resize_rings(struct io_ring_ctx *ctx, void __user *arg)if (unlikely (ret))return ret; - ret =rings_size (p->flags, p->sq_entries, p->cq_entries, rl); -if (ret) -return ret; -memset (&rd,0 ,sizeof (rd)); rd.size =PAGE_ALIGN (rl->rings_size);if (p->flags &IORING_SETUP_NO_MMAP ) {
Signature · 1 line
2.49.0
PATCH 7/7 io_uring: move cq/sq user offset init around
In reply to Pavel BegunkovMove user SQ/CQ offset initialisation at the end of io_prepare_config() where it already calculated all information to set it properly.
--- io_uring/io_uring.c | 41 ++++++++++++++++++++--------------------- 1 file changed, 20 insertions(+), 21 deletions(-)
io_uring/io_uring.c +20 -21
@@ -3519,27 +3519,6 @@ static int io_uring_fill_params(struct io_uring_params *p) p->cq_entries =2 * p->sq_entries; } - p->sq_off.head = offsetof(struct io_rings, sq.head); - p->sq_off.tail = offsetof(struct io_rings, sq.tail); - p->sq_off.ring_mask = offsetof(struct io_rings , sq_ring_mask); - p->sq_off.ring_entries = offsetof(struct io_rings , sq_ring_entries); - p->sq_off.flags = offsetof(struct io_rings , sq_flags); - p->sq_off.dropped = offsetof(struct io_rings , sq_dropped); - p->sq_off.resv1 =0 ; -if (!(p->flags &IORING_SETUP_NO_MMAP )) - p->sq_off.user_addr =0 ; - - p->cq_off.head = offsetof(struct io_rings, cq.head); - p->cq_off.tail = offsetof(struct io_rings, cq.tail); - p->cq_off.ring_mask = offsetof(struct io_rings , cq_ring_mask); - p->cq_off.ring_entries = offsetof(struct io_rings , cq_ring_entries); - p->cq_off.overflow = offsetof(struct io_rings , cq_overflow); - p->cq_off.cqes = offsetof(struct io_rings , cqes); - p->cq_off.flags = offsetof(struct io_rings , cq_flags); - p->cq_off.resv1 =0 ; -if (!(p->flags &IORING_SETUP_NO_MMAP )) - p->cq_off.user_addr =0 ; -return 0 ; }
@@ -3561,6 +3540,26 @@ int io_prepare_config(struct io_ctx_config *config)if (ret)return ret; + p->sq_off.head = offsetof(struct io_rings, sq.head); + p->sq_off.tail = offsetof(struct io_rings, sq.tail); + p->sq_off.ring_mask = offsetof(struct io_rings , sq_ring_mask); + p->sq_off.ring_entries = offsetof(struct io_rings , sq_ring_entries); + p->sq_off.flags = offsetof(struct io_rings , sq_flags); + p->sq_off.dropped = offsetof(struct io_rings , sq_dropped); + p->sq_off.resv1 =0 ; +if (!(p->flags &IORING_SETUP_NO_MMAP )) + p->sq_off.user_addr =0 ; + + p->cq_off.head = offsetof(struct io_rings, cq.head); + p->cq_off.tail = offsetof(struct io_rings, cq.tail); + p->cq_off.ring_mask = offsetof(struct io_rings , cq_ring_mask); + p->cq_off.ring_entries = offsetof(struct io_rings , cq_ring_entries); + p->cq_off.overflow = offsetof(struct io_rings , cq_overflow); + p->cq_off.cqes = offsetof(struct io_rings , cqes); + p->cq_off.flags = offsetof(struct io_rings , cq_flags); + p->cq_off.resv1 =0 ; +if (!(p->flags &IORING_SETUP_NO_MMAP )) + p->cq_off.user_addr =0 ;if (!(p->flags &IORING_SETUP_NO_SQARRAY )) p->sq_off.array = config->layout.sq_array_offset;
Signature · 1 line
2.49.0
Jens Axboe
maintainerQuoted 15 lines · Pavel Begunkov
There are several goals for this patch set. It deduplicates ring size and offset calculations between ring setup and resizing. It moves most of verification earlier before any allocations, which usually means simpler error handling. And it keeps the logic localised instead of spreading it across the file.
Pavel Begunkov (7):
io_uring: refactorrings_sizenosqarray handling
io_uring: usesize_addhelpers for ring offsets
io_uring: convert params to pointer in ring reisze
io_uring: introduce structio_ctx_config
io_uring: keep ring laoyut in a structure
io_uring: pre-calculate scq layout
io_uring: move cq/sq user offset init around
Quoted 4 lines
io_uring/io_uring.c | 137 ++++++++++++++++++++++++-------------------- io_uring/io_uring.h | 19 +++++- io_uring/register.c | 65 +++++++++------------ 3 files changed, 119 insertions(+), 102 deletions(-)
All looks pretty straightforward to me. I'm going to let it simmer for a day or so to give other folks a chance to take a look too.
Signature · 1 line
Jens Axboe
Quoted 15 lines · Pavel Begunkov
There are several goals for this patch set. It deduplicates ring size and offset calculations between ring setup and resizing. It moves most of verification earlier before any allocations, which usually means simpler error handling. And it keeps the logic localised instead of spreading it across the file.
Pavel Begunkov (7):
io_uring: refactorrings_sizenosqarray handling
io_uring: usesize_addhelpers for ring offsets
io_uring: convert params to pointer in ring reisze
io_uring: introduce structio_ctx_config
io_uring: keep ring laoyut in a structure
io_uring: pre-calculate scq layout
io_uring: move cq/sq user offset init around
Quoted 1 line
[...]
Applied, thanks!
[1/7] io_uring: refactor rings_size nosqarray handling
commit: e279bb4b4c4d012808fb21ff41183a2e76c26679
[2/7] io_uring: use size_add helpers for ring offsets
commit: 94cd832916521d8d51b25b40691354c24831c655
[3/7] io_uring: convert params to pointer in ring reisze
commit: 929dbbb699110c9377da721ed7b44a660bb4ee01
[4/7] io_uring: introduce struct io_ctx_config
commit: 0f4b537363cb66c78e97bb58c26986af62856356
[5/7] io_uring: keep ring laoyut in a structure
commit: 001b76b7e755767d847e9aebf1fd6e525f1e58c8
[6/7] io_uring: pre-calculate scq layout
commit: eb76ff6a6829a9a54a385804cc9dbe4460f156d6
[7/7] io_uring: move cq/sq user offset init around
commit: d741c6255524f0691aea53381219fadcd2b38408Best regards,
Signature · 1 line
Jens Axboe