random region / rings cleanups
cover random region / rings cleanups
Random stuff that will my future changes simpler but flushing it out as it looks like a fine cleanup series. It also replaces io_create_region_mmap_safe() calls with non-mmap version where it's possible, and removes the helper in Patch 7 while handling io_register_mem_region() a bit more gracefully.
Pavel Begunkov (7):
io_uring: deduplicate array_size in io_allocate_scq_urings
io_uring: sanity check sizes before attempting allocation
io_uring: use no mmap safe region helpers on resizing
io_uring: remove extra args from io_register_free_rings
io_uring: don't free never created regions io_uring/kbuf: use io_create_region for kbuf creation
io_uring: only publish fully handled mem region
io_uring/io_uring.c | 26 ++++++++++++-------------- io_uring/kbuf.c | 2 +- io_uring/memmap.c | 21 --------------------- io_uring/memmap.h | 12 ++++++++++++ io_uring/register.c | 29 ++++++++++++++--------------- 5 files changed, 39 insertions(+), 51 deletions(-)
Signature · 1 line
2.49.0
PATCH 1/7 io_uring: deduplicate array_size in io_allocate_scq_urings
A minor cleanup precomputing the sq size first instead of branching array_size() in io_allocate_scq_urings().
--- io_uring/io_uring.c | 8 +++++--- 1 file changed, 5 insertions(+), 3 deletions(-)
io_uring/io_uring.c +5 -3
@@ -3607,6 +3607,7 @@ static __cold int io_allocate_scq_urings(struct io_ring_ctx *ctx,struct io_uring_region_desc rd;struct io_rings *rings;size_t size, sq_array_offset; +size_t sqe_size;int ret;/* make sure these are sane, as we already accounted them */
@@ -3636,10 +3637,11 @@ static __cold int io_allocate_scq_urings(struct io_ring_ctx *ctx, rings->sq_ring_entries = p->sq_entries; rings->cq_ring_entries = p->cq_entries; + sqe_size =sizeof (struct io_uring_sqe );if (p->flags &IORING_SETUP_SQE128 ) - size =array_size (2 *sizeof (struct io_uring_sqe ), p->sq_entries); -else - size =array_size (sizeof (struct io_uring_sqe ), p->sq_entries); + sqe_size *=2 ; + + size =array_size (sqe_size, p->sq_entries);if (size ==SIZE_MAX ) {io_rings_free (ctx);return -EOVERFLOW ;
Signature · 1 line
2.49.0
PATCH 2/7 io_uring: sanity check sizes before attempting allocation
In reply to Pavel BegunkovIt's a good practice to validate parameters before doing any heavy stuff like queue allocations. Do that for io_allocate_scq_urings().
--- io_uring/io_uring.c | 28 ++++++++++++---------------- 1 file changed, 12 insertions(+), 16 deletions(-)
io_uring/io_uring.c +12 -16
@@ -3606,21 +3606,27 @@ static __cold int io_allocate_scq_urings(struct io_ring_ctx *ctx, {struct io_uring_region_desc rd;struct io_rings *rings; -size_t size, sq_array_offset; -size_t sqe_size; +size_t sq_array_offset; +size_t sq_size, cq_size, sqe_size;int ret;/* make sure these are sane, as we already accounted them */ ctx->sq_entries = p->sq_entries; ctx->cq_entries = p->cq_entries; - size =rings_size (ctx->flags, p->sq_entries, p->cq_entries, + sqe_size =sizeof (struct io_uring_sqe ); +if (p->flags &IORING_SETUP_SQE128 ) + sqe_size *=2 ; + sq_size =array_size (sqe_size, p->sq_entries); +if (sq_size ==SIZE_MAX ) +return -EOVERFLOW ; + cq_size =rings_size (ctx->flags, p->sq_entries, p->cq_entries, &sq_array_offset); -if (size ==SIZE_MAX ) +if (cq_size ==SIZE_MAX )return -EOVERFLOW ;memset (&rd,0 ,sizeof (rd)); - rd.size =PAGE_ALIGN (size); + rd.size =PAGE_ALIGN (cq_size);if (ctx->flags &IORING_SETUP_NO_MMAP ) { rd.user_addr = p->cq_off.user_addr; rd.flags |=IORING_MEM_REGION_TYPE_USER ;
@@ -3637,18 +3643,8 @@ static __cold int io_allocate_scq_urings(struct io_ring_ctx *ctx, rings->sq_ring_entries = p->sq_entries; rings->cq_ring_entries = p->cq_entries; - sqe_size =sizeof (struct io_uring_sqe ); -if (p->flags &IORING_SETUP_SQE128 ) - sqe_size *=2 ; - - size =array_size (sqe_size, p->sq_entries); -if (size ==SIZE_MAX ) { -io_rings_free (ctx); -return -EOVERFLOW ; - } -memset (&rd,0 ,sizeof (rd)); - rd.size =PAGE_ALIGN (size); + rd.size =PAGE_ALIGN (sq_size);if (ctx->flags &IORING_SETUP_NO_MMAP ) { rd.user_addr = p->sq_off.user_addr; rd.flags |=IORING_MEM_REGION_TYPE_USER ;
Signature · 1 line
2.49.0
PATCH 3/7 io_uring: use no mmap safe region helpers on resizing
In reply to Pavel Begunkovio_create_region_mmap_safe() is only needed when the created region is exposed to userspace code via mmap. io_register_resize_rings() creates them locally on stack, so the no mmap_safe version of the helper is enough.
--- io_uring/register.c | 4 ++-- 1 file changed, 2 insertions(+), 2 deletions(-)
io_uring/register.c +2 -2
@@ -432,7 +432,7 @@ static int io_register_resize_rings(struct io_ring_ctx *ctx, void __user *arg) rd.user_addr = p.cq_off.user_addr; rd.flags |=IORING_MEM_REGION_TYPE_USER ; } - ret =io_create_region_mmap_safe (ctx, &n.ring_region, &rd,IORING_OFF_CQ_RING ); + ret =io_create_region (ctx, &n.ring_region, &rd,IORING_OFF_CQ_RING );if (ret) {io_register_free_rings (ctx, &p, &n);return ret;
@@ -472,7 +472,7 @@ static int io_register_resize_rings(struct io_ring_ctx *ctx, void __user *arg) rd.user_addr = p.sq_off.user_addr; rd.flags |=IORING_MEM_REGION_TYPE_USER ; } - ret =io_create_region_mmap_safe (ctx, &n.sq_region, &rd,IORING_OFF_SQES ); + ret =io_create_region (ctx, &n.sq_region, &rd,IORING_OFF_SQES );if (ret) {io_register_free_rings (ctx, &p, &n);return ret;
Signature · 1 line
2.49.0
PATCH 4/7 io_uring: remove extra args from io_register_free_rings
In reply to Pavel Begunkovio_register_free_rings() doesn't use its "struct io_uring_params" parameter, remove it.
--- io_uring/register.c | 11 +++++------ 1 file changed, 5 insertions(+), 6 deletions(-)
io_uring/register.c +5 -6
@@ -379,7 +379,6 @@ struct io_ring_ctx_rings { };static void io_register_free_rings (struct io_ring_ctx *ctx, -struct io_uring_params *p,struct io_ring_ctx_rings *r) {io_free_region (ctx, &r->sq_region);
@@ -434,7 +433,7 @@ static int io_register_resize_rings(struct io_ring_ctx *ctx, void __user *arg) } ret =io_create_region (ctx, &n.ring_region, &rd,IORING_OFF_CQ_RING );if (ret) { -io_register_free_rings (ctx, &p, &n); +io_register_free_rings (ctx, &n);return ret; } n.rings =io_region_get_ptr (&n.ring_region);
@@ -453,7 +452,7 @@ static int io_register_resize_rings(struct io_ring_ctx *ctx, void __user *arg)WRITE_ONCE (n.rings->cq_ring_entries, p.cq_entries);if (copy_to_user (arg, &p,sizeof (p))) { -io_register_free_rings (ctx, &p, &n); +io_register_free_rings (ctx, &n);return -EFAULT ; }
@@ -462,7 +461,7 @@ static int io_register_resize_rings(struct io_ring_ctx *ctx, void __user *arg)else size =array_size (sizeof (struct io_uring_sqe ), p.sq_entries);if (size ==SIZE_MAX ) { -io_register_free_rings (ctx, &p, &n); +io_register_free_rings (ctx, &n);return -EOVERFLOW ; }
@@ -474,7 +473,7 @@ static int io_register_resize_rings(struct io_ring_ctx *ctx, void __user *arg) } ret =io_create_region (ctx, &n.sq_region, &rd,IORING_OFF_SQES );if (ret) { -io_register_free_rings (ctx, &p, &n); +io_register_free_rings (ctx, &n);return ret; } n.sq_sqes =io_region_get_ptr (&n.sq_region);
@@ -564,7 +563,7 @@ static int io_register_resize_rings(struct io_ring_ctx *ctx, void __user *arg) out:spin_unlock (&ctx->completion_lock);mutex_unlock (&ctx->mmap_lock); -io_register_free_rings (ctx, &p, to_free); +io_register_free_rings (ctx, to_free);if (ctx->sq_data)io_sq_thread_unpark (ctx->sq_data);
Signature · 1 line
2.49.0
PATCH 5/7 io_uring: don't free never created regions
In reply to Pavel Begunkovio_free_region() tolerates empty regions but there is no reason to that either. If the first io_create_region() in io_register_resize_rings() fails, just return the error without attempting to clean it up.
--- io_uring/register.c | 5 ++--- 1 file changed, 2 insertions(+), 3 deletions(-)
io_uring/register.c +2 -3
@@ -432,10 +432,9 @@ static int io_register_resize_rings(struct io_ring_ctx *ctx, void __user *arg) rd.flags |=IORING_MEM_REGION_TYPE_USER ; } ret =io_create_region (ctx, &n.ring_region, &rd,IORING_OFF_CQ_RING ); -if (ret) { -io_register_free_rings (ctx, &n); +if (ret)return ret; - } + n.rings =io_region_get_ptr (&n.ring_region);/*
Signature · 1 line
2.49.0
PATCH 6/7 io_uring/kbuf: use io_create_region for kbuf creation
In reply to Pavel Begunkovkbuf ring is published by io_buffer_add_list(), which correctly protects with mmap_lock, there is no need to use io_create_region_mmap_safe() before as the region is not yet exposed to the userspace via mmap.
--- io_uring/kbuf.c | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-)
io_uring/kbuf.c +1 -1
@@ -630,7 +630,7 @@ int io_register_pbuf_ring(struct io_ring_ctx *ctx, void __user *arg) rd.user_addr = reg.ring_addr; rd.flags |=IORING_MEM_REGION_TYPE_USER ; } - ret =io_create_region_mmap_safe (ctx, &bl->region, &rd, mmap_offset); + ret =io_create_region (ctx, &bl->region, &rd, mmap_offset);if (ret) goto fail; br =io_region_get_ptr (&bl->region);
Signature · 1 line
2.49.0
PATCH 7/7 io_uring: only publish fully handled mem region
In reply to Pavel Begunkovio_register_mem_region() can try to remove a region right after publishing it. This non-atomicity is annoying. Do it in two steps similar to io_register_mem_region(), create memory first and publish it once the rest of the handling is done. Remove now unused io_create_region_mmap_safe(), which was assumed to be a temporary solution from day one.
--- io_uring/memmap.c | 21 --------------------- io_uring/memmap.h | 12 ++++++++++++ io_uring/register.c | 11 ++++++----- 3 files changed, 18 insertions(+), 26 deletions(-)
io_uring/memmap.c +0 -21
@@ -234,27 +234,6 @@ int io_create_region(struct io_ring_ctx *ctx, struct io_mapped_region *mr,return ret; } -int io_create_region_mmap_safe (struct io_ring_ctx *ctx,struct io_mapped_region *mr, -struct io_uring_region_desc *reg, -unsigned long mmap_offset) -{ -struct io_mapped_region tmp_mr; -int ret; - -memcpy (&tmp_mr, mr,sizeof (tmp_mr)); - ret =io_create_region (ctx, &tmp_mr, reg, mmap_offset); -if (ret) -return ret; - -/* -* Once published mmap can find it without holding only the ->mmap_lock -* and not ->uring_lock. -*/ -guard (mutex)(&ctx->mmap_lock); -memcpy (mr, &tmp_mr,sizeof (tmp_mr)); -return 0 ; -} -static struct io_mapped_region *io_mmap_get_region (struct io_ring_ctx *ctx,loff_t pgoff) {
io_uring/memmap.h +12 -0
@@ -36,4 +36,16 @@ static inline bool io_region_is_set(struct io_mapped_region *mr) return !!mr->nr_pages; } +static inline void io_region_publish (struct io_ring_ctx *ctx, +struct io_mapped_region *src_region, +struct io_mapped_region *dst_region) +{ +/* +* Once published mmap can find it without holding only the ->mmap_lock +* and not ->uring_lock. +*/ +guard (mutex)(&ctx->mmap_lock); + *dst_region = *src_region; +} +#endif
io_uring/register.c +6 -5
@@ -576,6 +576,7 @@ static int io_register_mem_region(struct io_ring_ctx *ctx, void __user *uarg)struct io_uring_mem_region_reg reg;struct io_uring_region_desc __user *rd_uptr;struct io_uring_region_desc rd; +struct io_mapped_region region = {};int ret;if (io_region_is_set (&ctx->param_region))
@@ -599,20 +600,20 @@ static int io_register_mem_region(struct io_ring_ctx *ctx, void __user *uarg) !(ctx->flags & IORING_SETUP_R_DISABLED))return -EINVAL ; - ret =io_create_region_mmap_safe (ctx, &ctx->param_region, &rd, -IORING_MAP_OFF_PARAM_REGION ); + ret =io_create_region (ctx, ®ion, &rd,IORING_MAP_OFF_PARAM_REGION );if (ret)return ret;if (copy_to_user (rd_uptr, &rd,sizeof (rd))) { -guard (mutex)(&ctx->mmap_lock); -io_free_region (ctx, &ctx->param_region); +io_free_region (ctx, ®ion);return -EFAULT ; }if (reg.flags &IORING_MEM_REGION_REG_WAIT_ARG ) { - ctx->cq_wait_arg =io_region_get_ptr (&ctx->param_region); + ctx->cq_wait_arg =io_region_get_ptr (®ion); ctx->cq_wait_size = rd.size; } + +io_region_publish (ctx, ®ion, &ctx->param_region);return 0 ; }
Signature · 1 line
2.49.0
Quoted 6 lines · Pavel Begunkov
Random stuff that will my future changes simpler but flushing it out as it looks like a fine cleanup series. It also replaces
io_create_region_mmap_safe()calls with non-mmap version where it's possible, and removes the helper in Patch 7 while handlingio_register_mem_region()a bit more gracefully.
Quoted 8 lines
Pavel Begunkov (7):
io_uring: deduplicatearray_sizeinio_allocate_scq_urings
io_uring: sanity check sizes before attempting allocation
io_uring: use no mmap safe region helpers on resizing
io_uring: remove extra args fromio_register_free_rings
io_uring: don't free never created regionsio_uring/kbuf: useio_create_regionfor kbuf creation
io_uring: only publish fully handled mem region
Quoted 7 lines
io_uring/io_uring.c | 26 ++++++++++++-------------- io_uring/kbuf.c | 2 +- io_uring/memmap.c | 21 --------------------- io_uring/memmap.h | 12 ++++++++++++ io_uring/register.c | 29 ++++++++++++++--------------- 5 files changed, 39 insertions(+), 51 deletions(-)
Signature · 1 line
Gabriel Krisman Bertazi
Quoted 15 lines · Pavel Begunkov
Random stuff that will my future changes simpler but flushing it out as it looks like a fine cleanup series. It also replaces
io_create_region_mmap_safe()calls with non-mmap version where it's possible, and removes the helper in Patch 7 while handlingio_register_mem_region()a bit more gracefully.Pavel Begunkov (7):
io_uring: deduplicatearray_sizeinio_allocate_scq_urings
io_uring: sanity check sizes before attempting allocation
io_uring: use no mmap safe region helpers on resizing
io_uring: remove extra args fromio_register_free_rings
io_uring: don't free never created regionsio_uring/kbuf: useio_create_regionfor kbuf creation
io_uring: only publish fully handled mem region
Quoted 1 line
[...]
Applied, thanks!
[1/7] io_uring: deduplicate array_size in io_allocate_scq_urings
commit: 12aced0a551e18f2162091e388c3a36ea75ccb13
[2/7] io_uring: sanity check sizes before attempting allocation
commit: 284306f6e6045e3f7b932914d1368df90033e87e
[3/7] io_uring: use no mmap safe region helpers on resizing
commit: 4c53e392a194f2bb37403a5b9bcf8e77401234cc
[4/7] io_uring: remove extra args from io_register_free_rings
commit: 0c89dbbcadf126920e6f9ebfa64e2538af84fef3
[5/7] io_uring: don't free never created regions
commit: 6e9752977caa47c200f88d7df1ff114955a03bad
[6/7] io_uring/kbuf: use io_create_region for kbuf creation
commit: dec10a1ad1d5f9d46e7f6e7c8b414a805e00717c
[7/7] io_uring: only publish fully handled mem region
commit: 5b6d8a032e807c48a843fb81d9e3d74391f731eaBest regards,
Signature · 1 line
Jens Axboe