Add support for ring resizing
Jens Axboe
maintainercover Add support for ring resizing
Hi,
Something that's come up several times over the years is how to deal with ring sizing. The SQ ring sizing is usually trivial - it just controls the batch submit size, and usually it's not that difficult to just submit if the app fails to get a free SQE.
For the CQ ring, it's a different story. For networked workloads, it can be hard to appropriately size the CQ ring without knowing exactly how busy a given ring will be. This leads to applications grossly over-sizing the ring, just in case, which is wasteful.
Here's a stab at supporting ring resizing. It supports resizing of both rings, SQ and CQ, as it's really no different than just doing the CQ ring itself. liburing has a 'resize-rings' branch with a bit of support code, and a test case:
https://git.kernel.dk/cgit/liburing/log/?h=resize-rings
and these patches can also be found here:
https://git.kernel.dk/cgit/linux/log/?h=io_uring-ring-resize
include/uapi/linux/io_uring.h | 3 + io_uring/io_uring.c | 84 ++++++++++-------- io_uring/io_uring.h | 6 ++ io_uring/register.c | 161 ++++++++++++++++++++++++++++++++++ 4 files changed, 216 insertions(+), 38 deletions(-)
Signature · 1 line
Jens Axboe
Jens Axboe
maintainerPATCH 1/3 io_uring: move max entry definition and ring sizing into header
In preparation for needing this somewhere else, move the definitions for the maximum CQ and SQ ring size into io_uring.h. Make the rings_size() helper available as well, and have it take just the setup flags argument rather than the fill ring pointer. That's all that is needed.
--- io_uring/io_uring.c | 14 ++++++-------- io_uring/io_uring.h | 5 +++++ 2 files changed, 11 insertions(+), 8 deletions(-)
io_uring/io_uring.c +6 -8
@@ -105,9 +105,6 @@#include "alloc_cache.h" #include "eventfd.h" -#define IORING_MAX_ENTRIES 32768 -#define IORING_MAX_CQ_ENTRIES (2 * IORING_MAX_ENTRIES) -#define SQE_COMMON_FLAGS (IOSQE_FIXED_FILE | IOSQE_IO_LINK | \ IOSQE_IO_HARDLINK | IOSQE_ASYNC)
@@ -2667,8 +2664,8 @@ static void io_rings_free(struct io_ring_ctx *ctx) ctx->sq_sqes =NULL ; } -static unsigned long rings_size (struct io_ring_ctx *ctx,unsigned int sq_entries, -unsigned int cq_entries,size_t *sq_offset) +unsigned long rings_size (unsigned int flags,unsigned int sq_entries, +unsigned int cq_entries,size_t *sq_offset) {struct io_rings *rings;size_t off, sq_array_size;
@@ -2676,7 +2673,7 @@ static unsigned long rings_size(struct io_ring_ctx *ctx, unsigned int sq_entries off =struct_size (rings, cqes, cq_entries);if (off ==SIZE_MAX )return SIZE_MAX ; -if (ctx->flags &IORING_SETUP_CQE32 ) { +if (flags &IORING_SETUP_CQE32 ) {if (check_shl_overflow (off,1 , &off))return SIZE_MAX ; }
@@ -2687,7 +2684,7 @@ static unsigned long rings_size(struct io_ring_ctx *ctx, unsigned int sq_entriesreturn SIZE_MAX ;#endif -if (ctx->flags &IORING_SETUP_NO_SQARRAY ) { +if (flags &IORING_SETUP_NO_SQARRAY ) { *sq_offset =SIZE_MAX ;return off; }
@@ -3434,7 +3431,8 @@ static __cold int io_allocate_scq_urings(struct io_ring_ctx *ctx, ctx->sq_entries = p->sq_entries; ctx->cq_entries = p->cq_entries; - size =rings_size (ctx, p->sq_entries, p->cq_entries, &sq_array_offset); + size =rings_size (ctx->flags, p->sq_entries, p->cq_entries, + &sq_array_offset);if (size ==SIZE_MAX )return -EOVERFLOW ;
io_uring/io_uring.h +5 -0
@@ -65,6 +65,11 @@ static inline bool io_should_wake(struct io_wait_queue *iowq)return dist >=0 ||atomic_read (&ctx->cq_timeouts) != iowq->nr_timeouts; } +#define IORING_MAX_ENTRIES 32768 +#define IORING_MAX_CQ_ENTRIES (2 * IORING_MAX_ENTRIES) + +unsigned long rings_size (unsigned int flags,unsigned int sq_entries, +unsigned int cq_entries,size_t *sq_offset);bool io_cqe_cache_refill (struct io_ring_ctx *ctx,bool overflow);int io_run_task_work_sig (struct io_ring_ctx *ctx);void io_req_defer_failed (struct io_kiocb *req,s32 res);
Signature · 1 line
2.45.2
Jens Axboe
maintainerPATCH 2/3 io_uring: abstract out a bit of the ring filling logic
In reply to Jens AxboeAbstract out a io_uring_fill_params() helper, which fills out the necessary bits of struct io_uring_params. Add it to io_uring.h as well, in preparation for having another internal user of it.
--- io_uring/io_uring.c | 70 ++++++++++++++++++++++++++------------------- io_uring/io_uring.h | 1 + 2 files changed, 41 insertions(+), 30 deletions(-)
io_uring/io_uring.c +40 -30
@@ -3498,14 +3498,8 @@ static struct file *io_uring_get_file(struct io_ring_ctx *ctx) O_RDWR | O_CLOEXEC, NULL); } -static __cold int io_uring_create (unsigned entries,struct io_uring_params *p, -struct io_uring_params __user *params) +int io_uring_fill_params (unsigned entries,struct io_uring_params *p) { -struct io_ring_ctx *ctx; -struct io_uring_task *tctx; -struct file *file; -int ret; -if (!entries)return -EINVAL ;if (entries >IORING_MAX_ENTRIES ) {
@@ -3547,6 +3541,42 @@ static __cold int io_uring_create(unsigned entries, struct io_uring_params *p, p->cq_entries =2 * p->sq_entries; } + p->sq_off.head = offsetof(struct io_rings, sq.head); + p->sq_off.tail = offsetof(struct io_rings, sq.tail); + p->sq_off.ring_mask = offsetof(struct io_rings , sq_ring_mask); + p->sq_off.ring_entries = offsetof(struct io_rings , sq_ring_entries); + p->sq_off.flags = offsetof(struct io_rings , sq_flags); + p->sq_off.dropped = offsetof(struct io_rings , sq_dropped); + p->sq_off.resv1 =0 ; +if (!(p->flags &IORING_SETUP_NO_MMAP )) + p->sq_off.user_addr =0 ; + + p->cq_off.head = offsetof(struct io_rings, cq.head); + p->cq_off.tail = offsetof(struct io_rings, cq.tail); + p->cq_off.ring_mask = offsetof(struct io_rings , cq_ring_mask); + p->cq_off.ring_entries = offsetof(struct io_rings , cq_ring_entries); + p->cq_off.overflow = offsetof(struct io_rings , cq_overflow); + p->cq_off.cqes = offsetof(struct io_rings , cqes); + p->cq_off.flags = offsetof(struct io_rings , cq_flags); + p->cq_off.resv1 =0 ; +if (!(p->flags &IORING_SETUP_NO_MMAP )) + p->cq_off.user_addr =0 ; + +return 0 ; +} + +static __cold int io_uring_create (unsigned entries,struct io_uring_params *p, +struct io_uring_params __user *params) +{ +struct io_ring_ctx *ctx; +struct io_uring_task *tctx; +struct file *file; +int ret; + + ret =io_uring_fill_params (entries, p); +if (unlikely (ret)) +return ret; + ctx =io_ring_ctx_alloc (p);if (!ctx)return -ENOMEM ;
@@ -3630,6 +3660,9 @@ static __cold int io_uring_create(unsigned entries, struct io_uring_params *p,if (ret) goto err; +if (!(p->flags &IORING_SETUP_NO_SQARRAY )) + p->sq_off.array = (char *)ctx->sq_array - (char *)ctx->rings; + ret =io_sq_offload_create (ctx, p);if (ret) goto err;
@@ -3638,29 +3671,6 @@ static __cold int io_uring_create(unsigned entries, struct io_uring_params *p,if (ret) goto err; - p->sq_off.head = offsetof(struct io_rings, sq.head); - p->sq_off.tail = offsetof(struct io_rings, sq.tail); - p->sq_off.ring_mask = offsetof(struct io_rings , sq_ring_mask); - p->sq_off.ring_entries = offsetof(struct io_rings , sq_ring_entries); - p->sq_off.flags = offsetof(struct io_rings , sq_flags); - p->sq_off.dropped = offsetof(struct io_rings , sq_dropped); -if (!(ctx->flags &IORING_SETUP_NO_SQARRAY )) - p->sq_off.array = (char *)ctx->sq_array - (char *)ctx->rings; - p->sq_off.resv1 =0 ; -if (!(ctx->flags &IORING_SETUP_NO_MMAP )) - p->sq_off.user_addr =0 ; - - p->cq_off.head = offsetof(struct io_rings, cq.head); - p->cq_off.tail = offsetof(struct io_rings, cq.tail); - p->cq_off.ring_mask = offsetof(struct io_rings , cq_ring_mask); - p->cq_off.ring_entries = offsetof(struct io_rings , cq_ring_entries); - p->cq_off.overflow = offsetof(struct io_rings , cq_overflow); - p->cq_off.cqes = offsetof(struct io_rings , cqes); - p->cq_off.flags = offsetof(struct io_rings , cq_flags); - p->cq_off.resv1 =0 ; -if (!(ctx->flags &IORING_SETUP_NO_MMAP )) - p->cq_off.user_addr =0 ; - p->features =IORING_FEAT_SINGLE_MMAP |IORING_FEAT_NODROP |IORING_FEAT_SUBMIT_STABLE |IORING_FEAT_RW_CUR_POS | IORING_FEAT_CUR_PERSONALITY | IORING_FEAT_FAST_POLL |
io_uring/io_uring.h +1 -0
@@ -70,6 +70,7 @@ static inline bool io_should_wake(struct io_wait_queue *iowq)unsigned long rings_size (unsigned int flags,unsigned int sq_entries,unsigned int cq_entries,size_t *sq_offset); +int io_uring_fill_params (unsigned entries,struct io_uring_params *p);bool io_cqe_cache_refill (struct io_ring_ctx *ctx,bool overflow);int io_run_task_work_sig (struct io_ring_ctx *ctx);void io_req_defer_failed (struct io_kiocb *req,s32 res);
Signature · 1 line
2.45.2
Jens Axboe
maintainerPATCH 3/3 io_uring/register: add IORING_REGISTER_RESIZE_RINGS
In reply to Jens AxboeOnce a ring has been created, the size of the CQ and SQ rings are fixed. Usually this isn't a problem on the SQ ring side, as it merely controls the available number of requests that can be submitted in a single system call, and there's rarely a need to change that.
For the CQ ring, it's a different story. For most efficient use of io_uring, it's important that the CQ ring never overflows. This means that applications must size it for the worst case scenario, which can be wasteful.
Add IORING_REGISTER_RESIZE_RINGS, which allows an application to resize the existing rings. It takes a struct io_uring_params argument, the same one which is used to setup the ring initially, and resizes rings according to the sizes given.
Certain properties are always inherited from the original ring setup, like SQE128/CQE32 and other setup options. The implementation only allows flag associated with how the CQ ring is sized and clamped.
Existing unconsumed SQE and CQE entries are copied as part of the process. Any register op holds ->uring_lock, which prevents new submissions, and the internal mapping holds the completion lock as well across moving CQ ring state.
--- include/uapi/linux/io_uring.h | 3 + io_uring/register.c | 161 ++++++++++++++++++++++++++++++++++ 2 files changed, 164 insertions(+)
include/uapi/linux/io_uring.h +3 -0
@@ -615,6 +615,9 @@ enum io_uring_register_op {/* send MSG_RING without having a ring */ IORING_REGISTER_SEND_MSG_RING =31 , +/* resize CQ ring */ +IORING_REGISTER_RESIZE_RINGS =33 , +/* this goes last */ IORING_REGISTER_LAST,
io_uring/register.c +161 -0
179 lines not on this page · raw message
Signature · 1 line
2.45.2
Quoted 10 lines · Jens Axboe
Here's a stab at supporting ring resizing. It supports resizing of both rings, SQ and CQ, as it's really no different than just doing the CQ ring itself. liburing has a 'resize-rings' branch with a bit of support code, and a test case:
https://git.kernel.dk/cgit/liburing/log/?h=resize-rings
and these patches can also be found here:
https://git.kernel.dk/cgit/linux/log/?h=io_uring-ring-resize
You'd need to properly synchronize that path with io_uring_mmap(), right? Take a lock that prevents concurrent mmap() from accessing ctx->ring_pages while the resize is concurrently freeing that array, so that you don't get UAF?
And I guess ideally you'd also zap the already-mapped pages from corresponding VMAs with something like unmap_mapping_range(), though that won't make a difference security-wise since the pages are refcounted by the userspace mapping anyway.
Jens Axboe
maintainerQuoted 8 lines · Jann Horn
Quoted 7 lines · Jens Axboe
Here's a stab at supporting ring resizing. It supports resizing of both rings, SQ and CQ, as it's really no different than just doing the CQ ring itself. liburing has a 'resize-rings' branch with a bit of support code, and a test case:
Quoted 8 lines
Quoted 3 lines
and these patches can also be found here:
https://git.kernel.dk/cgit/linux/log/?h=io_uring-ring-resize
You'd need to properly synchronize that path with
io_uring_mmap(), right? Take a lock that prevents concurrentmmap()from accessingctx->ring_pageswhile the resize is concurrently freeing that array, so that you don't get UAF?
Yep indeed! It's missing the mmap_lock, I'll add that.
Quoted 4 lines
And I guess ideally you'd also zap the already-mapped pages from corresponding VMAs with something like
unmap_mapping_range(), though that won't make a difference security-wise since the pages are refcounted by the userspace mapping anyway.
Yes don't think we need to do anything there, just have userspace unmap the old range upon return.
Signature · 1 line
Jens Axboe