Add support for ring resizing
Jens Axboe
maintainerv3 cover Add support for ring resizing
Hi,
Here's v3 of the ring resizing support. For the v1 posting and details, look here:
https://lore.kernel.org/io-uring/[email protected]/T/#md3a2f049b0527592cc6d8ea25b46bde9fa8e5c68
include/uapi/linux/io_uring.h | 3 + io_uring/io_uring.c | 84 +++++++------ io_uring/io_uring.h | 6 + io_uring/memmap.c | 4 + io_uring/register.c | 214 ++++++++++++++++++++++++++++++++++ 5 files changed, 273 insertions(+), 38 deletions(-)
Since v2:
- Add patch explicitly checking for valid rings at mmap time. More of a documentation patch than anything, doesn't fix any current or future issues. But it makes it explicit that they have to be valid.
- Fix an issue with resizing when the sizes were identical, causing a cleared io_uring_params to be returned and liburing using that to re-mmap the rings. Ensure io_uring_params is filled in before getting copied back, even if nothing was done.
- Fix an issue with SQPOLL, it needs to get quiesced before we can proceed with the resize.
- Hold the mmap_sem and completion lock across the final part of the operation, where state is copied and the rings swapped. This prevents concurrent IO from accessing rings that are going away, and mmap from seeing any NULL or going-away mappings until the swap has been completed.
- Add some more comments.
Signature · 1 line
Jens Axboe
Jens Axboe
maintainerPATCH v3 1/4 io_uring: move max entry definition and ring sizing into header
In preparation for needing this somewhere else, move the definitions for the maximum CQ and SQ ring size into io_uring.h. Make the rings_size() helper available as well, and have it take just the setup flags argument rather than the fill ring pointer. That's all that is needed.
--- io_uring/io_uring.c | 14 ++++++-------- io_uring/io_uring.h | 5 +++++ 2 files changed, 11 insertions(+), 8 deletions(-)
io_uring/io_uring.c +6 -8
@@ -105,9 +105,6 @@#include "alloc_cache.h" #include "eventfd.h" -#define IORING_MAX_ENTRIES 32768 -#define IORING_MAX_CQ_ENTRIES (2 * IORING_MAX_ENTRIES) -#define SQE_COMMON_FLAGS (IOSQE_FIXED_FILE | IOSQE_IO_LINK | \ IOSQE_IO_HARDLINK | IOSQE_ASYNC)
@@ -2667,8 +2664,8 @@ static void io_rings_free(struct io_ring_ctx *ctx) ctx->sq_sqes =NULL ; } -static unsigned long rings_size (struct io_ring_ctx *ctx,unsigned int sq_entries, -unsigned int cq_entries,size_t *sq_offset) +unsigned long rings_size (unsigned int flags,unsigned int sq_entries, +unsigned int cq_entries,size_t *sq_offset) {struct io_rings *rings;size_t off, sq_array_size;
@@ -2676,7 +2673,7 @@ static unsigned long rings_size(struct io_ring_ctx *ctx, unsigned int sq_entries off =struct_size (rings, cqes, cq_entries);if (off ==SIZE_MAX )return SIZE_MAX ; -if (ctx->flags &IORING_SETUP_CQE32 ) { +if (flags &IORING_SETUP_CQE32 ) {if (check_shl_overflow (off,1 , &off))return SIZE_MAX ; }
@@ -2687,7 +2684,7 @@ static unsigned long rings_size(struct io_ring_ctx *ctx, unsigned int sq_entriesreturn SIZE_MAX ;#endif -if (ctx->flags &IORING_SETUP_NO_SQARRAY ) { +if (flags &IORING_SETUP_NO_SQARRAY ) { *sq_offset =SIZE_MAX ;return off; }
@@ -3434,7 +3431,8 @@ static __cold int io_allocate_scq_urings(struct io_ring_ctx *ctx, ctx->sq_entries = p->sq_entries; ctx->cq_entries = p->cq_entries; - size =rings_size (ctx, p->sq_entries, p->cq_entries, &sq_array_offset); + size =rings_size (ctx->flags, p->sq_entries, p->cq_entries, + &sq_array_offset);if (size ==SIZE_MAX )return -EOVERFLOW ;
io_uring/io_uring.h +5 -0
@@ -65,6 +65,11 @@ static inline bool io_should_wake(struct io_wait_queue *iowq)return dist >=0 ||atomic_read (&ctx->cq_timeouts) != iowq->nr_timeouts; } +#define IORING_MAX_ENTRIES 32768 +#define IORING_MAX_CQ_ENTRIES (2 * IORING_MAX_ENTRIES) + +unsigned long rings_size (unsigned int flags,unsigned int sq_entries, +unsigned int cq_entries,size_t *sq_offset);bool io_cqe_cache_refill (struct io_ring_ctx *ctx,bool overflow);int io_run_task_work_sig (struct io_ring_ctx *ctx);void io_req_defer_failed (struct io_kiocb *req,s32 res);
Signature · 1 line
2.45.2
Jens Axboe
maintainerPATCH v3 2/4 io_uring: abstract out a bit of the ring filling logic
In reply to Jens AxboeAbstract out a io_uring_fill_params() helper, which fills out the necessary bits of struct io_uring_params. Add it to io_uring.h as well, in preparation for having another internal user of it.
--- io_uring/io_uring.c | 70 ++++++++++++++++++++++++++------------------- io_uring/io_uring.h | 1 + 2 files changed, 41 insertions(+), 30 deletions(-)
io_uring/io_uring.c +40 -30
@@ -3498,14 +3498,8 @@ static struct file *io_uring_get_file(struct io_ring_ctx *ctx) O_RDWR | O_CLOEXEC, NULL); } -static __cold int io_uring_create (unsigned entries,struct io_uring_params *p, -struct io_uring_params __user *params) +int io_uring_fill_params (unsigned entries,struct io_uring_params *p) { -struct io_ring_ctx *ctx; -struct io_uring_task *tctx; -struct file *file; -int ret; -if (!entries)return -EINVAL ;if (entries >IORING_MAX_ENTRIES ) {
@@ -3547,6 +3541,42 @@ static __cold int io_uring_create(unsigned entries, struct io_uring_params *p, p->cq_entries =2 * p->sq_entries; } + p->sq_off.head = offsetof(struct io_rings, sq.head); + p->sq_off.tail = offsetof(struct io_rings, sq.tail); + p->sq_off.ring_mask = offsetof(struct io_rings , sq_ring_mask); + p->sq_off.ring_entries = offsetof(struct io_rings , sq_ring_entries); + p->sq_off.flags = offsetof(struct io_rings , sq_flags); + p->sq_off.dropped = offsetof(struct io_rings , sq_dropped); + p->sq_off.resv1 =0 ; +if (!(p->flags &IORING_SETUP_NO_MMAP )) + p->sq_off.user_addr =0 ; + + p->cq_off.head = offsetof(struct io_rings, cq.head); + p->cq_off.tail = offsetof(struct io_rings, cq.tail); + p->cq_off.ring_mask = offsetof(struct io_rings , cq_ring_mask); + p->cq_off.ring_entries = offsetof(struct io_rings , cq_ring_entries); + p->cq_off.overflow = offsetof(struct io_rings , cq_overflow); + p->cq_off.cqes = offsetof(struct io_rings , cqes); + p->cq_off.flags = offsetof(struct io_rings , cq_flags); + p->cq_off.resv1 =0 ; +if (!(p->flags &IORING_SETUP_NO_MMAP )) + p->cq_off.user_addr =0 ; + +return 0 ; +} + +static __cold int io_uring_create (unsigned entries,struct io_uring_params *p, +struct io_uring_params __user *params) +{ +struct io_ring_ctx *ctx; +struct io_uring_task *tctx; +struct file *file; +int ret; + + ret =io_uring_fill_params (entries, p); +if (unlikely (ret)) +return ret; + ctx =io_ring_ctx_alloc (p);if (!ctx)return -ENOMEM ;
@@ -3630,6 +3660,9 @@ static __cold int io_uring_create(unsigned entries, struct io_uring_params *p,if (ret) goto err; +if (!(p->flags &IORING_SETUP_NO_SQARRAY )) + p->sq_off.array = (char *)ctx->sq_array - (char *)ctx->rings; + ret =io_sq_offload_create (ctx, p);if (ret) goto err;
@@ -3638,29 +3671,6 @@ static __cold int io_uring_create(unsigned entries, struct io_uring_params *p,if (ret) goto err; - p->sq_off.head = offsetof(struct io_rings, sq.head); - p->sq_off.tail = offsetof(struct io_rings, sq.tail); - p->sq_off.ring_mask = offsetof(struct io_rings , sq_ring_mask); - p->sq_off.ring_entries = offsetof(struct io_rings , sq_ring_entries); - p->sq_off.flags = offsetof(struct io_rings , sq_flags); - p->sq_off.dropped = offsetof(struct io_rings , sq_dropped); -if (!(ctx->flags &IORING_SETUP_NO_SQARRAY )) - p->sq_off.array = (char *)ctx->sq_array - (char *)ctx->rings; - p->sq_off.resv1 =0 ; -if (!(ctx->flags &IORING_SETUP_NO_MMAP )) - p->sq_off.user_addr =0 ; - - p->cq_off.head = offsetof(struct io_rings, cq.head); - p->cq_off.tail = offsetof(struct io_rings, cq.tail); - p->cq_off.ring_mask = offsetof(struct io_rings , cq_ring_mask); - p->cq_off.ring_entries = offsetof(struct io_rings , cq_ring_entries); - p->cq_off.overflow = offsetof(struct io_rings , cq_overflow); - p->cq_off.cqes = offsetof(struct io_rings , cqes); - p->cq_off.flags = offsetof(struct io_rings , cq_flags); - p->cq_off.resv1 =0 ; -if (!(ctx->flags &IORING_SETUP_NO_MMAP )) - p->cq_off.user_addr =0 ; - p->features =IORING_FEAT_SINGLE_MMAP |IORING_FEAT_NODROP |IORING_FEAT_SUBMIT_STABLE |IORING_FEAT_RW_CUR_POS | IORING_FEAT_CUR_PERSONALITY | IORING_FEAT_FAST_POLL |
io_uring/io_uring.h +1 -0
@@ -70,6 +70,7 @@ static inline bool io_should_wake(struct io_wait_queue *iowq)unsigned long rings_size (unsigned int flags,unsigned int sq_entries,unsigned int cq_entries,size_t *sq_offset); +int io_uring_fill_params (unsigned entries,struct io_uring_params *p);bool io_cqe_cache_refill (struct io_ring_ctx *ctx,bool overflow);int io_run_task_work_sig (struct io_ring_ctx *ctx);void io_req_defer_failed (struct io_kiocb *req,s32 res);
Signature · 1 line
2.45.2
Jens Axboe
maintainerPATCH v3 3/4 io_uring/memmap: explicitly return -EFAULT for mmap on NULL rings
In reply to Jens AxboeThe later mapping will actually check this too, but in terms of code clarify, explicitly check for whether or not the rings and sqes are valid during validation. That makes it explicit that if they are non-NULL, they are valid and can get mapped.
--- io_uring/memmap.c | 4 ++++ 1 file changed, 4 insertions(+)
io_uring/memmap.c +4 -0
@@ -204,11 +204,15 @@ static void *io_uring_validate_mmap_request(struct file *file, loff_t pgoff,/* Don't allow mmap if the ring was setup without it */ if (ctx->flags &IORING_SETUP_NO_MMAP )return ERR_PTR (-EINVAL ); +if (!ctx->rings) +return ERR_PTR (-EFAULT );return ctx->rings;case IORING_OFF_SQES :/* Don't allow mmap if the ring was setup without it */ if (ctx->flags &IORING_SETUP_NO_MMAP )return ERR_PTR (-EINVAL ); +if (!ctx->sq_sqes) +return ERR_PTR (-EFAULT );return ctx->sq_sqes;case IORING_OFF_PBUF_RING : {struct io_buffer_list *bl;
Signature · 1 line
2.45.2
Jens Axboe
maintainerPATCH v3 4/4 io_uring/register: add IORING_REGISTER_RESIZE_RINGS
In reply to Jens AxboeOnce a ring has been created, the size of the CQ and SQ rings are fixed. Usually this isn't a problem on the SQ ring side, as it merely controls the available number of requests that can be submitted in a single system call, and there's rarely a need to change that.
For the CQ ring, it's a different story. For most efficient use of io_uring, it's important that the CQ ring never overflows. This means that applications must size it for the worst case scenario, which can be wasteful.
Add IORING_REGISTER_RESIZE_RINGS, which allows an application to resize the existing rings. It takes a struct io_uring_params argument, the same one which is used to setup the ring initially, and resizes rings according to the sizes given.
Certain properties are always inherited from the original ring setup, like SQE128/CQE32 and other setup options. The implementation only allows flag associated with how the CQ ring is sized and clamped.
Existing unconsumed SQE and CQE entries are copied as part of the process. If either the SQ or CQ resized destination ring cannot hold the entries already present in the source rings, then the operation is failed with -EOVERFLOW. Any register op holds ->uring_lock, which prevents new submissions, and the internal mapping holds the completion lock as well across moving CQ ring state.
--- include/uapi/linux/io_uring.h | 3 + io_uring/register.c | 214 ++++++++++++++++++++++++++++++++++ 2 files changed, 217 insertions(+)
include/uapi/linux/io_uring.h +3 -0
@@ -615,6 +615,9 @@ enum io_uring_register_op {/* send MSG_RING without having a ring */ IORING_REGISTER_SEND_MSG_RING =31 , +/* resize CQ ring */ +IORING_REGISTER_RESIZE_RINGS =33 , +/* this goes last */ IORING_REGISTER_LAST,
io_uring/register.c +214 -0
232 lines not on this page · raw message
Signature · 1 line
2.45.2
Re: [PATCH 4/4] io_uring/register: add IORING_REGISTER_RESIZE_RINGS
Quoted 4 lines · Jens Axboe
Add
IORING_REGISTER_RESIZE_RINGS, which allows an application to resize the existing rings. It takes a structio_uring_paramsargument, the same one which is used to setup the ring initially, and resizes rings according to the sizes given.
[...]
io_uring/register.c static int io_register_clock(struct io_ring_ctx *ctx, · line 496
Quoted 8 lines
io_uring/register.c +8
-0+* We'll do the swap. Clear out existing mappings to prevent mmap +* from seeing them, as we'll unmap them. Any attempt to mmap existing +* rings beyond this point will fail. Not that it could proceed at this +* point anyway, as we'll hold the mmap_sem until we've done the swap. +* Likewise, hold the completion * lock over the duration of the actual +* swap. +*/ +mmap_write_lock (current->mm);
Why does the mmap lock for current->mm suffice here? I see nothing in io_uring_mmap() that limits mmap() to tasks with the same mm_struct.
Jens Axboe
maintainerRe: [PATCH 4/4] io_uring/register: add IORING_REGISTER_RESIZE_RINGS
Quoted 15 lines · Jann Horn
Quoted 4 lines · On Thu, Oct 24, 2024 at 7:08?PM Jens Axboe <[email protected]> wrote:
Add
IORING_REGISTER_RESIZE_RINGS, which allows an application to resize the existing rings. It takes a structio_uring_paramsargument, the same one which is used to setup the ring initially, and resizes rings according to the sizes given.[...]
io_uring/register.c static int io_register_clock(struct io_ring_ctx *ctx, · line 496
Quoted 8 lines
io_uring/register.c +8
-0+* We'll do the swap. Clear out existing mappings to prevent mmap +* from seeing them, as we'll unmap them. Any attempt to mmap existing +* rings beyond this point will fail. Not that it could proceed at this +* point anyway, as we'll hold the mmap_sem until we've done the swap. +* Likewise, hold the completion * lock over the duration of the actual +* swap. +*/ +mmap_write_lock (current->mm);
Quoted 2 lines
Why does the mmap lock for
current->mmsuffice here? I see nothing inio_uring_mmap()that limitsmmap()to tasks with the samemm_struct.
Ehm does ->mmap() not hold ->mmap_sem already? I was under that understanding. Obviously if it doesn't, then yeah this won't be enough. Checked, and it does.
Ah I see what you mean now, task with different mm. But how would that come about? The io_uring fd is CLOEXEC, and it can't get passed.
Signature · 1 line
Jens Axboe
Re: [PATCH 4/4] io_uring/register: add IORING_REGISTER_RESIZE_RINGS
Quoted 19 lines · Jens Axboe
Quoted 15 lines · Jann Horn
Quoted 4 lines · On Thu, Oct 24, 2024 at 7:08?PM Jens Axboe <[email protected]> wrote:
Add
IORING_REGISTER_RESIZE_RINGS, which allows an application to resize the existing rings. It takes a structio_uring_paramsargument, the same one which is used to setup the ring initially, and resizes rings according to the sizes given.[...]
io_uring/register.c static int io_register_clock(struct io_ring_ctx *ctx, · line 496
Quoted 8 lines
io_uring/register.c +8
-0+* We'll do the swap. Clear out existing mappings to prevent mmap +* from seeing them, as we'll unmap them. Any attempt to mmap existing +* rings beyond this point will fail. Not that it could proceed at this +* point anyway, as we'll hold the mmap_sem until we've done the swap. +* Likewise, hold the completion * lock over the duration of the actual +* swap. +*/ +mmap_write_lock (current->mm);Quoted 2 lines
Why does the mmap lock for
current->mmsuffice here? I see nothing inio_uring_mmap()that limitsmmap()to tasks with the samemm_struct.
Quoted 6 lines
Ehm does ->mmap() not hold ->mmap_sem already? I was under that understanding. Obviously if it doesn't, then yeah this won't be enough. Checked, and it does.
Ah I see what you mean now, task with different mm. But how would that come about? The
io_uringfd is CLOEXEC, and it can't get passed.
Yeah, that's what I meant, tasks with different mm. I think there are a few ways to get the io_uring fd into a different task, the ones I can immediately think of:
- O_CLOEXEC only applies on execve(), fork() should still inherit the fd
- O_CLOEXEC can be cleared via fcntl()
- you can use clone() to create two tasks that share FD tables
without sharing an mm
Jens Axboe
maintainerRe: [PATCH 4/4] io_uring/register: add IORING_REGISTER_RESIZE_RINGS
Quoted 27 lines · Jann Horn
Quoted 19 lines · On Thu, Oct 24, 2024 at 9:50?PM Jens Axboe <[email protected]> wrote:
Quoted 15 lines · Jann Horn
Quoted 4 lines · On Thu, Oct 24, 2024 at 7:08?PM Jens Axboe <[email protected]> wrote:
Add
IORING_REGISTER_RESIZE_RINGS, which allows an application to resize the existing rings. It takes a structio_uring_paramsargument, the same one which is used to setup the ring initially, and resizes rings according to the sizes given.[...]
io_uring/register.c static int io_register_clock(struct io_ring_ctx *ctx, · line 496
Quoted 8 lines
io_uring/register.c +8
-0+* We'll do the swap. Clear out existing mappings to prevent mmap +* from seeing them, as we'll unmap them. Any attempt to mmap existing +* rings beyond this point will fail. Not that it could proceed at this +* point anyway, as we'll hold the mmap_sem until we've done the swap. +* Likewise, hold the completion * lock over the duration of the actual +* swap. +*/ +mmap_write_lock (current->mm);Quoted 2 lines
Why does the mmap lock for
current->mmsuffice here? I see nothing inio_uring_mmap()that limitsmmap()to tasks with the samemm_struct.Quoted 6 lines
Ehm does ->mmap() not hold ->mmap_sem already? I was under that understanding. Obviously if it doesn't, then yeah this won't be enough. Checked, and it does.
Ah I see what you mean now, task with different mm. But how would that come about? The
io_uringfd is CLOEXEC, and it can't get passed.
Quoted 8 lines
Yeah, that's what I meant, tasks with different mm. I think there are a few ways to get the
io_uringfd into a different task, the ones I can immediately think of:-
O_CLOEXEConly applies onexecve(),fork()should still inherit the fd
-O_CLOEXECcan be cleared viafcntl()
- you can useclone()to create two tasks that share FD tables
without sharing an mm
OK good catch, yes then it won't be enough. Might just make sense to exclude mmap separately, then. Thanks, I'll work on that for v4! Probably trivial enough with just a separate mutex that's held over mmap, and that register grabs too. Basically replacing mmap_write_lock() on the resize side with ctx->mmap_lock, and grabbing that over validate/mmap on the io_uring mmap side. Need to ponder the implications of that in terms of how the locks nest, as you'd have mmap_sem -> ctx->mmap_lock, but the other way around for resizing. But that should just be for the pinning parts, which is why it does the mmap_sem grabbing where it does right now.
Signature · 1 line
Jens Axboe
Re: [PATCH 4/4] io_uring/register: add IORING_REGISTER_RESIZE_RINGS
Quoted 32 lines · Jens Axboe
Quoted 31 lines · Jann Horn
Quoted 19 lines · On Thu, Oct 24, 2024 at 9:50?PM Jens Axboe <[email protected]> wrote:
Quoted 15 lines · Jann Horn
Quoted 4 lines · On Thu, Oct 24, 2024 at 7:08?PM Jens Axboe <[email protected]> wrote:
Add
IORING_REGISTER_RESIZE_RINGS, which allows an application to resize the existing rings. It takes a structio_uring_paramsargument, the same one which is used to setup the ring initially, and resizes rings according to the sizes given.[...]
io_uring/register.c static int io_register_clock(struct io_ring_ctx *ctx, · line 496
Quoted 8 lines
io_uring/register.c +8
-0+* We'll do the swap. Clear out existing mappings to prevent mmap +* from seeing them, as we'll unmap them. Any attempt to mmap existing +* rings beyond this point will fail. Not that it could proceed at this +* point anyway, as we'll hold the mmap_sem until we've done the swap. +* Likewise, hold the completion * lock over the duration of the actual +* swap. +*/ +mmap_write_lock (current->mm);Quoted 2 lines
Why does the mmap lock for
current->mmsuffice here? I see nothing inio_uring_mmap()that limitsmmap()to tasks with the samemm_struct.Quoted 6 lines
Ehm does ->mmap() not hold ->mmap_sem already? I was under that understanding. Obviously if it doesn't, then yeah this won't be enough. Checked, and it does.
Ah I see what you mean now, task with different mm. But how would that come about? The
io_uringfd is CLOEXEC, and it can't get passed.Yeah, that's what I meant, tasks with different mm. I think there are a few ways to get the
io_uringfd into a different task, the ones I can immediately think of:
Quoted 7 lines
Quoted 4 lines
-
O_CLOEXEConly applies onexecve(),fork()should still inherit the fd
-O_CLOEXECcan be cleared viafcntl()
- you can useclone()to create two tasks that share FD tables
without sharing an mmOK good catch, yes then it won't be enough. Might just make sense to exclude mmap separately, then. Thanks, I'll work on that for v4!
Yeah, that sounds reasonable to me.
Jens Axboe
maintainerRe: [PATCH 4/4] io_uring/register: add IORING_REGISTER_RESIZE_RINGS
Quoted 38 lines · Jann Horn
Quoted 37 lines · On Thu, Oct 24, 2024 at 9:59?PM Jens Axboe <[email protected]> wrote:
Quoted 35 lines · Jann Horn
Quoted 19 lines · On Thu, Oct 24, 2024 at 9:50?PM Jens Axboe <[email protected]> wrote:
Quoted 15 lines · Jann Horn
Quoted 4 lines · On Thu, Oct 24, 2024 at 7:08?PM Jens Axboe <[email protected]> wrote:
Add
IORING_REGISTER_RESIZE_RINGS, which allows an application to resize the existing rings. It takes a structio_uring_paramsargument, the same one which is used to setup the ring initially, and resizes rings according to the sizes given.[...]
Review
Quoted 8 lines
(quoted diff) +8
-0+ * We'll do the swap. Clear out existing mappings to prevent mmap + * from seeing them, as we'll unmap them. Any attempt to mmap existing + * rings beyond this point will fail. Not that it could proceed at this + * point anyway, as we'll hold the mmap_sem until we've done the swap. + * Likewise, hold the completion * lock over the duration of the actual + * swap. + */ + mmap_write_lock(current->mm);Quoted 2 lines
Why does the mmap lock for
current->mmsuffice here? I see nothing inio_uring_mmap()that limitsmmap()to tasks with the samemm_struct.Quoted 6 lines
Ehm does ->mmap() not hold ->mmap_sem already? I was under that understanding. Obviously if it doesn't, then yeah this won't be enough. Checked, and it does.
Ah I see what you mean now, task with different mm. But how would that come about? The
io_uringfd is CLOEXEC, and it can't get passed.Yeah, that's what I meant, tasks with different mm. I think there are a few ways to get the
io_uringfd into a different task, the ones I can immediately think of:-
O_CLOEXEConly applies onexecve(),fork()should still inherit the fd
-O_CLOEXECcan be cleared viafcntl()
- you can useclone()to create two tasks that share FD tables
without sharing an mm
Quoted 4 lines
Quoted 2 lines
OK good catch, yes then it won't be enough. Might just make sense to exclude mmap separately, then. Thanks, I'll work on that for v4!
Yeah, that sounds reasonable to me.
Something like this should do it, it's really just replacing mmap_sem with a ring private lock. And since the ordering already had to deal with uring_lock vs mmap_sem ABBA issues, this should slot straight in as well.
Totally untested, will write a fork() test case as well for the resize-rings test cases.
include/linux/io_uring_types.h +7 -0
@@ -423,6 +423,13 @@ struct io_ring_ctx {/* protected by ->completion_lock */ unsigned evfd_last_cq_tail; +/* +* Protection for resize vs mmap races - both the mmap and resize +* side will need to grab this lock, to prevent either side from +* being run concurrently with the other. +*/ +struct mutex resize_lock; +/* * If IORING_SETUP_NO_MMAP is used, then the below holds * the gup'ed pages for the two rings, and the sqes.
io_uring/io_uring.c +1 -0
@@ -353,6 +353,7 @@ static __cold struct io_ring_ctx *io_ring_ctx_alloc(struct io_uring_params *p)INIT_WQ_LIST (&ctx->submit_state.compl_reqs);INIT_HLIST_HEAD (&ctx->cancelable_uring_cmd);io_napi_init (ctx); +mutex_init (&ctx->resize_lock);return ctx;
io_uring/memmap.c +8 -0
@@ -251,6 +251,8 @@ __cold int io_uring_mmap(struct file *file, struct vm_area_struct *vma)unsigned int npages;void *ptr; +guard (mutex)(&ctx->resize_lock); + ptr =io_uring_validate_mmap_request (file, vma->vm_pgoff, sz);if (IS_ERR (ptr))return PTR_ERR (ptr);
@@ -274,6 +276,7 @@ unsigned long io_uring_get_unmapped_area(struct file *filp, unsigned long addr, unsigned long len, unsigned long pgoff, unsigned long flags) { + struct io_ring_ctx *ctx = filp->private_data;void *ptr;/*
@@ -284,6 +287,8 @@ unsigned long io_uring_get_unmapped_area(struct file *filp, unsigned long addr,if (addr)return -EINVAL ; +guard (mutex)(&ctx->resize_lock); + ptr =io_uring_validate_mmap_request (filp, pgoff, len);if (IS_ERR (ptr))return -ENOMEM ;
@@ -329,8 +334,11 @@ unsigned long io_uring_get_unmapped_area(struct file *file, unsigned long addr, unsigned long len, unsigned long pgoff, unsigned long flags) { + struct io_uring_ctx *ctx = file->private_data;void *ptr; +guard (mutex)(&ctx->resize_lock); + ptr =io_uring_validate_mmap_request (file, pgoff, len);if (IS_ERR (ptr))return PTR_ERR (ptr);
io_uring/register.c +9 -8
@@ -486,14 +486,15 @@ static int io_register_resize_rings(struct io_ring_ctx *ctx, void __user *arg) }/* -* We'll do the swap. Clear out existing mappings to prevent mmap -* from seeing them, as we'll unmap them. Any attempt to mmap existing -* rings beyond this point will fail. Not that it could proceed at this -* point anyway, as we'll hold the mmap_sem until we've done the swap. -* Likewise, hold the completion * lock over the duration of the actual -* swap. +* We'll do the swap. Grab the ctx->resize_lock, which will exclude +* any new mmap's on the ring fd. Clear out existing mappings to prevent +* mmap from seeing them, as we'll unmap them. Any attempt to mmap +* existing rings beyond this point will fail. Not that it could proceed +* at this point anyway, as the io_uring mmap side needs go grab the +* ctx->resize_lock as well. Likewise, hold the completion lock over the +* duration of the actual swap. */ -mmap_write_lock (current->mm); +mutex_lock (&ctx->resize_lock);spin_lock (&ctx->completion_lock); o.rings = ctx->rings; ctx->rings =NULL ;
@@ -560,7 +561,7 @@ static int io_register_resize_rings(struct io_ring_ctx *ctx, void __user *arg) ret =0 ; out:spin_unlock (&ctx->completion_lock); -mmap_write_unlock (current->mm); +mutex_unlock (&ctx->resize_lock);io_register_free_rings (&p, to_free);if (ctx->sq_data)
Signature · 1 line
Jens Axboe
Re: [PATCH 4/4] io_uring/register: add IORING_REGISTER_RESIZE_RINGS
Quoted 42 lines · Jens Axboe
Quoted 41 lines · Jann Horn
Quoted 39 lines · On Thu, Oct 24, 2024 at 9:59?PM Jens Axboe <[email protected]> wrote:
Quoted 27 lines · Jann Horn
Quoted 19 lines · On Thu, Oct 24, 2024 at 9:50?PM Jens Axboe <[email protected]> wrote:
Quoted 15 lines · Jann Horn
Quoted 4 lines · On Thu, Oct 24, 2024 at 7:08?PM Jens Axboe <[email protected]> wrote:
Add
IORING_REGISTER_RESIZE_RINGS, which allows an application to resize the existing rings. It takes a structio_uring_paramsargument, the same one which is used to setup the ring initially, and resizes rings according to the sizes given.[...]
Review
Quoted 8 lines
(quoted diff) +8
-0+ * We'll do the swap. Clear out existing mappings to prevent mmap + * from seeing them, as we'll unmap them. Any attempt to mmap existing + * rings beyond this point will fail. Not that it could proceed at this + * point anyway, as we'll hold the mmap_sem until we've done the swap. + * Likewise, hold the completion * lock over the duration of the actual + * swap. + */ + mmap_write_lock(current->mm);Quoted 2 lines
Why does the mmap lock for
current->mmsuffice here? I see nothing inio_uring_mmap()that limitsmmap()to tasks with the samemm_struct.Quoted 6 lines
Ehm does ->mmap() not hold ->mmap_sem already? I was under that understanding. Obviously if it doesn't, then yeah this won't be enough. Checked, and it does.
Ah I see what you mean now, task with different mm. But how would that come about? The
io_uringfd is CLOEXEC, and it can't get passed.Quoted 8 lines
Yeah, that's what I meant, tasks with different mm. I think there are a few ways to get the
io_uringfd into a different task, the ones I can immediately think of:-
O_CLOEXEConly applies onexecve(),fork()should still inherit the fd
-O_CLOEXECcan be cleared viafcntl()
- you can useclone()to create two tasks that share FD tables
without sharing an mmOK good catch, yes then it won't be enough. Might just make sense to exclude mmap separately, then. Thanks, I'll work on that for v4!
Quoted 6 lines
Quoted 1 line
Yeah, that sounds reasonable to me.
Something like this should do it, it's really just replacing
mmap_semwith a ring private lock. And since the ordering already had to deal withuring_lockvsmmap_semABBA issues, this should slot straight in as well.
Looks good to me at a glance.
Jens Axboe
maintainerRe: [PATCH 4/4] io_uring/register: add IORING_REGISTER_RESIZE_RINGS
Quoted 43 lines · Jann Horn
Quoted 42 lines · On Thu, Oct 24, 2024 at 10:25?PM Jens Axboe <[email protected]> wrote:
Quoted 41 lines · Jann Horn
Quoted 39 lines · On Thu, Oct 24, 2024 at 9:59?PM Jens Axboe <[email protected]> wrote:
Quoted 27 lines · Jann Horn
Quoted 19 lines · On Thu, Oct 24, 2024 at 9:50?PM Jens Axboe <[email protected]> wrote:
Quoted 15 lines · Jann Horn
Quoted 4 lines · On Thu, Oct 24, 2024 at 7:08?PM Jens Axboe <[email protected]> wrote:
Add
IORING_REGISTER_RESIZE_RINGS, which allows an application to resize the existing rings. It takes a structio_uring_paramsargument, the same one which is used to setup the ring initially, and resizes rings according to the sizes given.[...]
Review
Quoted 8 lines
(quoted diff) +8
-0+ * We'll do the swap. Clear out existing mappings to prevent mmap + * from seeing them, as we'll unmap them. Any attempt to mmap existing + * rings beyond this point will fail. Not that it could proceed at this + * point anyway, as we'll hold the mmap_sem until we've done the swap. + * Likewise, hold the completion * lock over the duration of the actual + * swap. + */ + mmap_write_lock(current->mm);Quoted 2 lines
Why does the mmap lock for
current->mmsuffice here? I see nothing inio_uring_mmap()that limitsmmap()to tasks with the samemm_struct.Quoted 6 lines
Ehm does ->mmap() not hold ->mmap_sem already? I was under that understanding. Obviously if it doesn't, then yeah this won't be enough. Checked, and it does.
Ah I see what you mean now, task with different mm. But how would that come about? The
io_uringfd is CLOEXEC, and it can't get passed.Quoted 8 lines
Yeah, that's what I meant, tasks with different mm. I think there are a few ways to get the
io_uringfd into a different task, the ones I can immediately think of:-
O_CLOEXEConly applies onexecve(),fork()should still inherit the fd
-O_CLOEXECcan be cleared viafcntl()
- you can useclone()to create two tasks that share FD tables
without sharing an mmOK good catch, yes then it won't be enough. Might just make sense to exclude mmap separately, then. Thanks, I'll work on that for v4!
Quoted 8 lines
Quoted 6 lines
Quoted 1 line
Yeah, that sounds reasonable to me.
Something like this should do it, it's really just replacing
mmap_semwith a ring private lock. And since the ordering already had to deal withuring_lockvsmmap_semABBA issues, this should slot straight in as well.Looks good to me at a glance.
Great, thanks for checking Jann. In the first place as well, appreciate it.
FWIW, compiled and ran through the testing, looks fine so far here.
Signature · 1 line
Jens Axboe
Jens Axboe
maintainerRe: [PATCH 4/4] io_uring/register: add IORING_REGISTER_RESIZE_RINGS
Quoted 51 lines · Jens Axboe
Quoted 50 lines · Jann Horn
Quoted 48 lines · On Thu, Oct 24, 2024 at 10:25?PM Jens Axboe <[email protected]> wrote:
Quoted 38 lines · Jann Horn
Quoted 37 lines · On Thu, Oct 24, 2024 at 9:59?PM Jens Axboe <[email protected]> wrote:
Quoted 35 lines · Jann Horn
Quoted 19 lines · On Thu, Oct 24, 2024 at 9:50?PM Jens Axboe <[email protected]> wrote:
Quoted 15 lines · Jann Horn
Quoted 4 lines · On Thu, Oct 24, 2024 at 7:08?PM Jens Axboe <[email protected]> wrote:
Add
IORING_REGISTER_RESIZE_RINGS, which allows an application to resize the existing rings. It takes a structio_uring_paramsargument, the same one which is used to setup the ring initially, and resizes rings according to the sizes given.[...]
Review
Quoted 8 lines
(quoted diff) +8
-0+ * We'll do the swap. Clear out existing mappings to prevent mmap + * from seeing them, as we'll unmap them. Any attempt to mmap existing + * rings beyond this point will fail. Not that it could proceed at this + * point anyway, as we'll hold the mmap_sem until we've done the swap. + * Likewise, hold the completion * lock over the duration of the actual + * swap. + */ + mmap_write_lock(current->mm);Quoted 2 lines
Why does the mmap lock for
current->mmsuffice here? I see nothing inio_uring_mmap()that limitsmmap()to tasks with the samemm_struct.Quoted 6 lines
Ehm does ->mmap() not hold ->mmap_sem already? I was under that understanding. Obviously if it doesn't, then yeah this won't be enough. Checked, and it does.
Ah I see what you mean now, task with different mm. But how would that come about? The
io_uringfd is CLOEXEC, and it can't get passed.Yeah, that's what I meant, tasks with different mm. I think there are a few ways to get the
io_uringfd into a different task, the ones I can immediately think of:-
O_CLOEXEConly applies onexecve(),fork()should still inherit the fd
-O_CLOEXECcan be cleared viafcntl()
- you can useclone()to create two tasks that share FD tables
without sharing an mmQuoted 4 lines
Quoted 2 lines
OK good catch, yes then it won't be enough. Might just make sense to exclude mmap separately, then. Thanks, I'll work on that for v4!
Yeah, that sounds reasonable to me.
Something like this should do it, it's really just replacing
mmap_semwith a ring private lock. And since the ordering already had to deal withuring_lockvsmmap_semABBA issues, this should slot straight in as well.
Quoted 6 lines
Quoted 1 line
Looks good to me at a glance.
Great, thanks for checking Jann. In the first place as well, appreciate it.
FWIW, compiled and ran through the testing, looks fine so far here.
And also fwiw, I did write a test case for this, and it goes boom pretty quickly without the patch, no issues with the patch. Sample output:
==================================================================
BUG: KASAN: slab-use-after-free in vm_insert_pages+0x634/0x73c Read of size 8 at addr ffff0000d8a264e0 by task resize-rings.t/741 CPU: 5 UID: 1000 PID: 741 Comm: resize-rings.t Not tainted 6.12.0-rc4-00082-g0935537ea92a #7661 Hardware name: linux,dummy-virt (DT) Call trace: dump_backtrace.part.0+0xd0/0xe0 show_stack+0x14/0x1c dump_stack_lvl+0x68/0x8c print_report+0x16c/0x4c8 kasan_report+0xa0/0xe0 __asan_report_load8_noabort+0x1c/0x24 vm_insert_pages+0x634/0x73c io_uring_mmap_pages+0x1d4/0x2d8 io_uring_mmap+0x19c/0x1c0 mmap_region+0x844/0x19e0 do_mmap+0x5f4/0xb00 vm_mmap_pgoff+0x164/0x2a0 ksys_mmap_pgoff+0x2a8/0x3c0 __arm64_sys_mmap+0xc8/0x140 invoke_syscall+0x6c/0x260 el0_svc_common.constprop.0+0x158/0x224 do_el0_svc+0x3c/0x5c el0_svc+0x44/0xb4 el0t_64_sync_handler+0x118/0x124 el0t_64_sync+0x168/0x16c Allocated by task 733: kasan_save_stack+0x28/0x4c kasan_save_track+0x1c/0x40 kasan_save_alloc_info+0x3c/0x4c __kasan_kmalloc+0xac/0xb0 __kmalloc_node_noprof+0x1b4/0x3f0 __kvmalloc_node_noprof+0x68/0x134 io_pages_map+0x50/0x448 io_register_resize_rings+0x484/0x1498 __arm64_sys_io_uring_register+0x780/0x1f3c invoke_syscall+0x6c/0x260 el0_svc_common.constprop.0+0x158/0x224 do_el0_svc+0x3c/0x5c el0_svc+0x44/0xb4 el0t_64_sync_handler+0x118/0x124 el0t_64_sync+0x168/0x16c Freed by task 733: kasan_save_stack+0x28/0x4c kasan_save_track+0x1c/0x40 kasan_save_free_info+0x48/0x94 __kasan_slab_free+0x48/0x60 kfree+0x120/0x494 kvfree+0x34/0x40 io_pages_unmap+0x1a4/0x308 io_register_free_rings.isra.0+0x6c/0x168 io_register_resize_rings+0xce4/0x1498 __arm64_sys_io_uring_register+0x780/0x1f3c invoke_syscall+0x6c/0x260 el0_svc_common.constprop.0+0x158/0x224 do_el0_svc+0x3c/0x5c el0_svc+0x44/0xb4 el0t_64_sync_handler+0x118/0x124 el0t_64_sync+0x168/0x16c The buggy address belongs to the object at ffff0000d8a264e0 which belongs to the cache kmalloc-cg-8 of size 8
The buggy address is located 0 bytes inside of
freed 8-byte region [ffff0000d8a264e0, ffff0000d8a264e8)
Signature · 1 line
Jens Axboe