Various bug fixes
Jens Axboe
maintainercover Various bug fixes
Hi,
A random bag of fixes and cleanups. In detail:
- Patch 1, defensive cleanup for a patch merged in this merge window. Not reachable, but it's confusing and should get cleaned up.
- Patch 2, spectre masking for file updates.
- Patch 3, defensive cleanup for the imu cache, using kvfree() consistently. Idea being that it'd be easy to mess this up in the future if caching changes.
- Patch 4, more defensive cleanups, just hardening ensuring that only >= 0 is passed in for bytes consumed for the kbuf path.
- Patch 5, actual fix for futex, where multiple partial wakeups would end up waking the same queue multiple times, rather than moving on to the next one.
- Patch 6, actual fix for ring resizing with CQE32/SQE128 and pending entries in the SQ or CQ rings.
io_uring/alloc_cache.h | 2 +- io_uring/futex.c | 4 +++- io_uring/io_uring.c | 3 ++- io_uring/register.c | 36 ++++++++++++++++++++++++++++-------- io_uring/rsrc.c | 5 ++++- io_uring/rsrc.h | 9 +++++++-- io_uring/rw.c | 4 ++-- 7 files changed, 47 insertions(+), 16 deletions(-)
Signature · 1 line
Jens Axboe
Jens Axboe
maintainerPATCH 1/6 io_uring: fix spurious fput in registered ring path
Fix an issue with io_uring_ctx_get_file() not gating fput() on whether or not the file descriptor is a registered/direct one or not.
--- io_uring/io_uring.c | 3 ++- 1 file changed, 2 insertions(+), 1 deletion(-)
io_uring/io_uring.c +2 -1
@@ -2575,7 +2575,8 @@ struct file *io_uring_ctx_get_file(unsigned int fd, bool registered)return ERR_PTR (-EBADF );if (io_is_uring_fops (file))return file; -fput (file); +if (!registered) +fput (file);return ERR_PTR (-EOPNOTSUPP ); }
Signature · 1 line
2.53.0
Re: [PATCH 1/6] io_uring: fix spurious fput in registered ring path
Quoted 5 lines · Jens Axboe
Fix an issue with
io_uring_ctx_get_file()not gatingfput()on whether or not the file descriptor is a registered/direct one or not.Fixes c5e9f6a96bf7 ("io_uring: unify getting ctx from passed in file descriptor")Signed-off-by Jens Axboe <[email protected]>
Oh.
Signature · 1 line
Gabriel Krisman Bertazi
Jens Axboe
maintainerPATCH 2/6 io_uring/rsrc: unify nospec indexing for direct descriptors
In reply to Jens AxboeFor file updates, the node reset isn't capping the value via array_index_nospec() like the other paths do. Ensure it's all sane and have the update path do the proper capping as well.
--- io_uring/rsrc.c | 3 +++ io_uring/rsrc.h | 9 +++++++-- 2 files changed, 10 insertions(+), 2 deletions(-)
io_uring/rsrc.c +3 -0
@@ -238,6 +238,9 @@ static int __io_sqe_files_update(struct io_ring_ctx *ctx,continue ; i = up->offset + done; +if (i >= ctx->file_table.data.nr) +break ; + i =array_index_nospec (i, ctx->file_table.data.nr);if (io_reset_rsrc_node (ctx, &ctx->file_table.data, i))io_file_bitmap_clear (&ctx->file_table, i);
io_uring/rsrc.h +7 -2
@@ -109,10 +109,15 @@ static inline void io_put_rsrc_node(struct io_ring_ctx *ctx, struct io_rsrc_node }static inline bool io_reset_rsrc_node (struct io_ring_ctx *ctx, -struct io_rsrc_data *data,int index) +struct io_rsrc_data *data, +unsigned int index) { -struct io_rsrc_node *node = data->nodes[index]; +struct io_rsrc_node *node; +if (index >= data->nr) +return false; + index =array_index_nospec (index, data->nr); + node = data->nodes[index];if (!node)return false;io_put_rsrc_node (ctx, node);
Signature · 1 line
2.53.0
Re: [PATCH 2/6] io_uring/rsrc: unify nospec indexing for direct descriptors
Quoted 5 lines · Jens Axboe
For file updates, the node reset isn't capping the value via
array_index_nospec()like the other paths do. Ensure it's all sane and have the update path do the proper capping as well.Signed-off-by Jens Axboe <[email protected]>
Quoted 42 lines
--- io_uring/rsrc.c | 3 +++ io_uring/rsrc.h | 9 +++++++-- 2 files changed, 10 insertions(+), 2 deletions(-)io_uring/rsrc.c +3
-0@@ -238,6 +238,9 @@ static int __io_sqe_files_update(struct io_ring_ctx *ctx,continue ; i = up->offset + done; +if (i >= ctx->file_table.data.nr) +break ; + i =array_index_nospec (i, ctx->file_table.data.nr);if (io_reset_rsrc_node (ctx, &ctx->file_table.data, i))io_file_bitmap_clear (&ctx->file_table, i);io_uring/rsrc.h +7
-2@@ -109,10 +109,15 @@ static inline void io_put_rsrc_node(struct io_ring_ctx *ctx, struct io_rsrc_node }static inline bool io_reset_rsrc_node (struct io_ring_ctx *ctx, -struct io_rsrc_data *data,int index) +struct io_rsrc_data *data, +unsigned int index) { -struct io_rsrc_node *node = data->nodes[index]; +struct io_rsrc_node *node; +if (index >= data->nr) +return false; + index =array_index_nospec (index, data->nr); + node = data->nodes[index];if (!node)return false;io_put_rsrc_node (ctx, node);
Signature · 1 line
Gabriel Krisman Bertazi
Jens Axboe
maintainerPATCH 3/6 io_uring/rsrc: use kvfree() for the imu cache
In reply to Jens AxboeCurrently anything that requires kvmalloc_flex() for allocations will not get re-cached, and hence the cache freeing path is correct in that it always uses kfree() to free the allocated memory. But this seems a bit fragile as it's something that could get mix should that situation change, so switch io_free_imu() and io_alloc_cache_free() to use kvfree as the desctructor.
--- io_uring/alloc_cache.h | 2 +- io_uring/rsrc.c | 2 +- 2 files changed, 2 insertions(+), 2 deletions(-)
io_uring/alloc_cache.h +1 -1
@@ -64,7 +64,7 @@ static inline void *io_cache_alloc(struct io_alloc_cache *cache, gfp_t gfp)static inline void io_cache_free (struct io_alloc_cache *cache,void *obj) {if (!io_alloc_cache_put (cache, obj)) -kfree (obj); +kvfree (obj); }#endif
io_uring/rsrc.c +1 -1
@@ -168,7 +168,7 @@ bool io_rsrc_cache_init(struct io_ring_ctx *ctx)void io_rsrc_cache_free (struct io_ring_ctx *ctx) {io_alloc_cache_free (&ctx->node_cache, kfree); -io_alloc_cache_free (&ctx->imu_cache, kfree); +io_alloc_cache_free (&ctx->imu_cache, kvfree); }static void io_clear_table_tags (struct io_rsrc_data *data)
Signature · 1 line
2.53.0
Jens Axboe
maintainerPATCH 4/6 io_uring/rw: add defensive hardening for negative kbuf lengths
In reply to Jens AxboeNo real bug here, just being a bit defensive in ensuring that whatever gets passed into io_put_kbuf() is always >= 0 and not some random error value.
--- io_uring/rw.c | 4 ++-- 1 file changed, 2 insertions(+), 2 deletions(-)
io_uring/rw.c +2 -2
@@ -580,7 +580,7 @@ void io_req_rw_complete(struct io_tw_req tw_req, io_tw_token_t tw)io_req_io_end (req);if (req->flags & (REQ_F_BUFFER_SELECTED |REQ_F_BUFFER_RING )) - req->cqe.flags |=io_put_kbuf (req, req->cqe.res,NULL ); + req->cqe.flags |=io_put_kbuf (req,max (req->cqe.res,0 ),NULL );io_req_rw_cleanup (req,0 );io_req_task_complete (tw_req, tw);
@@ -1379,7 +1379,7 @@ int io_do_iopoll(struct io_ring_ctx *ctx, bool force_nonspin)list_del (&req->iopoll_node);wq_list_add_tail (&req->comp_list, &ctx->submit_state.compl_reqs); nr_events++; - req->cqe.flags =io_put_kbuf (req, req->cqe.res,NULL ); + req->cqe.flags =io_put_kbuf (req,max (req->cqe.res,0 ),NULL );if (!io_is_uring_cmd (req))io_req_rw_cleanup (req,0 ); }
Signature · 1 line
2.53.0
Re: [PATCH 4/6] io_uring/rw: add defensive hardening for negative kbuf lengths
Quoted 23 lines · Jens Axboe
No real bug here, just being a bit defensive in ensuring that whatever gets passed into
io_put_kbuf()is always >= 0 and not some random error value.Signed-off-by Jens Axboe <[email protected]>--- io_uring/rw.c | 4 ++-- 1 file changed, 2 insertions(+), 2 deletions(-)io_uring/rw.c +1
-1@@ -580,7 +580,7 @@ void io_req_rw_complete(struct io_tw_req tw_req, io_tw_token_t tw)io_req_io_end (req);if (req->flags & (REQ_F_BUFFER_SELECTED |REQ_F_BUFFER_RING )) - req->cqe.flags |=io_put_kbuf (req, req->cqe.res,NULL ); + req->cqe.flags |=io_put_kbuf (req,max (req->cqe.res,0 ),NULL );io_req_rw_cleanup (req,0 );io_req_task_complete (tw_req, tw);
io_uring/rw.c int io_do_iopoll(struct io_ring_ctx *ctx, bool force_nonspin) · line 1385
Quoted 8 lines
io_uring/rw.c +1
-1@@ -1379,7 +1379,7 @@ int io_do_iopoll(struct io_ring_ctx *ctx, bool force_nonspin)list_del (&req->iopoll_node);wq_list_add_tail (&req->comp_list, &ctx->submit_state.compl_reqs); nr_events++; - req->cqe.flags =io_put_kbuf (req, req->cqe.res,NULL ); + req->cqe.flags =io_put_kbuf (req,max (req->cqe.res,0 ),NULL );if (!io_is_uring_cmd (req))io_req_rw_cleanup (req,0 ); }
Much more readable if it were rolled as an early return inside io_put_kbuf, but clearly:
Signature · 1 line
Gabriel Krisman Bertazi
Jens Axboe
maintainerPATCH 5/6 io_uring/futex: ensure partial wakes are appropriately dequeued
In reply to Jens AxboeIf a FUTEX_WAITV vectored operation is only partially woken, we should call __futex_wake_mark() on the queue to account for that. If not, then a later wakeup will wake the same entry, rather than the next one in line.
--- io_uring/futex.c | 4 +++- 1 file changed, 3 insertions(+), 1 deletion(-)
io_uring/futex.c +3 -1
@@ -159,8 +159,10 @@ static void io_futex_wakev_fn(struct wake_q_head *wake_q, struct futex_q *q)struct io_kiocb *req = q->wake_data;struct io_futexv_data *ifd = req->async_data; -if (!io_futexv_claim (ifd)) +if (!io_futexv_claim (ifd)) { +__futex_wake_mark (q);return ; + }if (unlikely (!__futex_wake_mark (q)))return ;
Signature · 1 line
2.53.0
Re: [PATCH 5/6] io_uring/futex: ensure partial wakes are appropriately dequeued
Quoted 7 lines · Jens Axboe
If a
FUTEX_WAITVvectored operation is only partially woken, we should call__futex_wake_mark()on the queue to account for that. If not, then a later wakeup will wake the same entry, rather than the next one in line.Fixes 8f350194d5cfd ("io_uring: add support for vectored futex waits")Signed-off-by Jens Axboe <[email protected]>
Quoted 19 lines
--- io_uring/futex.c | 4 +++- 1 file changed, 3 insertions(+), 1 deletion(-)io_uring/futex.c +3
-1@@ -159,8 +159,10 @@ static void io_futex_wakev_fn(struct wake_q_head *wake_q, struct futex_q *q)struct io_kiocb *req = q->wake_data;struct io_futexv_data *ifd = req->async_data; -if (!io_futexv_claim (ifd)) +if (!io_futexv_claim (ifd)) { +__futex_wake_mark (q);return ; + }if (unlikely (!__futex_wake_mark (q)))return ;
Signature · 1 line
Gabriel Krisman Bertazi
Jens Axboe
maintainerPATCH 6/6 io_uring/register: fix ring resizing with mixed/large SQEs/CQEs
In reply to Jens AxboeThe ring resizing only properly handles "normal" sized SQEs or CQEs, if there are pending entries around a resize. This normally should not be the case, but the code is supposed to handle this regardless.
For the mixed SQE/CQE cases, the current copying works fine as they are indexed in the same way. Each half is just copied separately. But for fixed large SQEs and CQEs, the iteration and copy need to take that into account.
--- io_uring/register.c | 36 ++++++++++++++++++++++++++++-------- 1 file changed, 28 insertions(+), 8 deletions(-)
io_uring/register.c +28 -8
@@ -599,10 +599,20 @@ static int io_register_resize_rings(struct io_ring_ctx *ctx, void __user *arg)if (tail - old_head > p->sq_entries) goto overflow;for (i = old_head; i < tail; i++) { -unsigned src_head = i & (ctx->sq_entries -1 ); -unsigned dst_head = i & (p->sq_entries -1 ); - - n.sq_sqes[dst_head] = o.sq_sqes[src_head]; +unsigned index, dst_mask, src_mask; +size_t sq_size; + + index = i; + sq_size =sizeof (struct io_uring_sqe ); + src_mask = ctx->sq_entries -1 ; + dst_mask = p->sq_entries -1 ; +if (ctx->flags &IORING_SETUP_SQE128 ) { + index <<=1 ; + sq_size <<=1 ; + src_mask = (ctx->sq_entries <<1 ) -1 ; + dst_mask = (p->sq_entries <<1 ) -1 ; + } +memcpy (&n.sq_sqes[index & dst_mask], &o.sq_sqes[index & src_mask], sq_size); }WRITE_ONCE (n.rings->sq.head, old_head);WRITE_ONCE (n.rings->sq.tail, tail);
@@ -619,10 +629,20 @@ static int io_register_resize_rings(struct io_ring_ctx *ctx, void __user *arg) goto out; }for (i = old_head; i < tail; i++) { -unsigned src_head = i & (ctx->cq_entries -1 ); -unsigned dst_head = i & (p->cq_entries -1 ); - - n.rings->cqes[dst_head] = o.rings->cqes[src_head]; +unsigned index, dst_mask, src_mask; +size_t cq_size; + + index = i; + cq_size =sizeof (struct io_uring_cqe ); + src_mask = ctx->cq_entries -1 ; + dst_mask = p->cq_entries -1 ; +if (ctx->flags &IORING_SETUP_CQE32 ) { + index <<=1 ; + cq_size <<=1 ; + src_mask = (ctx->cq_entries <<1 ) -1 ; + dst_mask = (p->cq_entries <<1 ) -1 ; + } +memcpy (&n.rings->cqes[index & dst_mask], &o.rings->cqes[index & src_mask], cq_size); }WRITE_ONCE (n.rings->cq.head, old_head);WRITE_ONCE (n.rings->cq.tail, tail);
Signature · 1 line
2.53.0
Re: [PATCH 6/6] io_uring/register: fix ring resizing with mixed/large SQEs/CQEs
Quoted 63 lines · Jens Axboe
The ring resizing only properly handles "normal" sized SQEs or CQEs, if there are pending entries around a resize. This normally should not be the case, but the code is supposed to handle this regardless.
For the mixed SQE/CQE cases, the current copying works fine as they are indexed in the same way. Each half is just copied separately. But for fixed large SQEs and CQEs, the iteration and copy need to take that into account.
Fixes 79cfe9e59c2a ("io_uring/register: add IORING_REGISTER_RESIZE_RINGS")Signed-off-by Jens Axboe <[email protected]>--- io_uring/register.c | 36 ++++++++++++++++++++++++++++-------- 1 file changed, 28 insertions(+), 8 deletions(-)io_uring/register.c +21
-8@@ -599,10 +599,20 @@ static int io_register_resize_rings(struct io_ring_ctx *ctx, void __user *arg)if (tail - old_head > p->sq_entries) goto overflow;for (i = old_head; i < tail; i++) { -unsigned src_head = i & (ctx->sq_entries -1 ); -unsigned dst_head = i & (p->sq_entries -1 ); - - n.sq_sqes[dst_head] = o.sq_sqes[src_head]; +unsigned index, dst_mask, src_mask; +size_t sq_size; + + index = i; + sq_size =sizeof (struct io_uring_sqe ); + src_mask = ctx->sq_entries -1 ; + dst_mask = p->sq_entries -1 ; +if (ctx->flags &IORING_SETUP_SQE128 ) { + index <<=1 ; + sq_size <<=1 ; + src_mask = (ctx->sq_entries <<1 ) -1 ; + dst_mask = (p->sq_entries <<1 ) -1 ; + } +memcpy (&n.sq_sqes[index & dst_mask], &o.sq_sqes[index & src_mask], sq_size); }WRITE_ONCE (n.rings->sq.head, old_head);WRITE_ONCE (n.rings->sq.tail, tail);@@ -619,10 +629,20 @@ static int io_register_resize_rings(struct io_ring_ctx *ctx, void __user *arg) goto out; }for (i = old_head; i < tail; i++) { -unsigned src_head = i & (ctx->cq_entries -1 ); -unsigned dst_head = i & (p->cq_entries -1 ); - - n.rings->cqes[dst_head] = o.rings->cqes[src_head]; +unsigned index, dst_mask, src_mask; +size_t cq_size; + + index = i; + cq_size =sizeof (struct io_uring_cqe ); + src_mask = ctx->cq_entries -1 ; + dst_mask = p->cq_entries -1 ;
io_uring/register.c static int io_register_resize_rings(struct io_ring_ctx *ctx, void __user *arg) · line 648
Quoted 10 lines
io_uring/register.c +7
-0+if (ctx->flags &IORING_SETUP_CQE32 ) { + index <<=1 ; + cq_size <<=1 ; + src_mask = (ctx->cq_entries <<1 ) -1 ; + dst_mask = (p->cq_entries <<1 ) -1 ; + } +memcpy (&n.rings->cqes[index & dst_mask], &o.rings->cqes[index & src_mask], cq_size); }WRITE_ONCE (n.rings->cq.head, old_head);WRITE_ONCE (n.rings->cq.tail, tail);
Signature · 1 line
Gabriel Krisman Bertazi