io_uring: fix locking in __io_run_local_work
cover io_uring: fix locking in __io_run_local_work
If locked was not set in __io_run_local_work, but some task work managed to lock the context, it would leave things locked indefinitely. Fix that by passing the pointer in.
Patch 1 is a tiny cleanup to simplify things
Patch 2 is the fix
Dylan Yudaken (2):
io_uring: use io_run_local_work_locked helper
io_uring: unlock if __io_run_local_work locked inside
io_uring/io_uring.c | 11 +++++------ io_uring/io_uring.h | 12 ++++++++++-- 2 files changed, 15 insertions(+), 8 deletions(-)
base-commit: 247f34f7b80357943234f93f247a1ae6b6c3a740
Signature · 1 line
2.30.2
PATCH 1/2 io_uring: use io_run_local_work_locked helper
prefer to use io_run_local_work_locked helper for consistency
--- io_uring/io_uring.c | 3 +-- 1 file changed, 1 insertion(+), 2 deletions(-)
io_uring/io_uring.c +1 -2
@@ -1446,8 +1446,7 @@ static int io_iopoll_check(struct io_ring_ctx *ctx, long min) io_task_work_pending(ctx)) {u32 tail = ctx->cached_cq_tail; -if (!llist_empty (&ctx->work_llist)) -__io_run_local_work (ctx, true); + (void )io_run_local_work_locked (ctx);if (task_work_pending (current) ||wq_list_empty (&ctx->iopoll_list)) {
Signature · 1 line
2.30.2
PATCH 2/2 io_uring: unlock if __io_run_local_work locked inside
In reply to Dylan YudakenIt is possible for tw to lock the ring, and this was not propogated out to io_run_local_work. This can cause an unlock to be missed.
Instead pass a pointer to locked into __io_run_local_work.
--- io_uring/io_uring.c | 8 ++++---- io_uring/io_uring.h | 12 ++++++++++-- 2 files changed, 14 insertions(+), 6 deletions(-)
io_uring/io_uring.c +4 -4
@@ -1173,7 +1173,7 @@ static void __cold io_move_task_work_from_local(struct io_ring_ctx *ctx) } } -int __io_run_local_work (struct io_ring_ctx *ctx,bool locked) +int __io_run_local_work (struct io_ring_ctx *ctx,bool *locked) {struct llist_node *node;struct llist_node fake;
@@ -1192,7 +1192,7 @@ int __io_run_local_work(struct io_ring_ctx *ctx, bool locked) struct io_kiocb *req = container_of(node, struct io_kiocb, io_task_work.node); prefetch(container_of(next, struct io_kiocb, io_task_work.node)); - req->io_task_work.func (req, &locked); + req->io_task_work.func (req, locked); ret++; node = next; }
@@ -1208,7 +1208,7 @@ int __io_run_local_work(struct io_ring_ctx *ctx, bool locked) goto again; } -if (locked) +if (*locked)io_submit_flush_completions (ctx);trace_io_uring_local_work_run (ctx, ret, loops);return ret;
@@ -1225,7 +1225,7 @@ int io_run_local_work(struct io_ring_ctx *ctx)__set_current_state (TASK_RUNNING ); locked =mutex_trylock (&ctx->uring_lock); - ret =__io_run_local_work (ctx, locked); + ret =__io_run_local_work (ctx, &locked);if (locked)mutex_unlock (&ctx->uring_lock);
io_uring/io_uring.h +10 -2
@@ -27,7 +27,7 @@ enum {struct io_uring_cqe *__io_get_cqe (struct io_ring_ctx *ctx,bool overflow);bool io_req_cqe_overflow (struct io_kiocb *req);int io_run_task_work_sig (struct io_ring_ctx *ctx); -int __io_run_local_work (struct io_ring_ctx *ctx,bool locked); +int __io_run_local_work (struct io_ring_ctx *ctx,bool *locked);int io_run_local_work (struct io_ring_ctx *ctx);void io_req_complete_failed (struct io_kiocb *req,s32 res);void __io_req_complete (struct io_kiocb *req,unsigned issue_flags);
@@ -277,9 +277,17 @@ static inline int io_run_task_work_ctx(struct io_ring_ctx *ctx)static inline int io_run_local_work_locked (struct io_ring_ctx *ctx) { +bool locked; +int ret; +if (llist_empty (&ctx->work_llist))return 0 ; -return __io_run_local_work (ctx, true); + + locked = true; + ret =__io_run_local_work (ctx, &locked); +if (WARN_ON (!locked)) +mutex_lock (&ctx->uring_lock); +return ret; }static inline void io_tw_lock (struct io_ring_ctx *ctx,bool *locked)
Signature · 1 line
2.30.2
Jens Axboe
maintainerRe: [PATCH 2/2] io_uring: unlock if __io_run_local_work locked inside
Quoted 72 lines · Dylan Yudaken
It is possible for tw to lock the ring, and this was not propogated out to
io_run_local_work. This can cause an unlock to be missed.Instead pass a pointer to locked into
__io_run_local_work.Fixes 8ac5d85a89b4 ("io_uring: add local task_work run helper that is entered locked")Signed-off-by Dylan Yudaken <[email protected]>--- io_uring/io_uring.c | 8 ++++---- io_uring/io_uring.h | 12 ++++++++++-- 2 files changed, 14 insertions(+), 6 deletions(-)io_uring/io_uring.c +4
-4@@ -1173,7 +1173,7 @@ static void __cold io_move_task_work_from_local(struct io_ring_ctx *ctx) } } -int __io_run_local_work (struct io_ring_ctx *ctx,bool locked) +int __io_run_local_work (struct io_ring_ctx *ctx,bool *locked) {struct llist_node *node;struct llist_node fake;@@ -1192,7 +1192,7 @@ int __io_run_local_work(struct io_ring_ctx *ctx, bool locked) struct io_kiocb *req = container_of(node, struct io_kiocb, io_task_work.node); prefetch(container_of(next, struct io_kiocb, io_task_work.node)); - req->io_task_work.func (req, &locked); + req->io_task_work.func (req, locked); ret++; node = next; }@@ -1208,7 +1208,7 @@ int __io_run_local_work(struct io_ring_ctx *ctx, bool locked) goto again; } -if (locked) +if (*locked)io_submit_flush_completions (ctx);trace_io_uring_local_work_run (ctx, ret, loops);return ret;@@ -1225,7 +1225,7 @@ int io_run_local_work(struct io_ring_ctx *ctx)__set_current_state (TASK_RUNNING ); locked =mutex_trylock (&ctx->uring_lock); - ret =__io_run_local_work (ctx, locked); + ret =__io_run_local_work (ctx, &locked);if (locked)mutex_unlock (&ctx->uring_lock);io_uring/io_uring.h +4
-1@@ -27,7 +27,7 @@ enum {struct io_uring_cqe *__io_get_cqe (struct io_ring_ctx *ctx,bool overflow);bool io_req_cqe_overflow (struct io_kiocb *req);int io_run_task_work_sig (struct io_ring_ctx *ctx); -int __io_run_local_work (struct io_ring_ctx *ctx,bool locked); +int __io_run_local_work (struct io_ring_ctx *ctx,bool *locked);int io_run_local_work (struct io_ring_ctx *ctx);void io_req_complete_failed (struct io_kiocb *req,s32 res);void __io_req_complete (struct io_kiocb *req,unsigned issue_flags);@@ -277,9 +277,17 @@ static inline int io_run_task_work_ctx(struct io_ring_ctx *ctx)static inline int io_run_local_work_locked (struct io_ring_ctx *ctx) { +bool locked; +int ret; +
io_uring/io_uring.h static inline int io_run_task_work_ctx(struct io_ring_ctx *ctx) · line 291
Quoted 10 lines
io_uring/io_uring.h +6
-1if (llist_empty (&ctx->work_llist))return 0 ; -return __io_run_local_work (ctx, true); + + locked = true; + ret =__io_run_local_work (ctx, &locked); +if (WARN_ON (!locked)) +mutex_lock (&ctx->uring_lock); +return ret; }
If you think warning on !locked is a good idea, it should be a WARN_ON_ONCE(). Or is this leftover debugging?
Signature · 1 line
Jens Axboe
Re: [PATCH 2/2] io_uring: unlock if __io_run_local_work locked inside
Quoted 12 lines · Jens Axboe
Quoted 11 lines · Dylan Yudaken
It is possible for tw to lock the ring, and this was not propogated out to
io_run_local_work. This can cause an unlock to be missed.Instead pass a pointer to locked into
__io_run_local_work.Fixes 8ac5d85a89b4 ("io_uring: add local task_work run helper thatis entered locked")
Signed-off-by Dylan Yudaken <[email protected]>---
Quoted 8 lines
Quoted 4 lines
+ if (
WARN_ON(!locked))
+ mutex_lock(&ctx->uring_lock);
+ return ret;
}If you think warning on !locked is a good idea, it should be a
WARN_ON_ONCE(). Or is this leftover debugging?
It's not leftover. Basically it should not be (afaik) that tw will unlock the mutex, but I didn't want to leave a dangling unlocked mutex.
Maybe that is being too conservative and we can just kill both lines - we never used to check for this.
Happy for either way
Dylan
Quoted 8 lines · Dylan Yudaken
If locked was not set in
__io_run_local_work, but some task work managed to lock the context, it would leave things locked indefinitely. Fix that by passing the pointer in.Patch 1 is a tiny cleanup to simplify things
Patch 2 is the fix[...]
Applied, thanks!
[1/2] io_uring: use io_run_local_work_locked helper
commit: 8de11cdc96bf58b324c59a28512eb9513fd02553
[2/2] io_uring: unlock if __io_run_local_work locked inside
commit: b3026767e15b488860d4bbf1649d69612bab2c25Best regards,
Signature · 1 line
Jens Axboe
Jens Axboe
maintainerQuoted 8 lines · Jens Axboe
Quoted 7 lines · Dylan Yudaken
If locked was not set in
__io_run_local_work, but some task work managed to lock the context, it would leave things locked indefinitely. Fix that by passing the pointer in.Patch 1 is a tiny cleanup to simplify things
Patch 2 is the fix
Quoted 8 lines
Quoted 1 line
[...]
Applied, thanks!
[1/2]
io_uring: useio_run_local_work_lockedhelpercommit: 8de11cdc96bf58b324c59a28512eb9513fd02553 [2/2] io_uring: unlock if __io_run_local_work locked inside commit: b3026767e15b488860d4bbf1649d69612bab2c25
I made the WARN_ON() -> WARN_ON_ONCE() edit and added a small comment as well, while applying.
Signature · 1 line
Jens Axboe