Skip to contentSkip to conversation

io_uring: fix locking in __io_run_local_work

PATCH Applied Dylan Yudaken 2 participants 7 messages Oct 27, 2022 io_uring

lore
Share & tools
lore
#0 lore raw

cover io_uring: fix locking in __io_run_local_work

If locked was not set in __io_run_local_work, but some task work managed to lock the context, it would leave things locked indefinitely. Fix that by passing the pointer in.

Patch 1 is a tiny cleanup to simplify things
Patch 2 is the fix

Dylan Yudaken (2):
io_uring: use io_run_local_work_locked helper
io_uring: unlock if __io_run_local_work locked inside

 io_uring/io_uring.c | 11 +++++------
 io_uring/io_uring.h | 12 ++++++++++--
 2 files changed, 15 insertions(+), 8 deletions(-)

base-commit: 247f34f7b80357943234f93f247a1ae6b6c3a740

Signature · 1 line
2.30.2
#1 lore raw

PATCH 1/2 io_uring: use io_run_local_work_locked helper

prefer to use io_run_local_work_locked helper for consistency

---
 io_uring/io_uring.c | 3 +--
 1 file changed, 1 insertion(+), 2 deletions(-)
io_uring/io_uring.c +1 -2
index 6cc16e39b27f..8a0ce7379e89 100644
@@ -1446,8 +1446,7 @@ static int io_iopoll_check(struct io_ring_ctx *ctx, long min)
 		    io_task_work_pending(ctx)) {
 			u32 tail = ctx->cached_cq_tail;
 
-			if (!llist_empty(&ctx->work_llist))
-				__io_run_local_work(ctx, true);
+			(void) io_run_local_work_locked(ctx);
 
 			if (task_work_pending(current) ||
 			    wq_list_empty(&ctx->iopoll_list)) {
Signature · 1 line
2.30.2
#2 lore raw

PATCH 2/2 io_uring: unlock if __io_run_local_work locked inside

In reply to Dylan Yudaken

It is possible for tw to lock the ring, and this was not propogated out to io_run_local_work. This can cause an unlock to be missed.

Instead pass a pointer to locked into __io_run_local_work.

Fixes 8ac5d85a89b4 ("io_uring: add local task_work run helper that is entered locked")
---
 io_uring/io_uring.c |  8 ++++----
 io_uring/io_uring.h | 12 ++++++++++--
 2 files changed, 14 insertions(+), 6 deletions(-)
io_uring/io_uring.c +4 -4
index 8a0ce7379e89..ac8c488e3077 100644
@@ -1173,7 +1173,7 @@ static void __cold io_move_task_work_from_local(struct io_ring_ctx *ctx)
 	}
 }
 
-int __io_run_local_work(struct io_ring_ctx *ctx, bool locked)
+int __io_run_local_work(struct io_ring_ctx *ctx, bool *locked)
 {
 	struct llist_node *node;
 	struct llist_node fake;
@@ -1192,7 +1192,7 @@ int __io_run_local_work(struct io_ring_ctx *ctx, bool locked)
 		struct io_kiocb *req = container_of(node, struct io_kiocb,
 						    io_task_work.node);
 		prefetch(container_of(next, struct io_kiocb, io_task_work.node));
-		req->io_task_work.func(req, &locked);
+		req->io_task_work.func(req, locked);
 		ret++;
 		node = next;
 	}
@@ -1208,7 +1208,7 @@ int __io_run_local_work(struct io_ring_ctx *ctx, bool locked)
 		goto again;
 	}
 
-	if (locked)
+	if (*locked)
 		io_submit_flush_completions(ctx);
 	trace_io_uring_local_work_run(ctx, ret, loops);
 	return ret;
@@ -1225,7 +1225,7 @@ int io_run_local_work(struct io_ring_ctx *ctx)
 
 	__set_current_state(TASK_RUNNING);
 	locked = mutex_trylock(&ctx->uring_lock);
-	ret = __io_run_local_work(ctx, locked);
+	ret = __io_run_local_work(ctx, &locked);
 	if (locked)
 		mutex_unlock(&ctx->uring_lock);
 
io_uring/io_uring.h +10 -2
index ef77d2aa3172..331ec2869212 100644
@@ -27,7 +27,7 @@ enum {
 struct io_uring_cqe *__io_get_cqe(struct io_ring_ctx *ctx, bool overflow);
 bool io_req_cqe_overflow(struct io_kiocb *req);
 int io_run_task_work_sig(struct io_ring_ctx *ctx);
-int __io_run_local_work(struct io_ring_ctx *ctx, bool locked);
+int __io_run_local_work(struct io_ring_ctx *ctx, bool *locked);
 int io_run_local_work(struct io_ring_ctx *ctx);
 void io_req_complete_failed(struct io_kiocb *req, s32 res);
 void __io_req_complete(struct io_kiocb *req, unsigned issue_flags);
@@ -277,9 +277,17 @@ static inline int io_run_task_work_ctx(struct io_ring_ctx *ctx)
 
 static inline int io_run_local_work_locked(struct io_ring_ctx *ctx)
 {
+	bool locked;
+	int ret;
+
 	if (llist_empty(&ctx->work_llist))
 		return 0;
-	return __io_run_local_work(ctx, true);
+
+	locked = true;
+	ret = __io_run_local_work(ctx, &locked);
+	if (WARN_ON(!locked))
+		mutex_lock(&ctx->uring_lock);
+	return ret;
 }
 
 static inline void io_tw_lock(struct io_ring_ctx *ctx, bool *locked)
Signature · 1 line
2.30.2
maintainer
#3 lore raw

Re: [PATCH 2/2] io_uring: unlock if __io_run_local_work locked inside

Quoted 72 lines · Dylan Yudaken

It is possible for tw to lock the ring, and this was not propogated out to io_run_local_work. This can cause an unlock to be missed.

Instead pass a pointer to locked into __io_run_local_work.

Fixes 8ac5d85a89b4 ("io_uring: add local task_work run helper that is entered locked")
---
 io_uring/io_uring.c |  8 ++++----
 io_uring/io_uring.h | 12 ++++++++++--
 2 files changed, 14 insertions(+), 6 deletions(-)
io_uring/io_uring.c +4 -4
index 8a0ce7379e89..ac8c488e3077 100644
@@ -1173,7 +1173,7 @@ static void __cold io_move_task_work_from_local(struct io_ring_ctx *ctx)
 	}
 }
 
-int __io_run_local_work(struct io_ring_ctx *ctx, bool locked)
+int __io_run_local_work(struct io_ring_ctx *ctx, bool *locked)
 {
 	struct llist_node *node;
 	struct llist_node fake;
@@ -1192,7 +1192,7 @@ int __io_run_local_work(struct io_ring_ctx *ctx, bool locked)
 		struct io_kiocb *req = container_of(node, struct io_kiocb,
 						    io_task_work.node);
 		prefetch(container_of(next, struct io_kiocb, io_task_work.node));
-		req->io_task_work.func(req, &locked);
+		req->io_task_work.func(req, locked);
 		ret++;
 		node = next;
 	}
@@ -1208,7 +1208,7 @@ int __io_run_local_work(struct io_ring_ctx *ctx, bool locked)
 		goto again;
 	}
 
-	if (locked)
+	if (*locked)
 		io_submit_flush_completions(ctx);
 	trace_io_uring_local_work_run(ctx, ret, loops);
 	return ret;
@@ -1225,7 +1225,7 @@ int io_run_local_work(struct io_ring_ctx *ctx)
 
 	__set_current_state(TASK_RUNNING);
 	locked = mutex_trylock(&ctx->uring_lock);
-	ret = __io_run_local_work(ctx, locked);
+	ret = __io_run_local_work(ctx, &locked);
 	if (locked)
 		mutex_unlock(&ctx->uring_lock);
 
io_uring/io_uring.h +4 -1
index ef77d2aa3172..331ec2869212 100644
@@ -27,7 +27,7 @@ enum {
 struct io_uring_cqe *__io_get_cqe(struct io_ring_ctx *ctx, bool overflow);
 bool io_req_cqe_overflow(struct io_kiocb *req);
 int io_run_task_work_sig(struct io_ring_ctx *ctx);
-int __io_run_local_work(struct io_ring_ctx *ctx, bool locked);
+int __io_run_local_work(struct io_ring_ctx *ctx, bool *locked);
 int io_run_local_work(struct io_ring_ctx *ctx);
 void io_req_complete_failed(struct io_kiocb *req, s32 res);
 void __io_req_complete(struct io_kiocb *req, unsigned issue_flags);
@@ -277,9 +277,17 @@ static inline int io_run_task_work_ctx(struct io_ring_ctx *ctx)
 
 static inline int io_run_local_work_locked(struct io_ring_ctx *ctx)
 {
+	bool locked;
+	int ret;
+

io_uring/io_uring.h static inline int io_run_task_work_ctx(struct io_ring_ctx *ctx) · line 291

Quoted 10 lines
io_uring/io_uring.h +6 -1
 	if (llist_empty(&ctx->work_llist))
 		return 0;
-	return __io_run_local_work(ctx, true);
+
+	locked = true;
+	ret = __io_run_local_work(ctx, &locked);
+	if (WARN_ON(!locked))
+		mutex_lock(&ctx->uring_lock);
+	return ret;
 }

If you think warning on !locked is a good idea, it should be a WARN_ON_ONCE(). Or is this leftover debugging?

Signature · 1 line
Jens Axboe
#4 lore raw

Re: [PATCH 2/2] io_uring: unlock if __io_run_local_work locked inside

Quoted 12 lines · Jens Axboe
Quoted 11 lines · Dylan Yudaken

It is possible for tw to lock the ring, and this was not propogated out to
io_run_local_work. This can cause an unlock to be missed.

Instead pass a pointer to locked into __io_run_local_work.

Fixes 8ac5d85a89b4 ("io_uring: add local task_work run helper that

is entered locked")

---

Quoted 8 lines
Quoted 4 lines

+       if (WARN_ON(!locked))
+               mutex_lock(&ctx->uring_lock);
+       return ret;
 }

If you think warning on !locked is a good idea, it should be a WARN_ON_ONCE(). Or is this leftover debugging?

It's not leftover. Basically it should not be (afaik) that tw will unlock the mutex, but I didn't want to leave a dangling unlocked mutex.

Maybe that is being too conservative and we can just kill both lines - we never used to check for this.

Happy for either way

Dylan

maintainer Applied
#5 lore raw
In reply to Dylan Yudaken
Quoted 8 lines · Dylan Yudaken

If locked was not set in __io_run_local_work, but some task work managed to lock the context, it would leave things locked indefinitely. Fix that by passing the pointer in.

Patch 1 is a tiny cleanup to simplify things
Patch 2 is the fix

[...]

Applied, thanks!

[1/2] io_uring: use io_run_local_work_locked helper

      commit: 8de11cdc96bf58b324c59a28512eb9513fd02553
[2/2] io_uring: unlock if __io_run_local_work locked inside
      commit: b3026767e15b488860d4bbf1649d69612bab2c25

Best regards,

Signature · 1 line
Jens Axboe
maintainer
#6 lore raw
Quoted 8 lines · Jens Axboe
Quoted 7 lines · Dylan Yudaken

If locked was not set in __io_run_local_work, but some task work managed to lock the context, it would leave things locked indefinitely. Fix that by passing the pointer in.

Patch 1 is a tiny cleanup to simplify things
Patch 2 is the fix

Quoted 8 lines
Quoted 1 line

[...]

Applied, thanks!

[1/2] io_uring: use io_run_local_work_locked helper

      commit: 8de11cdc96bf58b324c59a28512eb9513fd02553
[2/2] io_uring: unlock if __io_run_local_work locked inside
      commit: b3026767e15b488860d4bbf1649d69612bab2c25

I made the WARN_ON() -> WARN_ON_ONCE() edit and added a small comment as well, while applying.

Signature · 1 line
Jens Axboe