Skip to contentSkip to conversation

fallback fix and task_work cleanup

PATCH In review Pavel Begunkov 2 participants 18 messages Jun 30, 2021

Subsystems (2)
lore
Share & tools
lore
#0 lore raw

cover fallback fix and task_work cleanup

Haven't see the bug in 1/3 in the wild, but should be possible, and so I'd like it for 5.14. Should it be stable? Perhaps, others may go 5.14 as well.

Pavel Begunkov (3):
io_uring: fix stuck fallback reqs
io_uring: simplify task_work func
io_uring: tweak io_req_task_work_add

 fs/io_uring.c | 131 +++++++++++++++++---------------------------------
 1 file changed, 45 insertions(+), 86 deletions(-)
Signature · 1 line
2.32.0
#1 lore raw

PATCH 1/3 io_uring: fix stuck fallback reqs

When task_work_add() fails, we use ->exit_task_work to queue the work. That will be run only in the cancellation path, which happens either when the ctx is dying or one of tasks with inflight requests is exiting or executing. There is a good chance that such a request would just get stuck in the list potentially hodling a file, all io_uring rsrc recycling or some other resources. Nothing terrible, it'll go away at some point, but we don't want to lock them up for longer than needed.

Replace that hand made ->exit_task_work with delayed_work + llist inspired by fput_many().

---
 fs/io_uring.c | 61 +++++++++++++++++----------------------------------
 1 file changed, 20 insertions(+), 41 deletions(-)
fs/io_uring.c +20 -41
index 92f85d4385a2..08ca835629ce 100644
@@ -465,7 +465,8 @@ struct io_ring_ctx {
 		struct mm_struct		*mm_account;
 
 		/* ctx exit and cancelation */
-		struct callback_head		*exit_task_work;
+		struct llist_head		fallback_llist;
+		struct delayed_work		fallback_work;
 		struct work_struct		exit_work;
 		struct list_head		tctx_list;
 		struct completion		ref_comp;
@@ -885,6 +886,8 @@ struct io_kiocb {
 	struct io_wq_work		work;
 	const struct cred		*creds;
 
+	struct llist_node		fallback_node;
+
 	/* store used ubuf, so we can prevent reloading */
 	struct io_mapped_ubuf		*imu;
 };
@@ -1100,6 +1103,8 @@ static void io_submit_flush_completions(struct io_ring_ctx *ctx);
 static bool io_poll_remove_waitqs(struct io_kiocb *req);
 static int io_req_prep_async(struct io_kiocb *req);
 
+static void io_fallback_req_func(struct work_struct *unused);
+
 static struct kmem_cache *req_cachep;
 
 static const struct file_operations io_uring_fops;
@@ -1231,6 +1236,7 @@ static struct io_ring_ctx *io_ring_ctx_alloc(struct io_uring_params *p)
 	INIT_LIST_HEAD(&ctx->tctx_list);
 	INIT_LIST_HEAD(&ctx->submit_state.comp.free_list);
 	INIT_LIST_HEAD(&ctx->locked_free_list);
+	INIT_DELAYED_WORK(&ctx->fallback_work, io_fallback_req_func);
 	return ctx;
 err:
 	kfree(ctx->dummy_ubuf);
@@ -2028,44 +2034,12 @@ static int io_req_task_work_add(struct io_kiocb *req)
 	return ret;
 }
 
-static bool io_run_task_work_head(struct callback_head **work_head)
-{
-	struct callback_head *work, *next;
-	bool executed = false;
-
-	do {
-		work = xchg(work_head, NULL);
-		if (!work)
-			break;
-
-		do {
-			next = work->next;
-			work->func(work);
-			work = next;
-			cond_resched();
-		} while (work);
-		executed = true;
-	} while (1);
-
-	return executed;
-}
-
-static void io_task_work_add_head(struct callback_head **work_head,
-				  struct callback_head *task_work)
-{
-	struct callback_head *head;
-
-	do {
-		head = READ_ONCE(*work_head);
-		task_work->next = head;
-	} while (cmpxchg(work_head, head, task_work) != head);
-}
-
 static void io_req_task_work_add_fallback(struct io_kiocb *req,
 					  task_work_func_t cb)
 {
 	init_task_work(&req->task_work, cb);
-	io_task_work_add_head(&req->ctx->exit_task_work, &req->task_work);
+	if (llist_add(&req->fallback_node, &req->ctx->fallback_llist))
+		schedule_delayed_work(&req->ctx->fallback_work, 1);
 }
 
 static void io_req_task_cancel(struct callback_head *cb)
@@ -2514,6 +2488,17 @@ static bool io_rw_should_reissue(struct io_kiocb *req)
 }
 #endif
 
+static void io_fallback_req_func(struct work_struct *work)
+{
+	struct io_ring_ctx *ctx = container_of(work, struct io_ring_ctx,
+						fallback_work.work);
+	struct llist_node *node = llist_del_all(&ctx->fallback_llist);
+	struct io_kiocb *req, *tmp;
+
+	llist_for_each_entry_safe(req, tmp, node, fallback_node)
+		req->task_work.func(&req->task_work);
+}
+
 static void __io_complete_rw(struct io_kiocb *req, long res, long res2,
 			     unsigned int issue_flags)
 {
@@ -8956,11 +8941,6 @@ static int io_unregister_personality(struct io_ring_ctx *ctx, unsigned id)
 	return -EINVAL;
 }
 
-static inline bool io_run_ctx_fallback(struct io_ring_ctx *ctx)
-{
-	return io_run_task_work_head(&ctx->exit_task_work);
-}
-
 struct io_tctx_exit {
 	struct callback_head		task_work;
 	struct completion		completion;
@@ -9225,7 +9205,6 @@ static void io_uring_try_cancel_requests(struct io_ring_ctx *ctx,
 		ret |= io_kill_timeouts(ctx, task, cancel_all);
 		if (task)
 			ret |= io_run_task_work();
-		ret |= io_run_ctx_fallback(ctx);
 		if (!ret)
 			break;
 		cond_resched();
Signature · 1 line
2.32.0
#2 lore raw

PATCH 2/3 io_uring: simplify task_work func

In reply to Pavel Begunkov

Since we don't really use req->task_work anymore, get rid of it together with the nasty ->func aliasing between ->io_task_work and ->task_work, and hide ->fallback_node inside of io_task_work.

Also, as task_work is gone now, replace the callback type from task_work_func_t to a function taking io_kiocb to avoid casting and simplify code.

---
 fs/io_uring.c | 72 ++++++++++++++++++++-------------------------------
 1 file changed, 28 insertions(+), 44 deletions(-)
fs/io_uring.c +28 -44
index 08ca835629ce..5fab427305e3 100644
@@ -808,9 +808,14 @@ struct async_poll {
 	struct io_poll_iocb	*double_poll;
 };
 
+typedef void (*io_req_tw_func_t)(struct io_kiocb *req);
+
 struct io_task_work {
-	struct io_wq_work_node	node;
-	task_work_func_t	func;
+	union {
+		struct io_wq_work_node	node;
+		struct llist_node	fallback_node;
+	};
+	io_req_tw_func_t		func;
 };
 
 enum {
@@ -876,18 +881,13 @@ struct io_kiocb {
 
 	/* used with ctx->iopoll_list with reads/writes */
 	struct list_head		inflight_entry;
-	union {
-		struct io_task_work	io_task_work;
-		struct callback_head	task_work;
-	};
+	struct io_task_work		io_task_work;
 	/* for polled requests, i.e. IORING_OP_POLL_ADD and async armed poll */
 	struct hlist_node		hash_node;
 	struct async_poll		*apoll;
 	struct io_wq_work		work;
 	const struct cred		*creds;
 
-	struct llist_node		fallback_node;
-
 	/* store used ubuf, so we can prevent reloading */
 	struct io_mapped_ubuf		*imu;
 };
@@ -1964,7 +1964,7 @@ static void tctx_task_work(struct callback_head *cb)
 				ctx = req->ctx;
 				percpu_ref_get(&ctx->refs);
 			}
-			req->task_work.func(&req->task_work);
+			req->io_task_work.func(req);
 			node = next;
 		}
 		if (wq_list_empty(&tctx->task_list)) {
@@ -2035,16 +2035,16 @@ static int io_req_task_work_add(struct io_kiocb *req)
 }
 
 static void io_req_task_work_add_fallback(struct io_kiocb *req,
-					  task_work_func_t cb)
+					  io_req_tw_func_t cb)
 {
-	init_task_work(&req->task_work, cb);
-	if (llist_add(&req->fallback_node, &req->ctx->fallback_llist))
+	req->io_task_work.func = cb;
+	if (llist_add(&req->io_task_work.fallback_node,
+		      &req->ctx->fallback_llist))
 		schedule_delayed_work(&req->ctx->fallback_work, 1);
 }
 
-static void io_req_task_cancel(struct callback_head *cb)
+static void io_req_task_cancel(struct io_kiocb *req)
 {
-	struct io_kiocb *req = container_of(cb, struct io_kiocb, task_work);
 	struct io_ring_ctx *ctx = req->ctx;
 
 	/* ctx is guaranteed to stay alive while we hold uring_lock */
@@ -2053,7 +2053,7 @@ static void io_req_task_cancel(struct callback_head *cb)
 	mutex_unlock(&ctx->uring_lock);
 }
 
-static void __io_req_task_submit(struct io_kiocb *req)
+static void io_req_task_submit(struct io_kiocb *req)
 {
 	struct io_ring_ctx *ctx = req->ctx;
 
@@ -2066,17 +2066,10 @@ static void __io_req_task_submit(struct io_kiocb *req)
 	mutex_unlock(&ctx->uring_lock);
 }
 
-static void io_req_task_submit(struct callback_head *cb)
-{
-	struct io_kiocb *req = container_of(cb, struct io_kiocb, task_work);
-
-	__io_req_task_submit(req);
-}
-
 static void io_req_task_queue_fail(struct io_kiocb *req, int ret)
 {
 	req->result = ret;
-	req->task_work.func = io_req_task_cancel;
+	req->io_task_work.func = io_req_task_cancel;
 
 	if (unlikely(io_req_task_work_add(req)))
 		io_req_task_work_add_fallback(req, io_req_task_cancel);
@@ -2084,7 +2077,7 @@ static void io_req_task_queue_fail(struct io_kiocb *req, int ret)
 
 static void io_req_task_queue(struct io_kiocb *req)
 {
-	req->task_work.func = io_req_task_submit;
+	req->io_task_work.func = io_req_task_submit;
 
 	if (unlikely(io_req_task_work_add(req)))
 		io_req_task_queue_fail(req, -ECANCELED);
@@ -2198,18 +2191,11 @@ static inline void io_put_req(struct io_kiocb *req)
 		io_free_req(req);
 }
 
-static void io_put_req_deferred_cb(struct callback_head *cb)
-{
-	struct io_kiocb *req = container_of(cb, struct io_kiocb, task_work);
-
-	io_free_req(req);
-}
-
 static void io_free_req_deferred(struct io_kiocb *req)
 {
-	req->task_work.func = io_put_req_deferred_cb;
+	req->io_task_work.func = io_free_req;
 	if (unlikely(io_req_task_work_add(req)))
-		io_req_task_work_add_fallback(req, io_put_req_deferred_cb);
+		io_req_task_work_add_fallback(req, io_free_req);
 }
 
 static inline void io_put_req_deferred(struct io_kiocb *req, int refs)
@@ -2495,8 +2481,8 @@ static void io_fallback_req_func(struct work_struct *work)
 	struct llist_node *node = llist_del_all(&ctx->fallback_llist);
 	struct io_kiocb *req, *tmp;
 
-	llist_for_each_entry_safe(req, tmp, node, fallback_node)
-		req->task_work.func(&req->task_work);
+	llist_for_each_entry_safe(req, tmp, node, io_task_work.fallback_node)
+		req->io_task_work.func(req);
 }
 
 static void __io_complete_rw(struct io_kiocb *req, long res, long res2,
@@ -4998,7 +4984,7 @@ struct io_poll_table {
 };
 
 static int __io_async_wake(struct io_kiocb *req, struct io_poll_iocb *poll,
-			   __poll_t mask, task_work_func_t func)
+			   __poll_t mask, io_req_tw_func_t func)
 {
 	int ret;
 
@@ -5011,7 +4997,7 @@ static int __io_async_wake(struct io_kiocb *req, struct io_poll_iocb *poll,
 	list_del_init(&poll->wait.entry);
 
 	req->result = mask;
-	req->task_work.func = func;
+	req->io_task_work.func = func;
 
 	/*
 	 * If this fails, then the task is exiting. When a task exits, the
@@ -5108,9 +5094,8 @@ static bool io_poll_complete(struct io_kiocb *req, __poll_t mask)
 	return !(flags & IORING_CQE_F_MORE);
 }
 
-static void io_poll_task_func(struct callback_head *cb)
+static void io_poll_task_func(struct io_kiocb *req)
 {
-	struct io_kiocb *req = container_of(cb, struct io_kiocb, task_work);
 	struct io_ring_ctx *ctx = req->ctx;
 	struct io_kiocb *nxt;
 
@@ -5132,7 +5117,7 @@ static void io_poll_task_func(struct callback_head *cb)
 		if (done) {
 			nxt = io_put_req_find_next(req);
 			if (nxt)
-				__io_req_task_submit(nxt);
+				io_req_task_submit(nxt);
 		}
 	}
 }
@@ -5241,9 +5226,8 @@ static void io_async_queue_proc(struct file *file, struct wait_queue_head *head,
 	__io_queue_proc(&apoll->poll, pt, head, &apoll->double_poll);
 }
 
-static void io_async_task_func(struct callback_head *cb)
+static void io_async_task_func(struct io_kiocb *req)
 {
-	struct io_kiocb *req = container_of(cb, struct io_kiocb, task_work);
 	struct async_poll *apoll = req->apoll;
 	struct io_ring_ctx *ctx = req->ctx;
 
@@ -5259,7 +5243,7 @@ static void io_async_task_func(struct callback_head *cb)
 	spin_unlock_irq(&ctx->completion_lock);
 
 	if (!READ_ONCE(apoll->poll.canceled))
-		__io_req_task_submit(req);
+		io_req_task_submit(req);
 	else
 		io_req_complete_failed(req, -ECANCELED);
 }
@@ -9006,7 +8990,7 @@ static void io_ring_exit_work(struct work_struct *work)
 	/*
 	 * Some may use context even when all refs and requests have been put,
 	 * and they are free to do so while still holding uring_lock or
-	 * completion_lock, see __io_req_task_submit(). Apart from other work,
+	 * completion_lock, see io_req_task_submit(). Apart from other work,
 	 * this lock/unlock section also waits them to finish.
 	 */
 	mutex_lock(&ctx->uring_lock);
Signature · 1 line
2.32.0
#3 lore raw

PATCH 3/3 io_uring: tweak io_req_task_work_add

In reply to Pavel Begunkov

Whenever possible we don't want to fallback a request. task_work_add() will be fine if the task is exiting, so don't check for PF_EXITING, there is anyway only a relatively small gap between setting the flag and doing the final task_work_run().

Also add likely for the hot path.

---
 fs/io_uring.c | 8 ++------
 1 file changed, 2 insertions(+), 6 deletions(-)
fs/io_uring.c +2 -6
index 5fab427305e3..1893a4229dbb 100644
@@ -1990,9 +1990,6 @@ static int io_req_task_work_add(struct io_kiocb *req)
 	unsigned long flags;
 	int ret = 0;
 
-	if (unlikely(tsk->flags & PF_EXITING))
-		return -ESRCH;
-
 	WARN_ON_ONCE(!tctx);
 
 	spin_lock_irqsave(&tctx->task_lock, flags);
@@ -2000,7 +1997,7 @@ static int io_req_task_work_add(struct io_kiocb *req)
 	spin_unlock_irqrestore(&tctx->task_lock, flags);
 
 	/* task_work already pending, we're done */
-	if (test_bit(0, &tctx->task_state) ||
+	if (likely(test_bit(0, &tctx->task_state)) ||
 	    test_and_set_bit(0, &tctx->task_state))
 		return 0;
 
@@ -2011,8 +2008,7 @@ static int io_req_task_work_add(struct io_kiocb *req)
 	 * will do the job.
 	 */
 	notify = (req->ctx->flags & IORING_SETUP_SQPOLL) ? TWA_NONE : TWA_SIGNAL;
-
-	if (!task_work_add(tsk, &tctx->task_work, notify)) {
+	if (likely(!task_work_add(tsk, &tctx->task_work, notify))) {
 		wake_up_process(tsk);
 		return 0;
 	}
Signature · 1 line
2.32.0
maintainer
#4 lore raw

Re: [PATCH 3/3] io_uring: tweak io_req_task_work_add

Quoted 6 lines · Pavel Begunkov

Whenever possible we don't want to fallback a request. task_work_add() will be fine if the task is exiting, so don't check for PF_EXITING, there is anyway only a relatively small gap between setting the flag and doing the final task_work_run().

Also add likely for the hot path.

I'm not a huge fan of likely/unlikely, and in particular constructs like:

fs/io_uring.c static int io_req_task_work_add(struct io_kiocb *req) · line 2002

Quoted 4 lines
fs/io_uring.c +1 -1
-	if (test_bit(0, &tctx->task_state) ||
+	if (likely(test_bit(0, &tctx->task_state)) ||
 	    test_and_set_bit(0, &tctx->task_state))
 		return 0;

where the state is combined. In any case, it should be a separate change. If there's an "Also" paragraph in a patch, then that's also usually a good clue that that particular change should've been separate :-)

Signature · 1 line
Jens Axboe
maintainer
#16 lore raw
In reply to Pavel Begunkov
Quoted 11 lines · Pavel Begunkov

Haven't see the bug in 1/3 in the wild, but should be possible, and so I'd like it for 5.14. Should it be stable? Perhaps, others may go 5.14 as well.

Pavel Begunkov (3):
io_uring: fix stuck fallback reqs
io_uring: simplify task_work func
io_uring: tweak io_req_task_work_add

 fs/io_uring.c | 131 +++++++++++++++++---------------------------------
 1 file changed, 45 insertions(+), 86 deletions(-)

Applied 1-2, thanks.

Signature · 1 line
Jens Axboe
#5 lore raw

Re: [PATCH 3/3] io_uring: tweak io_req_task_work_add

In reply to Jens Axboe
Quoted 10 lines · Jens Axboe
Quoted 6 lines · Pavel Begunkov

Whenever possible we don't want to fallback a request. task_work_add() will be fine if the task is exiting, so don't check for PF_EXITING, there is anyway only a relatively small gap between setting the flag and doing the final task_work_run().

Also add likely for the hot path.

I'm not a huge fan of likely/unlikely, and in particular constructs like:

Quoted 9 lines

fs/io_uring.c static int io_req_task_work_add(struct io_kiocb *req) · line 2002

Quoted 4 lines
fs/io_uring.c +1 -1
-	if (test_bit(0, &tctx->task_state) ||
+	if (likely(test_bit(0, &tctx->task_state)) ||
 	    test_and_set_bit(0, &tctx->task_state))
 		return 0;

where the state is combined. In any case, it should be a separate change. If there's an "Also" paragraph in a patch, then that's also usually a good clue that that particular change should've been separate :-)

Not sure what's wrong with likely above, but how about drop this one then?

Signature · 1 line
Pavel Begunkov
#17 lore raw
In reply to Jens Axboe
Quoted 10 lines · Jens Axboe
Quoted 9 lines · Pavel Begunkov

Haven't see the bug in 1/3 in the wild, but should be possible, and so I'd like it for 5.14. Should it be stable? Perhaps, others may go 5.14 as well.

Pavel Begunkov (3):
io_uring: fix stuck fallback reqs
io_uring: simplify task_work func
io_uring: tweak io_req_task_work_add

Quoted 4 lines
Quoted 2 lines
 fs/io_uring.c | 131 +++++++++++++++++---------------------------------
 1 file changed, 45 insertions(+), 86 deletions(-)

Applied 1-2, thanks.

Ah, you already did drop, thanks

Signature · 1 line
Pavel Begunkov
maintainer
#6 lore raw

Re: [PATCH 3/3] io_uring: tweak io_req_task_work_add

In reply to Pavel Begunkov
Quoted 21 lines · Pavel Begunkov
Quoted 10 lines · Jens Axboe
Quoted 6 lines · Pavel Begunkov

Whenever possible we don't want to fallback a request. task_work_add() will be fine if the task is exiting, so don't check for PF_EXITING, there is anyway only a relatively small gap between setting the flag and doing the final task_work_run().

Also add likely for the hot path.

I'm not a huge fan of likely/unlikely, and in particular constructs like:

Quoted 9 lines

fs/io_uring.c static int io_req_task_work_add(struct io_kiocb *req) · line 2002

Quoted 4 lines
fs/io_uring.c +1 -1
-	if (test_bit(0, &tctx->task_state) ||
+	if (likely(test_bit(0, &tctx->task_state)) ||
 	    test_and_set_bit(0, &tctx->task_state))
 		return 0;

where the state is combined. In any case, it should be a separate change. If there's an "Also" paragraph in a patch, then that's also usually a good clue that that particular change should've been separate :-)

Quoted 2 lines

Not sure what's wrong with likely above, but how about drop this one then?

Yep I did - we can do the exiting change separately, the commit message just needs to be clarified a bit on why it's ok to do now. And that last sentence dropped, of course.

Signature · 1 line
Jens Axboe
#7 lore raw

Re: [PATCH 3/3] io_uring: tweak io_req_task_work_add

Quoted 22 lines · Jens Axboe
Quoted 21 lines · Pavel Begunkov
Quoted 19 lines · Jens Axboe
Quoted 6 lines · Pavel Begunkov

Whenever possible we don't want to fallback a request. task_work_add() will be fine if the task is exiting, so don't check for PF_EXITING, there is anyway only a relatively small gap between setting the flag and doing the final task_work_run().

Also add likely for the hot path.

I'm not a huge fan of likely/unlikely, and in particular constructs like:

fs/io_uring.c static int io_req_task_work_add(struct io_kiocb *req) · line 2002

Quoted 4 lines
fs/io_uring.c +1 -1
-	if (test_bit(0, &tctx->task_state) ||
+	if (likely(test_bit(0, &tctx->task_state)) ||
 	    test_and_set_bit(0, &tctx->task_state))
 		return 0;

where the state is combined. In any case, it should be a separate change. If there's an "Also" paragraph in a patch, then that's also usually a good clue that that particular change should've been separate :-)

Quoted 4 lines
Quoted 2 lines

Not sure what's wrong with likely above, but how about drop this one then?

Yep I did - we can do the exiting change separately, the commit message

I think 1-2 is good enough for 5.14, I'll just send it for-next

Quoted 1 line

just needs to be clarified a bit on why it's ok to do now. And that

It should have been ok to do before those 2 patches, but haven't tracked where it lost actuality.

Signature · 1 line
Pavel Begunkov
maintainer
#8 lore raw

Re: [PATCH 3/3] io_uring: tweak io_req_task_work_add

Quoted 26 lines · Pavel Begunkov
Quoted 25 lines · Jens Axboe
Quoted 23 lines · Pavel Begunkov
Quoted 10 lines · Jens Axboe
Quoted 6 lines · Pavel Begunkov

Whenever possible we don't want to fallback a request. task_work_add() will be fine if the task is exiting, so don't check for PF_EXITING, there is anyway only a relatively small gap between setting the flag and doing the final task_work_run().

Also add likely for the hot path.

I'm not a huge fan of likely/unlikely, and in particular constructs like:

Quoted 9 lines

fs/io_uring.c static int io_req_task_work_add(struct io_kiocb *req) · line 2002

Quoted 4 lines
fs/io_uring.c +1 -1
-	if (test_bit(0, &tctx->task_state) ||
+	if (likely(test_bit(0, &tctx->task_state)) ||
 	    test_and_set_bit(0, &tctx->task_state))
 		return 0;

where the state is combined. In any case, it should be a separate change. If there's an "Also" paragraph in a patch, then that's also usually a good clue that that particular change should've been separate :-)

Not sure what's wrong with likely above, but how about drop this one then?

Quoted 8 lines
Quoted 1 line

Yep I did - we can do the exiting change separately, the commit message

I think 1-2 is good enough for 5.14, I'll just send it for-next

Quoted 1 line

just needs to be clarified a bit on why it's ok to do now. And that

It should have been ok to do before those 2 patches, but haven't tracked where it lost actuality.

Right, I was thinking it was related to the swapping of the signal exit and task work run ordering. But didn't look that far yet...

Signature · 1 line
Jens Axboe
maintainer
#9 lore raw

Re: [PATCH 3/3] io_uring: tweak io_req_task_work_add

Quoted 31 lines · Jens Axboe
Quoted 30 lines · Pavel Begunkov
Quoted 22 lines · Jens Axboe
Quoted 21 lines · Pavel Begunkov
Quoted 19 lines · Jens Axboe
Quoted 6 lines · Pavel Begunkov

Whenever possible we don't want to fallback a request. task_work_add() will be fine if the task is exiting, so don't check for PF_EXITING, there is anyway only a relatively small gap between setting the flag and doing the final task_work_run().

Also add likely for the hot path.

I'm not a huge fan of likely/unlikely, and in particular constructs like:

fs/io_uring.c static int io_req_task_work_add(struct io_kiocb *req) · line 2002

Quoted 4 lines
fs/io_uring.c +1 -1
-	if (test_bit(0, &tctx->task_state) ||
+	if (likely(test_bit(0, &tctx->task_state)) ||
 	    test_and_set_bit(0, &tctx->task_state))
 		return 0;

where the state is combined. In any case, it should be a separate change. If there's an "Also" paragraph in a patch, then that's also usually a good clue that that particular change should've been separate :-)

Quoted 4 lines
Quoted 2 lines

Not sure what's wrong with likely above, but how about drop this one then?

Yep I did - we can do the exiting change separately, the commit message

I think 1-2 is good enough for 5.14, I'll just send it for-next

Quoted 7 lines
Quoted 4 lines
Quoted 1 line

just needs to be clarified a bit on why it's ok to do now. And that

It should have been ok to do before those 2 patches, but haven't tracked where it lost actuality.

Right, I was thinking it was related to the swapping of the signal exit and task work run ordering. But didn't look that far yet...

BTW, in usual testing, even just the one hunk removing the exit check seems to result in quite a lot of memory leaks running test/poll-mshot-update. So something is funky with the patch.

Signature · 1 line
Jens Axboe
#14 lore raw

Re: [PATCH 3/3] io_uring: tweak io_req_task_work_add

In reply to Jens Axboe
Quoted 31 lines · Jens Axboe
Quoted 30 lines · Pavel Begunkov
Quoted 22 lines · Jens Axboe
Quoted 21 lines · Pavel Begunkov
Quoted 19 lines · Jens Axboe
Quoted 6 lines · Pavel Begunkov

Whenever possible we don't want to fallback a request. task_work_add() will be fine if the task is exiting, so don't check for PF_EXITING, there is anyway only a relatively small gap between setting the flag and doing the final task_work_run().

Also add likely for the hot path.

I'm not a huge fan of likely/unlikely, and in particular constructs like:

fs/io_uring.c static int io_req_task_work_add(struct io_kiocb *req) · line 2002

Quoted 4 lines
fs/io_uring.c +1 -1
-	if (test_bit(0, &tctx->task_state) ||
+	if (likely(test_bit(0, &tctx->task_state)) ||
 	    test_and_set_bit(0, &tctx->task_state))
 		return 0;

where the state is combined. In any case, it should be a separate change. If there's an "Also" paragraph in a patch, then that's also usually a good clue that that particular change should've been separate :-)

Quoted 4 lines
Quoted 2 lines

Not sure what's wrong with likely above, but how about drop this one then?

Yep I did - we can do the exiting change separately, the commit message

I think 1-2 is good enough for 5.14, I'll just send it for-next

Quoted 7 lines
Quoted 4 lines
Quoted 1 line

just needs to be clarified a bit on why it's ok to do now. And that

It should have been ok to do before those 2 patches, but haven't tracked where it lost actuality.

Right, I was thinking it was related to the swapping of the signal exit and task work run ordering. But didn't look that far yet...

commit 6200b0ae4ea28a4bfd8eb434e33e6201b7a6a282

Author: Jens Axboe <[email protected]>
Date:   Sun Sep 13 14:38:30 2020 -0600

io_uring: don't run task work on an exiting task

Came from there, times where there still was io_uring_flush() and no from do_exit() cancellations

Signature · 1 line
Pavel Begunkov
maintainer
#15 lore raw

Re: [PATCH 3/3] io_uring: tweak io_req_task_work_add

Quoted 40 lines · Pavel Begunkov
Quoted 31 lines · Jens Axboe
Quoted 30 lines · Pavel Begunkov
Quoted 22 lines · Jens Axboe
Quoted 21 lines · Pavel Begunkov
Quoted 19 lines · Jens Axboe
Quoted 6 lines · Pavel Begunkov

Whenever possible we don't want to fallback a request. task_work_add() will be fine if the task is exiting, so don't check for PF_EXITING, there is anyway only a relatively small gap between setting the flag and doing the final task_work_run().

Also add likely for the hot path.

I'm not a huge fan of likely/unlikely, and in particular constructs like:

Review

Quoted 4 lines
(quoted diff) +1 -1
-	if (test_bit(0, &tctx->task_state) ||
+	if (likely(test_bit(0, &tctx->task_state)) ||
 	    test_and_set_bit(0, &tctx->task_state))
 		return 0;

where the state is combined. In any case, it should be a separate change. If there's an "Also" paragraph in a patch, then that's also usually a good clue that that particular change should've been separate :-)

Quoted 4 lines
Quoted 2 lines

Not sure what's wrong with likely above, but how about drop this one then?

Yep I did - we can do the exiting change separately, the commit message

I think 1-2 is good enough for 5.14, I'll just send it for-next

Quoted 7 lines
Quoted 4 lines
Quoted 1 line

just needs to be clarified a bit on why it's ok to do now. And that

It should have been ok to do before those 2 patches, but haven't tracked where it lost actuality.

Right, I was thinking it was related to the swapping of the signal exit and task work run ordering. But didn't look that far yet...

Quoted 8 lines

commit 6200b0ae4ea28a4bfd8eb434e33e6201b7a6a282

Author: Jens Axboe <[email protected]>
Date:   Sun Sep 13 14:38:30 2020 -0600

io_uring: don't run task work on an exiting task

Came from there, times where there still was io_uring_flush() and no from do_exit() cancellations

Ah yes, that makes sense.

Signature · 1 line
Jens Axboe
#10 lore raw

Re: [PATCH 3/3] io_uring: tweak io_req_task_work_add

In reply to Jens Axboe
Quoted 37 lines · Jens Axboe
Quoted 36 lines · Jens Axboe
Quoted 34 lines · Pavel Begunkov
Quoted 22 lines · Jens Axboe
Quoted 21 lines · Pavel Begunkov
Quoted 19 lines · Jens Axboe
Quoted 6 lines · Pavel Begunkov

Whenever possible we don't want to fallback a request. task_work_add() will be fine if the task is exiting, so don't check for PF_EXITING, there is anyway only a relatively small gap between setting the flag and doing the final task_work_run().

Also add likely for the hot path.

I'm not a huge fan of likely/unlikely, and in particular constructs like:

Review

Quoted 4 lines
(quoted diff) +1 -1
-	if (test_bit(0, &tctx->task_state) ||
+	if (likely(test_bit(0, &tctx->task_state)) ||
 	    test_and_set_bit(0, &tctx->task_state))
 		return 0;

where the state is combined. In any case, it should be a separate change. If there's an "Also" paragraph in a patch, then that's also usually a good clue that that particular change should've been separate :-)

Quoted 4 lines
Quoted 2 lines

Not sure what's wrong with likely above, but how about drop this one then?

Yep I did - we can do the exiting change separately, the commit message

I think 1-2 is good enough for 5.14, I'll just send it for-next

Quoted 1 line

just needs to be clarified a bit on why it's ok to do now. And that

It should have been ok to do before those 2 patches, but haven't tracked where it lost actuality.

Quoted 6 lines
Quoted 2 lines

Right, I was thinking it was related to the swapping of the signal exit and task work run ordering. But didn't look that far yet...

BTW, in usual testing, even just the one hunk removing the exit check seems to result in quite a lot of memory leaks running test/poll-mshot-update. So something is funky with the patch.

Oh, have no idea why yet. Apparently, I commented out the test as it was always failing, and forgot to return it back... Will take a note

Signature · 1 line
Pavel Begunkov
#11 lore raw

Re: [PATCH 3/3] io_uring: tweak io_req_task_work_add

In reply to Jens Axboe
Quoted 37 lines · Jens Axboe
Quoted 36 lines · Jens Axboe
Quoted 34 lines · Pavel Begunkov
Quoted 22 lines · Jens Axboe
Quoted 21 lines · Pavel Begunkov
Quoted 19 lines · Jens Axboe
Quoted 6 lines · Pavel Begunkov

Whenever possible we don't want to fallback a request. task_work_add() will be fine if the task is exiting, so don't check for PF_EXITING, there is anyway only a relatively small gap between setting the flag and doing the final task_work_run().

Also add likely for the hot path.

I'm not a huge fan of likely/unlikely, and in particular constructs like:

Review

Quoted 4 lines
(quoted diff) +1 -1
-	if (test_bit(0, &tctx->task_state) ||
+	if (likely(test_bit(0, &tctx->task_state)) ||
 	    test_and_set_bit(0, &tctx->task_state))
 		return 0;

where the state is combined. In any case, it should be a separate change. If there's an "Also" paragraph in a patch, then that's also usually a good clue that that particular change should've been separate :-)

Quoted 4 lines
Quoted 2 lines

Not sure what's wrong with likely above, but how about drop this one then?

Yep I did - we can do the exiting change separately, the commit message

I think 1-2 is good enough for 5.14, I'll just send it for-next

Quoted 1 line

just needs to be clarified a bit on why it's ok to do now. And that

It should have been ok to do before those 2 patches, but haven't tracked where it lost actuality.

Quoted 6 lines
Quoted 2 lines

Right, I was thinking it was related to the swapping of the signal exit and task work run ordering. But didn't look that far yet...

BTW, in usual testing, even just the one hunk removing the exit check seems to result in quite a lot of memory leaks running test/poll-mshot-update. So something is funky with the patch.

I guess you're positive that patches 1-2 have nothing to do with that. Right?

Signature · 1 line
Pavel Begunkov
maintainer
#12 lore raw

Re: [PATCH 3/3] io_uring: tweak io_req_task_work_add

Quoted 41 lines · Pavel Begunkov
Quoted 40 lines · Jens Axboe
Quoted 38 lines · Jens Axboe
Quoted 26 lines · Pavel Begunkov
Quoted 25 lines · Jens Axboe
Quoted 23 lines · Pavel Begunkov
Quoted 10 lines · Jens Axboe
Quoted 6 lines · Pavel Begunkov

Whenever possible we don't want to fallback a request. task_work_add() will be fine if the task is exiting, so don't check for PF_EXITING, there is anyway only a relatively small gap between setting the flag and doing the final task_work_run().

Also add likely for the hot path.

I'm not a huge fan of likely/unlikely, and in particular constructs like:

Quoted 9 lines

Review

Quoted 4 lines
(quoted diff) +1 -1
-	if (test_bit(0, &tctx->task_state) ||
+	if (likely(test_bit(0, &tctx->task_state)) ||
 	    test_and_set_bit(0, &tctx->task_state))
 		return 0;

where the state is combined. In any case, it should be a separate change. If there's an "Also" paragraph in a patch, then that's also usually a good clue that that particular change should've been separate :-)

Not sure what's wrong with likely above, but how about drop this one then?

Quoted 8 lines
Quoted 1 line

Yep I did - we can do the exiting change separately, the commit message

I think 1-2 is good enough for 5.14, I'll just send it for-next

Quoted 1 line

just needs to be clarified a bit on why it's ok to do now. And that

It should have been ok to do before those 2 patches, but haven't tracked where it lost actuality.

Right, I was thinking it was related to the swapping of the signal exit and task work run ordering. But didn't look that far yet...

Quoted 6 lines
Quoted 3 lines

BTW, in usual testing, even just the one hunk removing the exit check seems to result in quite a lot of memory leaks running test/poll-mshot-update. So something is funky with the patch.

I guess you're positive that patches 1-2 have nothing to do with that. Right?

I double checked, and seems fine with those two alone. Ran the test twice, saw massive amounts of leaks with patches 1-3, and none with patches 1-2 only.

Signature · 1 line
Jens Axboe
#13 lore raw

Re: [PATCH 3/3] io_uring: tweak io_req_task_work_add

Quoted 46 lines · Jens Axboe
Quoted 45 lines · Pavel Begunkov
Quoted 43 lines · Jens Axboe
Quoted 31 lines · Jens Axboe
Quoted 30 lines · Pavel Begunkov
Quoted 22 lines · Jens Axboe
Quoted 21 lines · Pavel Begunkov
Quoted 19 lines · Jens Axboe
Quoted 6 lines · Pavel Begunkov

Whenever possible we don't want to fallback a request. task_work_add() will be fine if the task is exiting, so don't check for PF_EXITING, there is anyway only a relatively small gap between setting the flag and doing the final task_work_run().

Also add likely for the hot path.

I'm not a huge fan of likely/unlikely, and in particular constructs like:

Review

Quoted 4 lines
(quoted diff) +1 -1
-	if (test_bit(0, &tctx->task_state) ||
+	if (likely(test_bit(0, &tctx->task_state)) ||
 	    test_and_set_bit(0, &tctx->task_state))
 		return 0;

where the state is combined. In any case, it should be a separate change. If there's an "Also" paragraph in a patch, then that's also usually a good clue that that particular change should've been separate :-)

Quoted 4 lines
Quoted 2 lines

Not sure what's wrong with likely above, but how about drop this one then?

Yep I did - we can do the exiting change separately, the commit message

I think 1-2 is good enough for 5.14, I'll just send it for-next

Quoted 7 lines
Quoted 4 lines
Quoted 1 line

just needs to be clarified a bit on why it's ok to do now. And that

It should have been ok to do before those 2 patches, but haven't tracked where it lost actuality.

Right, I was thinking it was related to the swapping of the signal exit and task work run ordering. But didn't look that far yet...

BTW, in usual testing, even just the one hunk removing the exit check seems to result in quite a lot of memory leaks running test/poll-mshot-update. So something is funky with the patch.

Quoted 6 lines
Quoted 2 lines

I guess you're positive that patches 1-2 have nothing to do with that. Right?

I double checked, and seems fine with those two alone. Ran the test twice, saw massive amounts of leaks with patches 1-3, and none with patches 1-2 only.

I think there is a problem with the failing path of io_req_task_work_add(), the removing back part. Will send a patch tomorrow, but not able to test.

Signature · 1 line
Pavel Begunkov