Skip to contentSkip to conversation

clean up req free and CQ locking

PATCH Applied Pavel Begunkov 2 participants 14 messages Jun 23, 2023 io_uring

lore
Share & tools
lore
#0 lore raw

cover clean up req free and CQ locking

Patches 1-5 are cleaning how we free requests.
Patches 7-11 brush CQ / ->completion_lock locking

Pavel Begunkov (11):
io_uring: open code io_put_req_find_next
io_uring: remove io_free_req_tw
io_uring: inline io_dismantle_req()
io_uring: move io_clean_op()
io_uring: don't batch task put on reqs free
io_uring: remove IOU_F_TWQ_FORCE_NORMAL
io_uring: kill io_cq_unlock()
io_uring: fix acquire/release annotations
io_uring: inline __io_cq_unlock
io_uring: make io_cq_unlock_post static
io_uring: merge conditional unlock flush helpers

 io_uring/io_uring.c | 223 ++++++++++++++++----------------------------
 io_uring/io_uring.h |   7 +-
 2 files changed, 80 insertions(+), 150 deletions(-)
Signature · 1 line
2.40.0
#1 lore raw

PATCH 1/11 io_uring: open code io_put_req_find_next

There is only one user of io_put_req_find_next() and it doesn't make much sense to have it. Open code the function.

---
 io_uring/io_uring.c | 25 +++++++------------------
 1 file changed, 7 insertions(+), 18 deletions(-)
io_uring/io_uring.c +7 -18
index ae4cb3c4e730..b488a03ba009 100644
@@ -1586,22 +1586,6 @@ static void __io_submit_flush_completions(struct io_ring_ctx *ctx)
 	}
 }
 
-/*
- * Drop reference to request, return next in chain (if there is one) if this
- * was the last reference to this request.
- */
-static inline struct io_kiocb *io_put_req_find_next(struct io_kiocb *req)
-{
-	struct io_kiocb *nxt = NULL;
-
-	if (req_ref_put_and_test(req)) {
-		if (unlikely(req->flags & IO_REQ_LINK_FLAGS))
-			nxt = io_req_find_next(req);
-		io_free_req(req);
-	}
-	return nxt;
-}
-
 static unsigned io_cqring_events(struct io_ring_ctx *ctx)
 {
 	/* See comment at the top of this file */
@@ -1954,9 +1938,14 @@ int io_poll_issue(struct io_kiocb *req, struct io_tw_state *ts)
 struct io_wq_work *io_wq_free_work(struct io_wq_work *work)
 {
 	struct io_kiocb *req = container_of(work, struct io_kiocb, work);
+	struct io_kiocb *nxt = NULL;
 
-	req = io_put_req_find_next(req);
-	return req ? &req->work : NULL;
+	if (req_ref_put_and_test(req)) {
+		if (req->flags & IO_REQ_LINK_FLAGS)
+			nxt = io_req_find_next(req);
+		io_free_req(req);
+	}
+	return nxt ? &nxt->work : NULL;
 }
 
 void io_wq_submit_work(struct io_wq_work *work)
Signature · 1 line
2.40.0
#2 lore raw

PATCH 2/11 io_uring: remove io_free_req_tw

In reply to Pavel Begunkov

Request completion is a very hot path in general, but there are 3 places that can be doing it: io_free_batch_list(), io_req_complete_post() and io_free_req_tw().

io_free_req_tw() is used rather marginally and we don't care about it. Killing it can help to clean up and optimise the left two, do that by replacing it with io_req_task_complete().

There are two things to consider:
1) io_free_req() is called when all refs are put, so we need to reinit references. The easiest way to do that is to clear REQ_F_REFCOUNT.
2) We also don't need a cqe from it, so silence it with REQ_F_CQE_SKIP.

---
 io_uring/io_uring.c | 23 +++++------------------
 1 file changed, 5 insertions(+), 18 deletions(-)
io_uring/io_uring.c +5 -18
index b488a03ba009..43805d2621f5 100644
@@ -1121,26 +1121,13 @@ static inline void io_dismantle_req(struct io_kiocb *req)
 		io_put_file(req->file);
 }
 
-static __cold void io_free_req_tw(struct io_kiocb *req, struct io_tw_state *ts)
-{
-	struct io_ring_ctx *ctx = req->ctx;
-
-	if (req->rsrc_node) {
-		io_tw_lock(ctx, ts);
-		io_put_rsrc_node(ctx, req->rsrc_node);
-	}
-	io_dismantle_req(req);
-	io_put_task_remote(req->task, 1);
-
-	spin_lock(&ctx->completion_lock);
-	wq_list_add_head(&req->comp_list, &ctx->locked_free_list);
-	ctx->locked_free_nr++;
-	spin_unlock(&ctx->completion_lock);
-}
-
 __cold void io_free_req(struct io_kiocb *req)
 {
-	req->io_task_work.func = io_free_req_tw;
+	/* refs were already put, restore them for io_req_task_complete() */
+	req->flags &= ~REQ_F_REFCOUNT;
+	/* we only want to free it, don't post CQEs */
+	req->flags |= REQ_F_CQE_SKIP;
+	req->io_task_work.func = io_req_task_complete;
 	io_req_task_work_add(req);
 }
 
Signature · 1 line
2.40.0
#3 lore raw

PATCH 3/11 io_uring: inline io_dismantle_req()

In reply to Pavel Begunkov

io_dismantle_req() is only used in __io_req_complete_post(), open code it there.

---
 io_uring/io_uring.c | 17 +++++------------
 1 file changed, 5 insertions(+), 12 deletions(-)
io_uring/io_uring.c +5 -12
index 43805d2621f5..50fe345bdced 100644
@@ -146,7 +146,6 @@ static bool io_uring_try_cancel_requests(struct io_ring_ctx *ctx,
 					 struct task_struct *task,
 					 bool cancel_all);
 
-static void io_dismantle_req(struct io_kiocb *req);
 static void io_clean_op(struct io_kiocb *req);
 static void io_queue_sqe(struct io_kiocb *req);
 static void io_move_task_work_from_local(struct io_ring_ctx *ctx);
@@ -991,7 +990,11 @@ static void __io_req_complete_post(struct io_kiocb *req, unsigned issue_flags)
 			}
 		}
 		io_put_kbuf_comp(req);
-		io_dismantle_req(req);
+		if (unlikely(req->flags & IO_REQ_CLEAN_FLAGS))
+			io_clean_op(req);
+		if (!(req->flags & REQ_F_FIXED_FILE))
+			io_put_file(req->file);
+
 		rsrc_node = req->rsrc_node;
 		/*
 		 * Selected buffer deallocation in io_clean_op() assumes that
@@ -1111,16 +1114,6 @@ __cold bool __io_alloc_req_refill(struct io_ring_ctx *ctx)
 	return true;
 }
 
-static inline void io_dismantle_req(struct io_kiocb *req)
-{
-	unsigned int flags = req->flags;
-
-	if (unlikely(flags & IO_REQ_CLEAN_FLAGS))
-		io_clean_op(req);
-	if (!(flags & REQ_F_FIXED_FILE))
-		io_put_file(req->file);
-}
-
 __cold void io_free_req(struct io_kiocb *req)
 {
 	/* refs were already put, restore them for io_req_task_complete() */
Signature · 1 line
2.40.0
#4 lore raw

PATCH 4/11 io_uring: move io_clean_op()

In reply to Pavel Begunkov

Move io_clean_op() up in the source file and remove the forward declaration, as the function doesn't have tricky dependencies anymore.

---
 io_uring/io_uring.c | 67 ++++++++++++++++++++++-----------------------
 1 file changed, 33 insertions(+), 34 deletions(-)
io_uring/io_uring.c +33 -34
index 50fe345bdced..4d8613996644 100644
@@ -146,7 +146,6 @@ static bool io_uring_try_cancel_requests(struct io_ring_ctx *ctx,
 					 struct task_struct *task,
 					 bool cancel_all);
 
-static void io_clean_op(struct io_kiocb *req);
 static void io_queue_sqe(struct io_kiocb *req);
 static void io_move_task_work_from_local(struct io_ring_ctx *ctx);
 static void __io_submit_flush_completions(struct io_ring_ctx *ctx);
@@ -367,6 +366,39 @@ static bool req_need_defer(struct io_kiocb *req, u32 seq)
 	return false;
 }
 
+static void io_clean_op(struct io_kiocb *req)
+{
+	if (req->flags & REQ_F_BUFFER_SELECTED) {
+		spin_lock(&req->ctx->completion_lock);
+		io_put_kbuf_comp(req);
+		spin_unlock(&req->ctx->completion_lock);
+	}
+
+	if (req->flags & REQ_F_NEED_CLEANUP) {
+		const struct io_cold_def *def = &io_cold_defs[req->opcode];
+
+		if (def->cleanup)
+			def->cleanup(req);
+	}
+	if ((req->flags & REQ_F_POLLED) && req->apoll) {
+		kfree(req->apoll->double_poll);
+		kfree(req->apoll);
+		req->apoll = NULL;
+	}
+	if (req->flags & REQ_F_INFLIGHT) {
+		struct io_uring_task *tctx = req->task->io_uring;
+
+		atomic_dec(&tctx->inflight_tracked);
+	}
+	if (req->flags & REQ_F_CREDS)
+		put_cred(req->creds);
+	if (req->flags & REQ_F_ASYNC_DATA) {
+		kfree(req->async_data);
+		req->async_data = NULL;
+	}
+	req->flags &= ~IO_REQ_CLEAN_FLAGS;
+}
+
 static inline void io_req_track_inflight(struct io_kiocb *req)
 {
 	if (!(req->flags & REQ_F_INFLIGHT)) {
@@ -1823,39 +1855,6 @@ static __cold void io_drain_req(struct io_kiocb *req)
 	spin_unlock(&ctx->completion_lock);
 }
 
-static void io_clean_op(struct io_kiocb *req)
-{
-	if (req->flags & REQ_F_BUFFER_SELECTED) {
-		spin_lock(&req->ctx->completion_lock);
-		io_put_kbuf_comp(req);
-		spin_unlock(&req->ctx->completion_lock);
-	}
-
-	if (req->flags & REQ_F_NEED_CLEANUP) {
-		const struct io_cold_def *def = &io_cold_defs[req->opcode];
-
-		if (def->cleanup)
-			def->cleanup(req);
-	}
-	if ((req->flags & REQ_F_POLLED) && req->apoll) {
-		kfree(req->apoll->double_poll);
-		kfree(req->apoll);
-		req->apoll = NULL;
-	}
-	if (req->flags & REQ_F_INFLIGHT) {
-		struct io_uring_task *tctx = req->task->io_uring;
-
-		atomic_dec(&tctx->inflight_tracked);
-	}
-	if (req->flags & REQ_F_CREDS)
-		put_cred(req->creds);
-	if (req->flags & REQ_F_ASYNC_DATA) {
-		kfree(req->async_data);
-		req->async_data = NULL;
-	}
-	req->flags &= ~IO_REQ_CLEAN_FLAGS;
-}
-
 static bool io_assign_file(struct io_kiocb *req, const struct io_issue_def *def,
 			   unsigned int issue_flags)
 {
Signature · 1 line
2.40.0
#5 lore raw

PATCH 5/11 io_uring: don't batch task put on reqs free

In reply to Pavel Begunkov

We're trying to batch io_put_task() in io_free_batch_list(), but considering that the hot path is a simple inc, it's most cerainly and probably faster to just do io_put_task() instead of task tracking.

We don't care about io_put_task_remote() as it's only for IOPOLL where polling/waiting is done by not the submitter task.

---
 io_uring/io_uring.c | 32 ++++++++++----------------------
 1 file changed, 10 insertions(+), 22 deletions(-)
io_uring/io_uring.c +10 -22
index 4d8613996644..3eec5c761d0a 100644
@@ -754,29 +754,29 @@ static void io_cqring_overflow_flush(struct io_ring_ctx *ctx)
 }
 
 /* can be called by any task */
-static void io_put_task_remote(struct task_struct *task, int nr)
+static void io_put_task_remote(struct task_struct *task)
 {
 	struct io_uring_task *tctx = task->io_uring;
 
-	percpu_counter_sub(&tctx->inflight, nr);
+	percpu_counter_sub(&tctx->inflight, 1);
 	if (unlikely(atomic_read(&tctx->in_cancel)))
 		wake_up(&tctx->wait);
-	put_task_struct_many(task, nr);
+	put_task_struct(task);
 }
 
 /* used by a task to put its own references */
-static void io_put_task_local(struct task_struct *task, int nr)
+static void io_put_task_local(struct task_struct *task)
 {
-	task->io_uring->cached_refs += nr;
+	task->io_uring->cached_refs++;
 }
 
 /* must to be called somewhat shortly after putting a request */
-static inline void io_put_task(struct task_struct *task, int nr)
+static inline void io_put_task(struct task_struct *task)
 {
 	if (likely(task == current))
-		io_put_task_local(task, nr);
+		io_put_task_local(task);
 	else
-		io_put_task_remote(task, nr);
+		io_put_task_remote(task);
 }
 
 void io_task_refs_refill(struct io_uring_task *tctx)
@@ -1033,7 +1033,7 @@ static void __io_req_complete_post(struct io_kiocb *req, unsigned issue_flags)
 		 * we don't hold ->completion_lock. Clean them here to avoid
 		 * deadlocks.
 		 */
-		io_put_task_remote(req->task, 1);
+		io_put_task_remote(req->task);
 		wq_list_add_head(&req->comp_list, &ctx->locked_free_list);
 		ctx->locked_free_nr++;
 	}
@@ -1518,9 +1518,6 @@ void io_queue_next(struct io_kiocb *req)
 void io_free_batch_list(struct io_ring_ctx *ctx, struct io_wq_work_node *node)
 	__must_hold(&ctx->uring_lock)
 {
-	struct task_struct *task = NULL;
-	int task_refs = 0;
-
 	do {
 		struct io_kiocb *req = container_of(node, struct io_kiocb,
 						    comp_list);
@@ -1550,19 +1547,10 @@ void io_free_batch_list(struct io_ring_ctx *ctx, struct io_wq_work_node *node)
 
 		io_req_put_rsrc_locked(req, ctx);
 
-		if (req->task != task) {
-			if (task)
-				io_put_task(task, task_refs);
-			task = req->task;
-			task_refs = 0;
-		}
-		task_refs++;
+		io_put_task(req->task);
 		node = req->comp_list.next;
 		io_req_add_to_cache(req, ctx);
 	} while (node);
-
-	if (task)
-		io_put_task(task, task_refs);
 }
 
 static void __io_submit_flush_completions(struct io_ring_ctx *ctx)
Signature · 1 line
2.40.0
#6 lore raw

PATCH 6/11 io_uring: remove IOU_F_TWQ_FORCE_NORMAL

In reply to Pavel Begunkov

Extract a function for non-local task_work_add, and use it directly from io_move_task_work_from_local(). Now we don't use IOU_F_TWQ_FORCE_NORMAL and it can be killed.

As a small positive side effect we don't grab task->io_uring in io_req_normal_work_add anymore, which is not needed for io_req_local_work_add().

---
 io_uring/io_uring.c | 25 ++++++++++++++-----------
 io_uring/io_uring.h |  5 +----
 2 files changed, 15 insertions(+), 15 deletions(-)
io_uring/io_uring.c +14 -11
index 3eec5c761d0a..776d1aa73d26 100644
@@ -1317,7 +1317,7 @@ static __cold void io_fallback_tw(struct io_uring_task *tctx)
 	}
 }
 
-static void io_req_local_work_add(struct io_kiocb *req, unsigned flags)
+static inline void io_req_local_work_add(struct io_kiocb *req, unsigned flags)
 {
 	struct io_ring_ctx *ctx = req->ctx;
 	unsigned nr_wait, nr_tw, nr_tw_prev;
@@ -1368,19 +1368,11 @@ static void io_req_local_work_add(struct io_kiocb *req, unsigned flags)
 	wake_up_state(ctx->submitter_task, TASK_INTERRUPTIBLE);
 }
 
-void __io_req_task_work_add(struct io_kiocb *req, unsigned flags)
+static void io_req_normal_work_add(struct io_kiocb *req)
 {
 	struct io_uring_task *tctx = req->task->io_uring;
 	struct io_ring_ctx *ctx = req->ctx;
 
-	if (!(flags & IOU_F_TWQ_FORCE_NORMAL) &&
-	    (ctx->flags & IORING_SETUP_DEFER_TASKRUN)) {
-		rcu_read_lock();
-		io_req_local_work_add(req, flags);
-		rcu_read_unlock();
-		return;
-	}
-
 	/* task_work already pending, we're done */
 	if (!llist_add(&req->io_task_work.node, &tctx->task_list))
 		return;
@@ -1394,6 +1386,17 @@ void __io_req_task_work_add(struct io_kiocb *req, unsigned flags)
 	io_fallback_tw(tctx);
 }
 
+void __io_req_task_work_add(struct io_kiocb *req, unsigned flags)
+{
+	if (req->ctx->flags & IORING_SETUP_DEFER_TASKRUN) {
+		rcu_read_lock();
+		io_req_local_work_add(req, flags);
+		rcu_read_unlock();
+	} else {
+		io_req_normal_work_add(req);
+	}
+}
+
 static void __cold io_move_task_work_from_local(struct io_ring_ctx *ctx)
 {
 	struct llist_node *node;
@@ -1404,7 +1407,7 @@ static void __cold io_move_task_work_from_local(struct io_ring_ctx *ctx)
 						    io_task_work.node);
 
 		node = node->next;
-		__io_req_task_work_add(req, IOU_F_TWQ_FORCE_NORMAL);
+		io_req_normal_work_add(req);
 	}
 }
 
io_uring/io_uring.h +1 -4
index 9718897133db..20ba6df49b1f 100644
@@ -16,9 +16,6 @@
 #endif
 
 enum {
-	/* don't use deferred task_work */
-	IOU_F_TWQ_FORCE_NORMAL			= 1,
-
 	/*
 	 * A hint to not wake right away but delay until there are enough of
 	 * tw's queued to match the number of CQEs the task is waiting for.
@@ -26,7 +23,7 @@ enum {
 	 * Must not be used wirh requests generating more than one CQE.
 	 * It's also ignored unless IORING_SETUP_DEFER_TASKRUN is set.
 	 */
-	IOU_F_TWQ_LAZY_WAKE			= 2,
+	IOU_F_TWQ_LAZY_WAKE			= 1,
 };
 
 enum {
Signature · 1 line
2.40.0
#7 lore raw

PATCH 7/11 io_uring: kill io_cq_unlock()

In reply to Pavel Begunkov

We're abusing ->completion_lock helpers. io_cq_unlock() neither locking conditionally nor doing CQE flushing, which means that callers must have some side reason of taking the lock and should do it directly.

Open code io_cq_unlock() into io_cqring_overflow_kill() and clean it up.

---
 io_uring/io_uring.c | 10 ++--------
 1 file changed, 2 insertions(+), 8 deletions(-)
io_uring/io_uring.c +2 -8
index 776d1aa73d26..2f55abb676c0 100644
@@ -644,12 +644,6 @@ static inline void io_cq_lock(struct io_ring_ctx *ctx)
 	spin_lock(&ctx->completion_lock);
 }
 
-static inline void io_cq_unlock(struct io_ring_ctx *ctx)
-	__releases(ctx->completion_lock)
-{
-	spin_unlock(&ctx->completion_lock);
-}
-
 /* keep it inlined for io_submit_flush_completions() */
 static inline void __io_cq_unlock_post(struct io_ring_ctx *ctx)
 	__releases(ctx->completion_lock)
@@ -694,10 +688,10 @@ static void io_cqring_overflow_kill(struct io_ring_ctx *ctx)
 	struct io_overflow_cqe *ocqe;
 	LIST_HEAD(list);
 
-	io_cq_lock(ctx);
+	spin_lock(&ctx->completion_lock);
 	list_splice_init(&ctx->cq_overflow_list, &list);
 	clear_bit(IO_CHECK_CQ_OVERFLOW_BIT, &ctx->check_cq);
-	io_cq_unlock(ctx);
+	spin_unlock(&ctx->completion_lock);
 
 	while (!list_empty(&list)) {
 		ocqe = list_first_entry(&list, struct io_overflow_cqe, list);
Signature · 1 line
2.40.0
#8 lore raw

PATCH 8/11 io_uring: fix acquire/release annotations

In reply to Pavel Begunkov

We do conditional locking, so __io_cq_lock() and friends not always actually grab/release the lock, so kill misleading annotations.

---
 io_uring/io_uring.c | 3 ---
 1 file changed, 3 deletions(-)
io_uring/io_uring.c +0 -3
index 2f55abb676c0..8cb0f60d2885 100644
@@ -626,7 +626,6 @@ void __io_commit_cqring_flush(struct io_ring_ctx *ctx)
 }
 
 static inline void __io_cq_lock(struct io_ring_ctx *ctx)
-	__acquires(ctx->completion_lock)
 {
 	if (!ctx->task_complete)
 		spin_lock(&ctx->completion_lock);
@@ -646,7 +645,6 @@ static inline void io_cq_lock(struct io_ring_ctx *ctx)
 
 /* keep it inlined for io_submit_flush_completions() */
 static inline void __io_cq_unlock_post(struct io_ring_ctx *ctx)
-	__releases(ctx->completion_lock)
 {
 	io_commit_cqring(ctx);
 	__io_cq_unlock(ctx);
@@ -655,7 +653,6 @@ static inline void __io_cq_unlock_post(struct io_ring_ctx *ctx)
 }
 
 static void __io_cq_unlock_post_flush(struct io_ring_ctx *ctx)
-	__releases(ctx->completion_lock)
 {
 	io_commit_cqring(ctx);
 
Signature · 1 line
2.40.0
#9 lore raw

PATCH 9/11 io_uring: inline __io_cq_unlock

In reply to Pavel Begunkov

__io_cq_unlock is not very helpful, and users should be calling flush variants anyway. Open code the function.

---
 io_uring/io_uring.c | 12 ++++--------
 1 file changed, 4 insertions(+), 8 deletions(-)
io_uring/io_uring.c +4 -8
index 8cb0f60d2885..39d83b631107 100644
@@ -631,12 +631,6 @@ static inline void __io_cq_lock(struct io_ring_ctx *ctx)
 		spin_lock(&ctx->completion_lock);
 }
 
-static inline void __io_cq_unlock(struct io_ring_ctx *ctx)
-{
-	if (!ctx->task_complete)
-		spin_unlock(&ctx->completion_lock);
-}
-
 static inline void io_cq_lock(struct io_ring_ctx *ctx)
 	__acquires(ctx->completion_lock)
 {
@@ -647,7 +641,9 @@ static inline void io_cq_lock(struct io_ring_ctx *ctx)
 static inline void __io_cq_unlock_post(struct io_ring_ctx *ctx)
 {
 	io_commit_cqring(ctx);
-	__io_cq_unlock(ctx);
+	if (!ctx->task_complete)
+		spin_unlock(&ctx->completion_lock);
+
 	io_commit_cqring_flush(ctx);
 	io_cqring_wake(ctx);
 }
@@ -664,7 +660,7 @@ static void __io_cq_unlock_post_flush(struct io_ring_ctx *ctx)
 		 */
 		io_commit_cqring_flush(ctx);
 	} else {
-		__io_cq_unlock(ctx);
+		spin_unlock(&ctx->completion_lock);
 		io_commit_cqring_flush(ctx);
 		io_cqring_wake(ctx);
 	}
Signature · 1 line
2.40.0
#10 lore raw

PATCH 10/11 io_uring: make io_cq_unlock_post static

In reply to Pavel Begunkov

io_cq_unlock_post() is exclusively used in io_uring/io_uring.c, mark it static and don't expose to other files.

---
 io_uring/io_uring.c | 2 +-
 io_uring/io_uring.h | 2 --
 2 files changed, 1 insertion(+), 3 deletions(-)
io_uring/io_uring.c +1 -1
index 39d83b631107..70fffed83e95 100644
@@ -666,7 +666,7 @@ static void __io_cq_unlock_post_flush(struct io_ring_ctx *ctx)
 	}
 }
 
-void io_cq_unlock_post(struct io_ring_ctx *ctx)
+static void io_cq_unlock_post(struct io_ring_ctx *ctx)
 	__releases(ctx->completion_lock)
 {
 	io_commit_cqring(ctx);
io_uring/io_uring.h +0 -2
index 20ba6df49b1f..d3606d30cf6f 100644
@@ -110,8 +110,6 @@ static inline void io_req_task_work_add(struct io_kiocb *req)
 #define io_for_each_link(pos, head) \
 	for (pos = (head); pos; pos = pos->link)
 
-void io_cq_unlock_post(struct io_ring_ctx *ctx);
-
 static inline struct io_uring_cqe *io_get_cqe_overflow(struct io_ring_ctx *ctx,
 						       bool overflow)
 {
Signature · 1 line
2.40.0
#11 lore raw

PATCH 11/11 io_uring: merge conditional unlock flush helpers

In reply to Pavel Begunkov

There is no reason not to use __io_cq_unlock_post_flush for intermediate aux CQE flushing, all ->task_complete should apply there, i.e. if set it should be the submitter task. Combine them, get rid of of __io_cq_unlock_post() and rename the left function.

This place was also taking a couple percents of CPU according to profiles for max throughput net benchmarks due to multishot recv flooding it with completions.

---
 io_uring/io_uring.c | 13 +------------
 1 file changed, 1 insertion(+), 12 deletions(-)
io_uring/io_uring.c +1 -12
index 70fffed83e95..1b53a2ab0a27 100644
@@ -637,18 +637,7 @@ static inline void io_cq_lock(struct io_ring_ctx *ctx)
 	spin_lock(&ctx->completion_lock);
 }
 
-/* keep it inlined for io_submit_flush_completions() */
 static inline void __io_cq_unlock_post(struct io_ring_ctx *ctx)
-{
-	io_commit_cqring(ctx);
-	if (!ctx->task_complete)
-		spin_unlock(&ctx->completion_lock);
-
-	io_commit_cqring_flush(ctx);
-	io_cqring_wake(ctx);
-}
-
-static void __io_cq_unlock_post_flush(struct io_ring_ctx *ctx)
 {
 	io_commit_cqring(ctx);
 
@@ -1568,7 +1557,7 @@ static void __io_submit_flush_completions(struct io_ring_ctx *ctx)
 			}
 		}
 	}
-	__io_cq_unlock_post_flush(ctx);
+	__io_cq_unlock_post(ctx);
 
 	if (!wq_list_empty(&ctx->submit_state.compl_reqs)) {
 		io_free_batch_list(ctx, state->compl_reqs.first);
Signature · 1 line
2.40.0
maintainer
#12 lore raw
In reply to Pavel Begunkov
Quoted 16 lines · Pavel Begunkov

Patches 1-5 are cleaning how we free requests.
Patches 7-11 brush CQ / ->completion_lock locking

Pavel Begunkov (11):
io_uring: open code io_put_req_find_next
io_uring: remove io_free_req_tw
io_uring: inline io_dismantle_req()
io_uring: move io_clean_op()
io_uring: don't batch task put on reqs free
io_uring: remove IOU_F_TWQ_FORCE_NORMAL
io_uring: kill io_cq_unlock()
io_uring: fix acquire/release annotations
io_uring: inline __io_cq_unlock
io_uring: make io_cq_unlock_post static
io_uring: merge conditional unlock flush helpers

Quoted 3 lines
 io_uring/io_uring.c | 223 ++++++++++++++++----------------------------
 io_uring/io_uring.h |   7 +-
 2 files changed, 80 insertions(+), 150 deletions(-)

Was going to say that it's a bit late for 6.5, but looking over the patches, it's pretty straight forward and mostly just good cleanups and code reduction. I'll queue it up for 6.5, thanks.

Signature · 1 line
Jens Axboe
maintainer Applied
#13 lore raw
In reply to Pavel Begunkov
Quoted 16 lines · Pavel Begunkov

Patches 1-5 are cleaning how we free requests.
Patches 7-11 brush CQ / ->completion_lock locking

Pavel Begunkov (11):
io_uring: open code io_put_req_find_next
io_uring: remove io_free_req_tw
io_uring: inline io_dismantle_req()
io_uring: move io_clean_op()
io_uring: don't batch task put on reqs free
io_uring: remove IOU_F_TWQ_FORCE_NORMAL
io_uring: kill io_cq_unlock()
io_uring: fix acquire/release annotations
io_uring: inline __io_cq_unlock
io_uring: make io_cq_unlock_post static
io_uring: merge conditional unlock flush helpers

Quoted 1 line

[...]

Applied, thanks!

[01/11] io_uring: open code io_put_req_find_next

        commit: 247f97a5f19b642eba5f5c1cf95fc3169326b3fb
[02/11] io_uring: remove io_free_req_tw
        commit: 6ec9afc7f4cba58ab740c59d4c964d9422e2ea82
[03/11] io_uring: inline io_dismantle_req()
        commit: 3b7a612fd0dbd321e15a308b8ac1f8bbf81432bd
[04/11] io_uring: move io_clean_op()
        commit: 5a754dea27fb91a418f7429e24479e4184dee2e3
[05/11] io_uring: don't batch task put on reqs free
        commit: 2fdd6fb5ff958a0f6b403e3f3ffd645b60b2a2b2
[06/11] io_uring: remove IOU_F_TWQ_FORCE_NORMAL
        commit: 91c7884ac9a92ffbf78af7fc89603daf24f448a9
[07/11] io_uring: kill io_cq_unlock()
        commit: f432b76bcc93f36edb3d371f7b8d7881261dd6e7
[08/11] io_uring: fix acquire/release annotations
        commit: 55b6a69fed5df6f88ef0b2ace562b422162beb61
[09/11] io_uring: inline __io_cq_unlock
        commit: ff12617728fa5c7fb5325e164503ca4e936b80bd
[10/11] io_uring: make io_cq_unlock_post static
        commit: 0fdb9a196c6728b51e0e7a4f6fa292d9fd5793de
[11/11] io_uring: merge conditional unlock flush helpers
        commit: c98c81a4ac37b651be7eb9d16f562fc4acc5f867

Best regards,

Signature · 1 line
Jens Axboe