Skip to contentSkip to conversation

nvme-tcp: remove the direct send path

RFC PATCH In review Seiji Nishikawa 2 participants 5 messages Nov 22 NVMe

lore
Share & tools
lore
#0 lore raw

cover nvme-tcp: remove the direct send path

Hannes suggested removing the direct send path in nvme_tcp_queue_request() while reviewing commit 5db358f3a7f9 ("nvme-tcp:
fix io_work priority inversion when send_mutex is held"). This series does that, on top of nvme-7.4.

Patch 1 removes the direct send path, so io_work is the only sender. Patch 2 removes send_mutex, which has nothing left to protect after patch 1.

The direct send path was added to cut latency when the queue is idle, so I measured it on two machines with 1Gb NICs. The target is nvmet-tcp backed by null_blk (irqmode=0). The host runs fio like this, 60 seconds per run and 3 runs per case:

  fio --name=qd1 --filename=/dev/nvme0n1 --direct=1 --ioengine=libaio \
      --bs=4k --rw=randread --iodepth=1 --numjobs=1 --cpus_allowed=2 \
      --time_based --runtime=60 --ramp_time=5

Average of the 3 runs at queue depth 1, latency in usec, "patched" is this series:

IOPS slat clat lat clat p99

  randread   base     4153    7.1  232.3  239.3       277
             patched  4154    4.7  234.7  239.4       280
  randwrite  base     2472   24.5  375.6  400.1       553
             patched  2460   14.3  387.6  402.0       570

The send time moves from slat to clat. With the direct send path, io_submit() itself sends the command on the socket, so that time is counted in slat. Without it, io_submit() only queues the request and returns, and io_work sends it, so the same time is counted in clat. lat is the sum of the two, and it changes by 0.1 usec for randread and 1.9 usec (+0.5%) for randwrite.

At queue depth 32 with 4 jobs, both kernels reach the 1Gb link limit (28.2k IOPS), so there is no difference there.

On the same two machines, I also ran 5 controller resets during a 120 second randrw fio run on the patched kernel, and fio finished with no errors. Separately, blktests nvme over tcp on a VM with an nvmet tcp loopback target gives the same results with and without this series.

On a 1Gb network one I/O takes about 240 us, so a difference of a few microseconds is easy to miss. Numbers from faster networks would show whether it matters there. I am posting this as RFC so the performance side can be discussed on the list.

Seiji Nishikawa (2):
nvme-tcp: remove the direct send path
nvme-tcp: remove send_mutex

 drivers/nvme/host/tcp.c | 60 +++++++----------------------------------
 1 file changed, 9 insertions(+), 51 deletions(-)

base-commit: 68099cf27253a8c89d703a282d3d811446d9453b

Signature · 1 line
2.56.1
#1 lore raw

RFC PATCH 1/2 nvme-tcp: remove the direct send path

nvme_tcp_queue_request() sends the request directly from the submitting context when it runs on the io_cpu of the queue, the queue is empty and it can take send_mutex. Commit db5ad6b7f8cd ("nvme-tcp: try to send request in queue_rq context") added this to skip the workqueue hop when the queue is idle.

Since then this path has needed a series of fixes for races and priority inversions between it and io_work. These are
commit 5c11f7d9f843 ("nvme-tcp: Fix possible race of io_work and direct send"), commit a0fdd1418007 ("nvme-tcp: rerun io_work if req_list is not empty"), commit 70f437fb4395 ("nvme-tcp: fix io_work priority inversion"), commit 08acb54b063a ("nvme-tcp: defer TLS inline send to io_work") and commit 5db358f3a7f9 ("nvme-tcp: fix io_work priority inversion when send_mutex is held").

All of them come from having 2 contexts that send on the same queue and coordinate through send_mutex, while io_work runs on a high priority workqueue and the submitting task does not.

Remove the direct send path and always send from io_work. TLS queues already work this way since 08acb54b063a. With a single sender, io_work has to be queued whenever the last request of a batch is added, so the has_pending check in nvme_tcp_queue_request() goes away too, and nvme_tcp_send_all() has no caller left.

---
 drivers/nvme/host/tcp.c | 33 ++-------------------------------
 1 file changed, 2 insertions(+), 31 deletions(-)
drivers/nvme/host/tcp.c +2 -31
index 62a594b8ab9b..2b832ac3068a 100644
@@ -213,7 +213,6 @@ struct nvme_tcp_ctrl {
 static struct workqueue_struct *nvme_tcp_wq;
 static const struct blk_mq_ops nvme_tcp_mq_ops;
 static const struct blk_mq_ops nvme_tcp_admin_mq_ops;
-static int nvme_tcp_try_send(struct nvme_tcp_queue *queue);
 
 #ifdef CONFIG_DEBUG_LOCK_ALLOC
 /* lockdep can detect a circular dependency of the form
@@ -420,16 +419,6 @@ static inline void nvme_tcp_advance_req(struct nvme_tcp_request *req,
 	}
 }
 
-static inline void nvme_tcp_send_all(struct nvme_tcp_queue *queue)
-{
-	int ret;
-
-	/* drain the send queue as much as we can... */
-	do {
-		ret = nvme_tcp_try_send(queue);
-	} while (ret > 0);
-}
-
 static inline bool nvme_tcp_queue_has_pending(struct nvme_tcp_queue *queue)
 {
 	return !list_empty(&queue->send_list) ||
@@ -446,28 +435,10 @@ static inline void nvme_tcp_queue_request(struct nvme_tcp_request *req,
 		bool last)
 {
 	struct nvme_tcp_queue *queue = req->queue;
-	bool empty;
 
-	empty = llist_add(&req->lentry, &queue->req_list) &&
-		list_empty(&queue->send_list) && !queue->request;
-
-	/*
-	 * if we're the first on the send_list and we can try to send
-	 * directly, otherwise queue io_work. Also, only do that if we
-	 * are on the same cpu, so we don't introduce contention.
-	 *
-	 * TLS kTLS send takes ctx->tx_lock while blk_mq holds set->srcu.
-	 * lockdep reports circular locking via elevator_lock.  Defer TLS
-	 * sends to the io workqueue instead of inline from this path.
-	 */
-	if (queue->io_cpu == raw_smp_processor_id() &&
-	    !nvme_tcp_queue_tls(queue) &&
-	    empty && mutex_trylock(&queue->send_mutex)) {
-		nvme_tcp_send_all(queue);
-		mutex_unlock(&queue->send_mutex);
-	}
+	llist_add(&req->lentry, &queue->req_list);
 
-	if (last && nvme_tcp_queue_has_pending(queue))
+	if (last)
 		queue_work_on(queue->io_cpu, nvme_tcp_wq, &queue->io_work);
 }
 
Signature · 1 line
2.56.1
#2 lore raw
Thank you for your contribution! Sashiko AI review found 1 potential issue to consider: - [Medium] nvme-tcp: poll queue starvation and latency regression in nvme_tcp_queue_request() --

Thank you for your contribution! Sashiko AI review found 1 potential issue to consider:
- [Medium] nvme-tcp: poll queue starvation and latency regression in nvme_tcp_queue_request()
--

commit 7f6babf6da3e0f7ca956d6a355e996efe6f74e66
Author: Seiji Nishikawa <[email protected]>

nvme-tcp: remove the direct send path

This commit removes the inline direct send path in nvme_tcp_queue_request() and nvme_tcp_send_all(), deferring all request transmissions to queue->io_work on nvme_tcp_wq.

Quoted 4 lines
drivers/nvme/host/tcp.c +0 -0
index 62a594b8ab9bf..2b832ac3068a9 100644

[ ... ]

Quoted 20 lines
drivers/nvme/host/tcp.c +0 -15
@@ -446,28 +435,10 @@ static inline void nvme_tcp_queue_request(struct nvme_tcp_request *req,
 		bool last)
 {
 	struct nvme_tcp_queue *queue = req->queue;
-	bool empty;
 
-	empty = llist_add(&req->lentry, &queue->req_list) &&
-		list_empty(&queue->send_list) && !queue->request;
-
-	/*
-	 * if we're the first on the send_list and we can try to send
-	 * directly, otherwise queue io_work. Also, only do that if we
-	 * are on the same cpu, so we don't introduce contention.
-	 *
-	 * TLS kTLS send takes ctx->tx_lock while blk_mq holds set->srcu.
-	 * lockdep reports circular locking via elevator_lock.  Defer TLS
-	 * sends to the io workqueue instead of inline from this path.
-	 */
-	if (queue->io_cpu == raw_smp_processor_id() &&
-	    !nvme_tcp_queue_tls(queue) &&

drivers/nvme/host/tcp.c static inline void nvme_tcp_queue_request(struct nvme_tcp_request *req, · line 443

Quoted 10 lines
drivers/nvme/host/tcp.c +2 -5
-	    empty && mutex_trylock(&queue->send_mutex)) {
-		nvme_tcp_send_all(queue);
-		mutex_unlock(&queue->send_mutex);
-	}
+	llist_add(&req->lentry, &queue->req_list);
 
-	if (last && nvme_tcp_queue_has_pending(queue))
+	if (last)
 		queue_work_on(queue->io_cpu, nvme_tcp_wq, &queue->io_work);
 }

[Severity: Medium]
Does removing the inline send from nvme_tcp_queue_request() break polled I/O on HCTX_TYPE_POLL queues?

nvme_tcp_set_queue_io_cpu() binds queue->io_cpu for a poll queue to a CPU in set->map[HCTX_TYPE_POLL].mq_map, while nvme_tcp_poll() only polls the receive side and never drains queue->req_list:

drivers/nvme/host/tcp.c:nvme_tcp_poll() {
    ...
	set_bit(NVME_TCP_Q_POLLING, &queue->flags);
	if (sk_can_busy_loop(sk) && skb_queue_empty_lockless(&sk->sk_receive_queue))
		sk_busy_loop(sk, true);
	ret = nvme_tcp_try_recv(queue);
	clear_bit(NVME_TCP_Q_POLLING, &queue->flags);
    ...
}

Previously, a task submitting and polling on queue->io_cpu sent the request inline when the queue was idle and completed it without waking nvme_tcp_wq. With all sends deferred to queue->io_work on queue->io_cpu, a SCHED_FIFO or SCHED_RR task polling via io_uring IORING_SETUP_IOPOLL or blk_execute_rq() on queue->io_cpu starves the SCHED_NORMAL nvme_tcp_wq worker on the same CPU and spins in nvme_tcp_poll() waiting for a request still sitting in queue->req_list until RT throttling or the 30-second NVMe I/O timeout fires. For SCHED_NORMAL tasks on queue->io_cpu, waking the WQ_HIGHPRI worker sets TIF_NEED_RESCHED, causing blk_hctx_poll() and io_iopoll_check() to abort polling on need_resched() or forcing two context switches per polled I/O.

Signature · 1 line
Sashiko AI review · https://sashiko.dev/#/patchset/[email protected]?part=1
#3 lore raw
Quoted 3 lines · On Sun, Nov 22, 2026 at 10:49 AM <[email protected]> wrote:

[Severity: Medium]
Does removing the inline send from nvme_tcp_queue_request() break polled I/O on HCTX_TYPE_POLL queues?

[ ... ]

Quoted 4 lines

For SCHED_NORMAL tasks on queue->io_cpu, waking
the WQ_HIGHPRI worker sets TIF_NEED_RESCHED, causing blk_hctx_poll() and io_iopoll_check() to abort polling on need_resched() or forcing two context switches per polled I/O.

This is right. I ran the test with poll queues on the same two machines. The host connects with --nr-poll-queues=4, and fio runs with io_uring and --hipri, 4k random I/O at queue depth 1, on the io_cpu of a poll queue. Average of 3 runs of 60 seconds, latency in usec, ctx is the number of context switches of fio in one run:

                     IOPS   slat    clat     lat      ctx
randread   base      5017   4.72  193.95  198.67      553
           patched   4179   9.35  229.70  239.05  251,318
randwrite  base      3619   5.60  269.94  275.54      542
           patched   3621  10.62  264.53  275.16  217,806

For randread, lat goes up by 40 usec (20%) and IOPS goes down by 17%. ctx shows why. Without the direct send path, fio is switched out for the io_work kworker once per I/O. With it, fio stays on the CPU and keeps polling. randwrite shows no difference.

So removing the direct send path costs about 20% for polled reads at queue depth 1 on this setup. One way to avoid this is to keep the direct send path only for poll queues. Poll queues then still have 2 senders, so send_mutex stays and patch 2 goes away.

I will send a v2 that does this, unless the list prefers to remove the direct send path for all queues and take this cost.

Thanks,
Seiji

#4 lore raw

RFC PATCH 2/2 nvme-tcp: remove send_mutex

In reply to Seiji Nishikawa

With the direct send path gone, io_work is the only context that sends on a queue. A work item never runs concurrently with itself, so send_mutex does not protect anything anymore.

Remove it, together with the mutex_trylock() in nvme_tcp_io_work() and the got_send_mutex check, which is now always true.

Assisted-by LLM
---
 drivers/nvme/host/tcp.c | 27 +++++++--------------------
 1 file changed, 7 insertions(+), 20 deletions(-)
drivers/nvme/host/tcp.c +7 -20
index 2b832ac3068a..b116268fe2f5 100644
@@ -142,7 +142,6 @@ struct nvme_tcp_queue {
 	int			io_cpu;
 
 	struct mutex		queue_lock;
-	struct mutex		send_mutex;
 	struct mutex		pf_cache_lock;
 	struct llist_head	req_list;
 	struct list_head	send_list;
@@ -1482,18 +1481,13 @@ static void nvme_tcp_io_work(struct work_struct *w)
 
 	do {
 		bool pending = false;
-		bool got_send_mutex = false;
 		int result;
 
-		if (mutex_trylock(&queue->send_mutex)) {
-			got_send_mutex = true;
-			result = nvme_tcp_try_send(queue);
-			mutex_unlock(&queue->send_mutex);
-			if (result > 0)
-				pending = true;
-			else if (unlikely(result < 0))
-				break;
-		}
+		result = nvme_tcp_try_send(queue);
+		if (result > 0)
+			pending = true;
+		else if (unlikely(result < 0))
+			break;
 
 		result = nvme_tcp_try_recv(queue);
 		if (result > 0)
@@ -1501,13 +1495,8 @@ static void nvme_tcp_io_work(struct work_struct *w)
 		else if (unlikely(result < 0))
 			return;
 
-		/*
-		 * did we get some space after spending time in recv?
-		 * Only check this if we could take send_mutex. Otherwise
-		 * the task holding it sends the requests and queues io_work
-		 * again after it releases the mutex.
-		 */
-		if (got_send_mutex && nvme_tcp_queue_has_pending(queue) &&
+		/* did we get some space after spending time in recv? */
+		if (nvme_tcp_queue_has_pending(queue) &&
 		    sk_stream_is_writeable(queue->sock->sk))
 			pending = true;
 
@@ -2087,7 +2076,6 @@ static void nvme_tcp_init_queue(struct nvme_tcp_ctrl *ctrl, int qid)
 
 	queue->ctrl = ctrl;
 	mutex_init(&queue->queue_lock);
-	mutex_init(&queue->send_mutex);
 	mutex_init(&queue->pf_cache_lock);
 	INIT_WORK(&queue->io_work, nvme_tcp_io_work);
 	queue->hdr_digest = ctrl->ctrl.opts->hdr_digest;
@@ -2099,7 +2087,6 @@ static void nvme_tcp_exit_queue(struct nvme_tcp_ctrl *ctrl, int qid)
 	struct nvme_tcp_queue *queue = &ctrl->queues[qid];
 
 	mutex_destroy(&queue->queue_lock);
-	mutex_destroy(&queue->send_mutex);
 	mutex_destroy(&queue->pf_cache_lock);
 }
 
Signature · 1 line
2.56.1