Skip to contentSkip to conversation

nvme-rdma: parallelize I/O queue allocation and startup

PATCH Superseded Surabhi Gogte 3 participants 5 messages Jun 4 – Jun 25 NVMe

v2 of 4 Superseded Latest is v4 →

lore
Share & tools
lore
#0 lore raw

PATCH v2 1/1 nvme-rdma: parallelize I/O queue allocation and startup

Refactor nvme rdma I/O queue setup to use async API, combining allocation and startup into a single parallel operation per queue. This reduces connection and reconnection setup time when there are delays in establishing connections, which is especially important for high-core-count hosts.

Key changes:
- Use async API to facilitate parallel calls for io queue setup.
- Add qsetup_err atomic to nvme_rdma_ctrl for collecting errors across parallel threads.
- Refactor nvme_rdma_alloc_queue() to accept a pre-initialized queue pointer instead of (ctrl, idx, queue_size), updating all call sites including the admin queue path.
- Remove nvme_rdma_alloc_io_queues() and nvme_rdma_start_io_queues(); their logic is folded into nvme_rdma_setup_io_queues() and nvme_rdma_configure_io_queues().
- Move queue count negotiation (nvme_set_queue_count, nvmf_set_io_queues) from the removed nvme_rdma_alloc_io_queues() into nvme_rdma_configure_io_queues().

Testing on a 64-core host with 64 IO-queues shows nvme-rdma connection time reduced from ~1.4s to 416ms.

---
Changes since v1:
- Replace dedicated nvme_setup_wq workqueue with async API implementation.

---
 drivers/nvme/host/rdma.c | 122 ++++++++++++++++++++++-----------------
 1 file changed, 68 insertions(+), 54 deletions(-)
drivers/nvme/host/rdma.c +68 -54
index f77c960f7632..3b33f7be563b 100644
@@ -16,6 +16,7 @@
 #include <linux/types.h>
 #include <linux/list.h>
 #include <linux/mutex.h>
+#include <linux/async.h>
 #include <linux/scatterlist.h>
 #include <linux/nvme.h>
 #include <linux/unaligned.h>
@@ -125,6 +126,7 @@ struct nvme_rdma_ctrl {
 	struct nvme_ctrl	ctrl;
 	bool			use_inline_data;
 	u32			io_queues[HCTX_MAX_TYPES];
+	atomic_t		qsetup_err;
 };
 
 static inline struct nvme_rdma_ctrl *to_rdma_ctrl(struct nvme_ctrl *ctrl)
@@ -566,16 +568,14 @@ static int nvme_rdma_create_queue_ib(struct nvme_rdma_queue *queue)
 	return ret;
 }
 
-static int nvme_rdma_alloc_queue(struct nvme_rdma_ctrl *ctrl,
-		int idx, size_t queue_size)
+static int nvme_rdma_alloc_queue(struct nvme_rdma_queue *queue)
 {
-	struct nvme_rdma_queue *queue;
+	struct nvme_rdma_ctrl *ctrl = queue->ctrl;
+	int idx = nvme_rdma_queue_idx(queue);
 	struct sockaddr *src_addr = NULL;
 	int ret;
 
-	queue = &ctrl->queues[idx];
 	mutex_init(&queue->queue_lock);
-	queue->ctrl = ctrl;
 	if (idx && ctrl->ctrl.max_integrity_segments)
 		queue->pi_support = true;
 	else
@@ -587,8 +587,6 @@ static int nvme_rdma_alloc_queue(struct nvme_rdma_ctrl *ctrl,
 	else
 		queue->cmnd_capsule_len = sizeof(struct nvme_command);
 
-	queue->queue_size = queue_size;
-
 	queue->cm_id = rdma_create_id(&init_net, nvme_rdma_cm_handler, queue,
 			RDMA_PS_TCP, IB_QPT_RC);
 	if (IS_ERR(queue->cm_id)) {
@@ -694,59 +692,57 @@ static int nvme_rdma_start_queue(struct nvme_rdma_ctrl *ctrl, int idx)
 	return ret;
 }
 
-static int nvme_rdma_start_io_queues(struct nvme_rdma_ctrl *ctrl,
-				     int first, int last)
+static void nvme_rdma_setup_queue_async(void *setup_queue, async_cookie_t cookie)
 {
-	int i, ret = 0;
+	struct nvme_rdma_queue *queue = setup_queue;
+	int ret;
 
-	for (i = first; i < last; i++) {
-		ret = nvme_rdma_start_queue(ctrl, i);
-		if (ret)
-			goto out_stop_queues;
-	}
+	ret = nvme_rdma_alloc_queue(queue);
+	if (ret)
+		goto out_err;
 
-	return 0;
+	ret = nvme_rdma_start_queue(queue->ctrl, nvme_rdma_queue_idx(queue));
+	if (ret)
+		goto out_err;
 
-out_stop_queues:
-	for (i--; i >= first; i--)
-		nvme_rdma_stop_queue(&ctrl->queues[i]);
-	return ret;
+	return;
+
+out_err:
+	atomic_cmpxchg(&queue->ctrl->qsetup_err, 0, ret);
 }
 
-static int nvme_rdma_alloc_io_queues(struct nvme_rdma_ctrl *ctrl)
+static int nvme_rdma_setup_io_queues(struct nvme_rdma_ctrl *ctrl, int first,
+				     int last, size_t queue_size)
 {
-	struct nvmf_ctrl_options *opts = ctrl->ctrl.opts;
-	unsigned int nr_io_queues;
+	int nr_queues = last - first;
 	int i, ret;
+	ASYNC_DOMAIN_EXCLUSIVE(setup_queue_domain);
 
-	nr_io_queues = nvmf_nr_io_queues(opts);
-	ret = nvme_set_queue_count(&ctrl->ctrl, &nr_io_queues);
-	if (ret)
-		return ret;
-
-	if (nr_io_queues == 0) {
-		dev_err(ctrl->ctrl.device,
-			"unable to set any I/O queues\n");
-		return -ENOMEM;
-	}
+	atomic_set(&ctrl->qsetup_err, 0);
 
-	ctrl->ctrl.queue_count = nr_io_queues + 1;
-	dev_info(ctrl->ctrl.device,
-		"creating %d I/O queues.\n", nr_io_queues);
+	for (i = 0; i < nr_queues; i++) {
+		struct nvme_rdma_queue *queue = &ctrl->queues[first + i];
 
-	nvmf_set_io_queues(opts, nr_io_queues, ctrl->io_queues);
-	for (i = 1; i < ctrl->ctrl.queue_count; i++) {
-		ret = nvme_rdma_alloc_queue(ctrl, i,
-				ctrl->ctrl.sqsize + 1);
-		if (ret)
-			goto out_free_queues;
+		queue->ctrl = ctrl;
+		queue->queue_size = queue_size;
+		async_schedule_domain(nvme_rdma_setup_queue_async, queue,
+				      &setup_queue_domain);
 	}
 
-	return 0;
+	async_synchronize_full_domain(&setup_queue_domain);
 
-out_free_queues:
-	for (i--; i >= 1; i--)
-		nvme_rdma_free_queue(&ctrl->queues[i]);
+	ret = atomic_read(&ctrl->qsetup_err);
+	if (ret) {
+		for (i = 0; i < nr_queues; i++) {
+			struct nvme_rdma_queue *queue =
+				&ctrl->queues[first + i];
+
+			if (test_bit(NVME_RDMA_Q_LIVE, &queue->flags))
+				nvme_rdma_stop_queue(queue);
+			if (test_bit(NVME_RDMA_Q_ALLOCATED, &queue->flags))
+				nvme_rdma_free_queue(queue);
+		}
+	}
 
 	return ret;
 }
@@ -783,7 +779,9 @@ static int nvme_rdma_configure_admin_queue(struct nvme_rdma_ctrl *ctrl,
 	bool pi_capable = false;
 	int error;
 
-	error = nvme_rdma_alloc_queue(ctrl, 0, NVME_AQ_DEPTH);
+	ctrl->queues[0].ctrl = ctrl;
+	ctrl->queues[0].queue_size = NVME_AQ_DEPTH;
+	error = nvme_rdma_alloc_queue(&ctrl->queues[0]);
 	if (error)
 		return error;
 
@@ -864,11 +862,22 @@ static int nvme_rdma_configure_admin_queue(struct nvme_rdma_ctrl *ctrl,
 static int nvme_rdma_configure_io_queues(struct nvme_rdma_ctrl *ctrl, bool new)
 {
 	int ret, nr_queues;
+	unsigned int nr_io_queues;
 
-	ret = nvme_rdma_alloc_io_queues(ctrl);
+	nr_io_queues = nvmf_nr_io_queues(ctrl->ctrl.opts);
+	ret = nvme_set_queue_count(&ctrl->ctrl, &nr_io_queues);
 	if (ret)
 		return ret;
 
+	if (nr_io_queues == 0) {
+		dev_err(ctrl->ctrl.device, "unable to set any I/O queues\n");
+		return -ENOMEM;
+	}
+
+	ctrl->ctrl.queue_count = nr_io_queues + 1;
+	dev_info(ctrl->ctrl.device, "creating %d I/O queues.\n", nr_io_queues);
+	nvmf_set_io_queues(ctrl->ctrl.opts, nr_io_queues, ctrl->io_queues);
+
 	if (new) {
 		ret = nvme_rdma_alloc_tag_set(&ctrl->ctrl);
 		if (ret)
@@ -881,7 +890,9 @@ static int nvme_rdma_configure_io_queues(struct nvme_rdma_ctrl *ctrl, bool new)
 	 * queue number might have changed.
 	 */
 	nr_queues = min(ctrl->tag_set.nr_hw_queues + 1, ctrl->ctrl.queue_count);
-	ret = nvme_rdma_start_io_queues(ctrl, 1, nr_queues);
+	ret = nvme_rdma_setup_io_queues(ctrl, 1, nr_queues,
+					ctrl->ctrl.sqsize + 1);
+
 	if (ret)
 		goto out_cleanup_tagset;
 
@@ -905,12 +916,15 @@ static int nvme_rdma_configure_io_queues(struct nvme_rdma_ctrl *ctrl, bool new)
 
 	/*
 	 * If the number of queues has increased (reconnect case)
-	 * start all new queues now.
+	 * setup all new queues now.
 	 */
-	ret = nvme_rdma_start_io_queues(ctrl, nr_queues,
-					ctrl->tag_set.nr_hw_queues + 1);
-	if (ret)
-		goto out_wait_freeze_timed_out;
+	if (ctrl->tag_set.nr_hw_queues + 1 > nr_queues) {
+		ret = nvme_rdma_setup_io_queues(ctrl, nr_queues,
+						ctrl->tag_set.nr_hw_queues + 1,
+						ctrl->ctrl.sqsize + 1);
+		if (ret)
+			goto out_wait_freeze_timed_out;
+	}
 
 	return 0;
 
Signature · 1 line
2.54.0
maintainer
#1 lore raw
Quoted 5 lines · Surabhi Gogte

Refactor nvme rdma I/O queue setup to use async API, combining allocation and startup into a single parallel operation per queue. This reduces connection and reconnection setup time when there are delays in establishing connections, which is especially important for high-core-count hosts.

Mostly looks fine.

Quoted 9 lines
drivers/nvme/host/rdma.c +1 -0
@@ -16,6 +16,7 @@
 #include <linux/types.h>
 #include <linux/list.h>
 #include <linux/mutex.h>
+#include <linux/async.h>
 #include <linux/scatterlist.h>
 #include <linux/nvme.h>
 #include <linux/unaligned.h>

drivers/nvme/host/rdma.c struct nvme_rdma_ctrl { · line 130

Quoted 5 lines
drivers/nvme/host/rdma.c +1 -0
@@ -125,6 +126,7 @@ struct nvme_rdma_ctrl {
 	struct nvme_ctrl	ctrl;
 	bool			use_inline_data;
 	u32			io_queues[HCTX_MAX_TYPES];
+	atomic_t		qsetup_err;
 };

This new field serves only to propogate an error from a local context, so I don't want to introduce a new field for it at this scope. I prefer you declare a special context struct for it to use with the async usage:

struct nvme_rdma_setup_ctx {
	struct nvme_rdma_queue  *queue;
	int                     *err;
};

And then make that the cookie passed to the async setup. I don't think it needs to be atomic here either: we really don't care if we see the first or last error, so forcing a cmpxchg for the first one is a bit overkill; you can just do READ/WRITE_ONCE instead and accept the race.

#2 lore raw
Quoted 11 lines · Keith Busch
Quoted 5 lines · Surabhi Gogte

Refactor nvme rdma I/O queue setup to use async API, combining allocation and startup into a single parallel operation per queue. This reduces connection and reconnection setup time when there are delays in establishing connections, which is especially important for high-core-count hosts.

Mostly looks fine.

Quoted 11 lines

drivers/nvme/host/rdma.c struct nvme_rdma_ctrl { · line 130

Quoted 6 lines
drivers/nvme/host/rdma.c +1 -0
@@ -125,6 +126,7 @@ struct nvme_rdma_ctrl {
      struct nvme_ctrl        ctrl;
      bool                    use_inline_data;
      u32                     io_queues[HCTX_MAX_TYPES];
+     atomic_t                qsetup_err;
 };

This new field serves only to propogate an error from a local context, so I don't want to introduce a new field for it at this scope. I prefer you declare a special context struct for it to use with the async usage:

Implemented in v3.

Quoted 9 lines
struct nvme_rdma_setup_ctx {
        struct nvme_rdma_queue  *queue;
        int                     *err;
};

And then make that the cookie passed to the async setup. I don't think it needs to be atomic here either: we really don't care if we see the first or last error, so forcing a cmpxchg for the first one is a bit overkill; you can just do READ/WRITE_ONCE instead and accept the race.

Yeah, that works too. added it in v3.

#3 lore raw
In reply to Surabhi Gogte
Quoted 1 line · Surabhi Gogte

+static void nvme_rdma_setup_queue_async(void *setup_queue, async_cookie_t cookie)

Overly long line. Easily fixed by renaming setup_queue to data as in the async_schedule_domain prototype.

drivers/nvme/host/rdma.c static int nvme_rdma_start_queue(struct nvme_rdma_ctrl *ctrl, int idx) · line 715

Quoted 2 lines
drivers/nvme/host/rdma.c +2 -0
+static int nvme_rdma_setup_io_queues(struct nvme_rdma_ctrl *ctrl, int first,
+				     int last, size_t queue_size)

should first and last be unsigned?

Also please use two-tab indents for prototype continuations.

drivers/nvme/host/rdma.c static int nvme_rdma_start_queue(struct nvme_rdma_ctrl *ctrl, int idx) · line 735

Quoted 7 lines
drivers/nvme/host/rdma.c +3 -3
+	async_synchronize_full_domain(&setup_queue_domain);
 
-out_free_queues:
-	for (i--; i >= 1; i--)
-		nvme_rdma_free_queue(&ctrl->queues[i]);
+	ret = atomic_read(&ctrl->qsetup_err);
+	if (ret) {

Jump to an error label if there was an error to reduce indentation a bit below.

drivers/nvme/host/rdma.c static int nvme_rdma_configure_admin_queue(struct nvme_rdma_ctrl *ctrl, · line 784

Quoted 3 lines
drivers/nvme/host/rdma.c +3 -0
+	ctrl->queues[0].ctrl = ctrl;
+	ctrl->queues[0].queue_size = NVME_AQ_DEPTH;
+	error = nvme_rdma_alloc_queue(&ctrl->queues[0]);

The nvme_rdma_alloc_queue prototype change would be a nice prep patch to split out from the main change.

#4 lore raw
Quoted 7 lines · Christoph Hellwig
Quoted 1 line · Surabhi Gogte

+static void nvme_rdma_setup_queue_async(void *setup_queue, async_cookie_t cookie)

Overly long line. Easily fixed by renaming setup_queue to data as in the async_schedule_domain prototype.

done.

Quoted 7 lines

drivers/nvme/host/rdma.c static int nvme_rdma_start_queue(struct nvme_rdma_ctrl *ctrl, int idx) · line 715

Quoted 2 lines
drivers/nvme/host/rdma.c +2 -0
+static int nvme_rdma_setup_io_queues(struct nvme_rdma_ctrl *ctrl, int first,
+                                  int last, size_t queue_size)

should first and last be unsigned?

Also please use two-tab indents for prototype continuations.

converted to unsigned int. yeah using 2-tab indents in v3.

Quoted 11 lines

drivers/nvme/host/rdma.c static int nvme_rdma_start_queue(struct nvme_rdma_ctrl *ctrl, int idx) · line 735

Quoted 7 lines
drivers/nvme/host/rdma.c +3 -3
+     async_synchronize_full_domain(&setup_queue_domain);
 
-out_free_queues:
-     for (i--; i >= 1; i--)
-             nvme_rdma_free_queue(&ctrl->queues[i]);
+     ret = atomic_read(&ctrl->qsetup_err);
+     if (ret) {

Jump to an error label if there was an error to reduce indentation a bit below.

done.

Quoted 7 lines

drivers/nvme/host/rdma.c static int nvme_rdma_configure_admin_queue(struct nvme_rdma_ctrl *ctrl, · line 784

Quoted 3 lines
drivers/nvme/host/rdma.c +3 -0
+     ctrl->queues[0].ctrl = ctrl;
+     ctrl->queues[0].queue_size = NVME_AQ_DEPTH;
+     error = nvme_rdma_alloc_queue(&ctrl->queues[0]);

The nvme_rdma_alloc_queue prototype change would be a nice prep patch to split out from the main change.

done. Separated the patches in v3.