Skip to contentSkip to conversation

nvme-rdma: parallelize I/O queue allocation and startup

PATCH Superseded Surabhi Gogte 2 participants 3 messages May 29 – Jun 2 NVMe

v1 of 4 Superseded Latest is v4 →

lore
Share & tools
lore
#0 lore raw

PATCH 1/1 nvme-rdma: parallelize I/O queue allocation and startup

Refactor nvme rdma I/O queue setup to use parallel work queues, combining allocation and startup into a single parallel operation per queue. This reduces connection and reconnection setup time when there are delays in establishing connections, which is especially important for high-core-count hosts.

Key changes:
- Use a dedicated nvme_setup_wq for per-queue setup work instead of nvme_wq or nvme_reset_wq. Since reconnect and reset paths run as workers on those queues, calling flush_work() on setup items queued to the same workqueue would trigger lockdep's recursive-flush detection and risk deadlock under worker-pool exhaustion.
- Add setup_work and queue_idx fields to nvme_rdma_queue to enable per-queue workqueue dispatch; add qsetup_err atomic to nvme_rdma_ctrl for collecting errors across parallel workers.
- Refactor nvme_rdma_alloc_queue() to accept a pre-initialized queue pointer instead of (ctrl, idx, queue_size), updating all call sites including the admin queue path.
- Remove nvme_rdma_alloc_io_queues() and nvme_rdma_start_io_queues(); their logic is folded into nvme_rdma_setup_io_queues() and nvme_rdma_configure_io_queues().
- Move queue count negotiation (nvme_set_queue_count, nvmf_set_io_queues) from the removed nvme_rdma_alloc_io_queues() into nvme_rdma_configure_io_queues().
- Introduce nvme_rdma_setup_io_queues() to dispatch alloc+start per queue in parallel and collect errors atomically.

Testing on a 64-core host with 64 IO-queues shows nvme-rdma connection time reduced from ~1.4s to 416ms.

---
 drivers/nvme/host/core.c |  15 ++++-
 drivers/nvme/host/nvme.h |   1 +
 drivers/nvme/host/rdma.c | 127 ++++++++++++++++++++++-----------------
 3 files changed, 86 insertions(+), 57 deletions(-)
drivers/nvme/host/core.c +13 -2
index dc388e24caad..f1247a61c797 100644
@@ -107,7 +107,8 @@ MODULE_PARM_DESC(disable_pi_offsets,
 	"disable protection information if it has an offset");
 
 /*
- * nvme_wq - hosts nvme related works that are not reset or delete
+ * nvme_wq - hosts nvme related works that are not queue setup, reset or delete
+ * nvme_setup_wq - hosts nvme queue setup works
  * nvme_reset_wq - hosts nvme reset works
  * nvme_delete_wq - hosts nvme delete works
  *
@@ -126,6 +127,9 @@ EXPORT_SYMBOL_GPL(nvme_reset_wq);
 struct workqueue_struct *nvme_delete_wq;
 EXPORT_SYMBOL_GPL(nvme_delete_wq);
 
+struct workqueue_struct *nvme_setup_wq;
+EXPORT_SYMBOL_GPL(nvme_setup_wq);
+
 static LIST_HEAD(nvme_subsystems);
 DEFINE_MUTEX(nvme_subsystems_lock);
 
@@ -5415,10 +5419,14 @@ static int __init nvme_core_init(void)
 	if (!nvme_delete_wq)
 		goto destroy_reset_wq;
 
+	nvme_setup_wq = alloc_workqueue("nvme-setup-wq", wq_flags, 0);
+	if (!nvme_setup_wq)
+		goto destroy_delete_wq;
+
 	result = alloc_chrdev_region(&nvme_ctrl_base_chr_devt, 0,
 			NVME_MINORS, "nvme");
 	if (result < 0)
-		goto destroy_delete_wq;
+		goto destroy_setup_wq;
 
 	result = class_register(&nvme_class);
 	if (result)
@@ -5452,6 +5460,8 @@ static int __init nvme_core_init(void)
 	class_unregister(&nvme_class);
 unregister_chrdev:
 	unregister_chrdev_region(nvme_ctrl_base_chr_devt, NVME_MINORS);
+destroy_setup_wq:
+	destroy_workqueue(nvme_setup_wq);
 destroy_delete_wq:
 	destroy_workqueue(nvme_delete_wq);
 destroy_reset_wq:
@@ -5470,6 +5480,7 @@ static void __exit nvme_core_exit(void)
 	class_unregister(&nvme_class);
 	unregister_chrdev_region(nvme_ns_chr_devt, NVME_MINORS);
 	unregister_chrdev_region(nvme_ctrl_base_chr_devt, NVME_MINORS);
+	destroy_workqueue(nvme_setup_wq);
 	destroy_workqueue(nvme_delete_wq);
 	destroy_workqueue(nvme_reset_wq);
 	destroy_workqueue(nvme_wq);
drivers/nvme/host/nvme.h +1 -0
index ccd5e05dac98..9cc49f5ad8af 100644
@@ -49,6 +49,7 @@ extern unsigned int admin_timeout;
 extern struct workqueue_struct *nvme_wq;
 extern struct workqueue_struct *nvme_reset_wq;
 extern struct workqueue_struct *nvme_delete_wq;
+extern struct workqueue_struct *nvme_setup_wq;
 extern struct mutex nvme_subsystems_lock;
 
 /*
drivers/nvme/host/rdma.c +72 -55

204 lines not on this page · raw message

Signature · 1 line
2.54.0
maintainer
#1 lore raw
Quoted 5 lines · Surabhi Gogte

Refactor nvme rdma I/O queue setup to use parallel work queues, combining allocation and startup into a single parallel operation per queue. This reduces connection and reconnection setup time when there are delays in establishing connections, which is especially important for high-core-count hosts.

Seems like a good idea.

drivers/nvme/host/core.c EXPORT_SYMBOL_GPL(nvme_reset_wq); · line 131

Quoted 2 lines
drivers/nvme/host/core.c +2 -0
+struct workqueue_struct *nvme_setup_wq;
+EXPORT_SYMBOL_GPL(nvme_setup_wq);

RDMA is the only one using it, so I don't think we want burden the common core with this. But I also think a new workqueue is overkill. Just use the async APIs. Suggestions below:

drivers/nvme/host/rdma.c static int nvme_rdma_start_queue(struct nvme_rdma_ctrl *ctrl, int idx) · line 699

Quoted 5 lines
drivers/nvme/host/rdma.c +3 -1
+static void nvme_rdma_setup_queue_work(struct work_struct *work)
 {
-	int i, ret = 0;
+	struct nvme_rdma_queue *queue =
+		container_of(work, struct nvme_rdma_queue, setup_work);

static void nvme_rdma_setup_queue_async(void *data, async_cookie_t cookie)

{
	struct nvme_rdma_queue *queue = data;
	...
}
Quoted 3 lines
drivers/nvme/host/rdma.c +2 -0
+static int nvme_rdma_setup_io_queues(struct nvme_rdma_ctrl *ctrl, int first,
+				     int last, size_t queue_size)
 {

...

Quoted 10 lines
drivers/nvme/host/rdma.c +3 -6
+	for (i = 0; i < nr_queues; i++) {
+		struct nvme_rdma_queue *queue = &ctrl->queues[first + i];
 
-	nvmf_set_io_queues(opts, nr_io_queues, ctrl->io_queues);
-	for (i = 1; i < ctrl->ctrl.queue_count; i++) {
-		ret = nvme_rdma_alloc_queue(ctrl, i,
-				ctrl->ctrl.sqsize + 1);
-		if (ret)
-			goto out_free_queues;
+		queue->ctrl = ctrl;

drivers/nvme/host/rdma.c static int nvme_rdma_start_queue(struct nvme_rdma_ctrl *ctrl, int idx)

Quoted 10 lines
drivers/nvme/host/rdma.c +6 -1
+		queue->queue_idx = first + i;
+		queue->queue_size = queue_size;
+		INIT_WORK(&queue->setup_work, nvme_rdma_setup_queue_work);
+		queue_work(nvme_setup_wq, &queue->setup_work);
 	}
 
 
-	return 0;
+	for (i = 0; i < nr_queues; i++)
+		flush_work(&ctrl->queues[first + i].setup_work);

And this part can be:

	ASYNC_DOMAIN_EXCLUSIVE(domain);
	...
	for (i = 0; i < nr_queues; i++) {
		...
		async_schedule_domain(nvme_rdma_setup_queue_async, queue,
			&domain);
        }

        async_synchronize_full_domain(&domain);

It's simpler this way.

#2 lore raw
Quoted 10 lines · Keith Busch
Quoted 5 lines · Surabhi Gogte

Refactor nvme rdma I/O queue setup to use parallel work queues, combining allocation and startup into a single parallel operation per queue. This reduces connection and reconnection setup time when there are delays in establishing connections, which is especially important for high-core-count hosts.

Seems like a good idea.

Thank you for the review!

Quoted 7 lines

drivers/nvme/host/core.c EXPORT_SYMBOL_GPL(nvme_reset_wq); · line 131

Quoted 2 lines
drivers/nvme/host/core.c +2 -0
+struct workqueue_struct *nvme_setup_wq;
+EXPORT_SYMBOL_GPL(nvme_setup_wq);

RDMA is the only one using it, so I don't think we want burden the common core with this. But I also think a new workqueue is overkill. Just use the async APIs. Suggestions below:

Plan was to extend nvme_setup_wq for other transports later. But we can use async APIs for that too. I will send a v2 version.