nvme-rdma: parallelize I/O queue allocation and startup
PATCH 1/1 nvme-rdma: parallelize I/O queue allocation and startup
Refactor nvme rdma I/O queue setup to use parallel work queues, combining allocation and startup into a single parallel operation per queue. This reduces connection and reconnection setup time when there are delays in establishing connections, which is especially important for high-core-count hosts.
Key changes:
- Use a dedicated nvme_setup_wq for per-queue setup work instead of nvme_wq or nvme_reset_wq. Since reconnect and reset paths run as workers on those queues, calling flush_work() on setup items queued to the same workqueue would trigger lockdep's recursive-flush detection and risk deadlock under worker-pool exhaustion.
- Add setup_work and queue_idx fields to nvme_rdma_queue to enable per-queue workqueue dispatch; add qsetup_err atomic to nvme_rdma_ctrl for collecting errors across parallel workers.
- Refactor nvme_rdma_alloc_queue() to accept a pre-initialized queue pointer instead of (ctrl, idx, queue_size), updating all call sites including the admin queue path.
- Remove nvme_rdma_alloc_io_queues() and nvme_rdma_start_io_queues(); their logic is folded into nvme_rdma_setup_io_queues() and nvme_rdma_configure_io_queues().
- Move queue count negotiation (nvme_set_queue_count, nvmf_set_io_queues) from the removed nvme_rdma_alloc_io_queues() into nvme_rdma_configure_io_queues().
- Introduce nvme_rdma_setup_io_queues() to dispatch alloc+start per queue in parallel and collect errors atomically.
Testing on a 64-core host with 64 IO-queues shows nvme-rdma connection time reduced from ~1.4s to 416ms.
--- drivers/nvme/host/core.c | 15 ++++- drivers/nvme/host/nvme.h | 1 + drivers/nvme/host/rdma.c | 127 ++++++++++++++++++++++----------------- 3 files changed, 86 insertions(+), 57 deletions(-)
drivers/nvme/host/core.c +13 -2
@@ -107,7 +107,8 @@ MODULE_PARM_DESC(disable_pi_offsets, "disable protection information if it has an offset");/* -* nvme_wq - hosts nvme related works that are not reset or delete +* nvme_wq - hosts nvme related works that are not queue setup, reset or delete +* nvme_setup_wq - hosts nvme queue setup works * nvme_reset_wq - hosts nvme reset works * nvme_delete_wq - hosts nvme delete works *
@@ -126,6 +127,9 @@ EXPORT_SYMBOL_GPL(nvme_reset_wq);struct workqueue_struct *nvme_delete_wq;EXPORT_SYMBOL_GPL (nvme_delete_wq); +struct workqueue_struct *nvme_setup_wq; +EXPORT_SYMBOL_GPL (nvme_setup_wq); +static LIST_HEAD (nvme_subsystems);DEFINE_MUTEX (nvme_subsystems_lock);
@@ -5415,10 +5419,14 @@ static int __init nvme_core_init(void)if (!nvme_delete_wq) goto destroy_reset_wq; + nvme_setup_wq =alloc_workqueue ("nvme-setup-wq" , wq_flags,0 ); +if (!nvme_setup_wq) + goto destroy_delete_wq; + result =alloc_chrdev_region (&nvme_ctrl_base_chr_devt,0 ,NVME_MINORS ,"nvme" );if (result <0 ) - goto destroy_delete_wq; + goto destroy_setup_wq; result =class_register (&nvme_class);if (result)
@@ -5452,6 +5460,8 @@ static int __init nvme_core_init(void)class_unregister (&nvme_class); unregister_chrdev:unregister_chrdev_region (nvme_ctrl_base_chr_devt,NVME_MINORS ); +destroy_setup_wq: +destroy_workqueue (nvme_setup_wq); destroy_delete_wq:destroy_workqueue (nvme_delete_wq); destroy_reset_wq:
@@ -5470,6 +5480,7 @@ static void __exit nvme_core_exit(void)class_unregister (&nvme_class);unregister_chrdev_region (nvme_ns_chr_devt,NVME_MINORS );unregister_chrdev_region (nvme_ctrl_base_chr_devt,NVME_MINORS ); +destroy_workqueue (nvme_setup_wq);destroy_workqueue (nvme_delete_wq);destroy_workqueue (nvme_reset_wq);destroy_workqueue (nvme_wq);
drivers/nvme/host/nvme.h +1 -0
@@ -49,6 +49,7 @@ extern unsigned int admin_timeout;extern struct workqueue_struct *nvme_wq;extern struct workqueue_struct *nvme_reset_wq;extern struct workqueue_struct *nvme_delete_wq; +extern struct workqueue_struct *nvme_setup_wq;extern struct mutex nvme_subsystems_lock;/*
drivers/nvme/host/rdma.c +72 -55
204 lines not on this page · raw message
Signature · 1 line
2.54.0
Keith Busch
maintainerQuoted 5 lines · Surabhi Gogte
Refactor nvme rdma I/O queue setup to use parallel work queues, combining allocation and startup into a single parallel operation per queue. This reduces connection and reconnection setup time when there are delays in establishing connections, which is especially important for high-core-count hosts.
Seems like a good idea.
drivers/nvme/host/core.c EXPORT_SYMBOL_GPL(nvme_reset_wq); · line 131
Quoted 2 lines
drivers/nvme/host/core.c +2
-0+struct workqueue_struct *nvme_setup_wq; +EXPORT_SYMBOL_GPL (nvme_setup_wq);
RDMA is the only one using it, so I don't think we want burden the common core with this. But I also think a new workqueue is overkill. Just use the async APIs. Suggestions below:
drivers/nvme/host/rdma.c static int nvme_rdma_start_queue(struct nvme_rdma_ctrl *ctrl, int idx) · line 699
Quoted 5 lines
drivers/nvme/host/rdma.c +3
-1+static void nvme_rdma_setup_queue_work (struct work_struct *work) { -int i, ret =0 ; +struct nvme_rdma_queue *queue = + container_of(work, struct nvme_rdma_queue, setup_work);
static void nvme_rdma_setup_queue_async(void *data, async_cookie_t cookie)
{
struct nvme_rdma_queue *queue = data;
...
}Quoted 3 lines
drivers/nvme/host/rdma.c +2
-0+static int nvme_rdma_setup_io_queues (struct nvme_rdma_ctrl *ctrl,int first, +int last,size_t queue_size) {
...
Quoted 10 lines
drivers/nvme/host/rdma.c +3
-6+for (i =0 ; i < nr_queues; i++) { +struct nvme_rdma_queue *queue = &ctrl->queues[first + i]; -nvmf_set_io_queues (opts, nr_io_queues, ctrl->io_queues); -for (i =1 ; i < ctrl->ctrl.queue_count; i++) { - ret =nvme_rdma_alloc_queue (ctrl, i, - ctrl->ctrl.sqsize +1 ); -if (ret) - goto out_free_queues; + queue->ctrl = ctrl;
drivers/nvme/host/rdma.c static int nvme_rdma_start_queue(struct nvme_rdma_ctrl *ctrl, int idx)
Quoted 10 lines
drivers/nvme/host/rdma.c +6
-1+ queue->queue_idx = first + i; + queue->queue_size = queue_size; +INIT_WORK (&queue->setup_work, nvme_rdma_setup_queue_work); +queue_work (nvme_setup_wq, &queue->setup_work); } -return 0 ; +for (i =0 ; i < nr_queues; i++) +flush_work (&ctrl->queues[first + i].setup_work);
And this part can be:
ASYNC_DOMAIN_EXCLUSIVE(domain);
...
for (i = 0; i < nr_queues; i++) {
...
async_schedule_domain(nvme_rdma_setup_queue_async, queue,
&domain);
}
async_synchronize_full_domain(&domain);It's simpler this way.
Quoted 10 lines · Keith Busch
Quoted 5 lines · Surabhi Gogte
Refactor nvme rdma I/O queue setup to use parallel work queues, combining allocation and startup into a single parallel operation per queue. This reduces connection and reconnection setup time when there are delays in establishing connections, which is especially important for high-core-count hosts.
Seems like a good idea.
Thank you for the review!
Quoted 7 lines
drivers/nvme/host/core.c EXPORT_SYMBOL_GPL(nvme_reset_wq); · line 131
Quoted 2 lines
drivers/nvme/host/core.c +2
-0+struct workqueue_struct *nvme_setup_wq; +EXPORT_SYMBOL_GPL (nvme_setup_wq);RDMA is the only one using it, so I don't think we want burden the common core with this. But I also think a new workqueue is overkill. Just use the async APIs. Suggestions below:
Plan was to extend nvme_setup_wq for other transports later. But we can use async APIs for that too. I will send a v2 version.