nvme-rdma: parallelize I/O queue setup
v4 cover nvme-rdma: parallelize I/O queue setup
This patch series parallelizes nvme-rdma connection and reconnection by setting up I/O queues in parallel instead of sequentially. Allocation and startup of each queue are combined into a single per-queue async work item, so per-queue connection latency overlaps across all queues. This matters most on high-core-count hosts with many I/O queues, where serial setup dominates connect time.
Patch 1 is a preparatory refactor: nvme_rdma_alloc_queue() takes a queue pointer so allocation and startup can be folded into a single async worker.
Patch 2 contains the implementation for async setup of I/O queues.
Testing on a 64-core host with 64 I/O queues shows nvme-rdma connection time reduced from ~1.4s to 416ms.
---
Changes from v3->v4:
- Fixed formatting.
- Replaced kmalloc_array with kmalloc_objs.
Changes from v2->v3:
- Split the series into two patches: extract the nvme_rdma_alloc_queue() refactor into a separate preparatory patch.
- Replace the atomic error flag in struct nvme_rdma_ctrl with a per-work nvme_rdma_setup_ctx { queue, err } struct.
- Fix formatting changes regarding line overflow indentation and nesting.
Changes from v1->v2:
- Remove separate workqueue and use the async API instead.
Previous versions:
v1: https://lore.kernel.org/all/[email protected]/
v2: https://lore.kernel.org/all/[email protected]/
v3: https://lore.kernel.org/all/[email protected]/
Surabhi Gogte (2):
nvme-rdma: refactor nvme_rdma_alloc_queue() to take a queue pointer
nvme-rdma: parallelize I/O queue allocation and startup
drivers/nvme/host/rdma.c | 136 ++++++++++++++++++++++++--------------- 1 file changed, 83 insertions(+), 53 deletions(-)
Signature · 1 line
2.54.0
PATCH v4 1/2 nvme-rdma: refactor nvme_rdma_alloc_queue() to take a queue pointer
Callers are responsible for initializing queue->ctrl and queue->queue_size before calling nvme_rdma_alloc_queue(), which now derives ctrl and idx from the queue pointer directly. This removes redundant assignments inside the function and simplifies the interface.
--- drivers/nvme/host/rdma.c | 19 +++++++++---------- 1 file changed, 9 insertions(+), 10 deletions(-)
drivers/nvme/host/rdma.c +9 -10
@@ -566,16 +566,14 @@ static int nvme_rdma_create_queue_ib(struct nvme_rdma_queue *queue)return ret; } -static int nvme_rdma_alloc_queue (struct nvme_rdma_ctrl *ctrl, -int idx,size_t queue_size) +static int nvme_rdma_alloc_queue (struct nvme_rdma_queue *queue) { -struct nvme_rdma_queue *queue; +struct nvme_rdma_ctrl *ctrl = queue->ctrl; +int idx =nvme_rdma_queue_idx (queue);struct sockaddr *src_addr =NULL ;int ret; - queue = &ctrl->queues[idx];mutex_init (&queue->queue_lock); - queue->ctrl = ctrl;if (idx && ctrl->ctrl.max_integrity_segments) queue->pi_support = true;else
@@ -587,8 +585,6 @@ static int nvme_rdma_alloc_queue(struct nvme_rdma_ctrl *ctrl,else queue->cmnd_capsule_len =sizeof (struct nvme_command ); - queue->queue_size = queue_size; - queue->cm_id =rdma_create_id (&init_net, nvme_rdma_cm_handler, queue,RDMA_PS_TCP ,IB_QPT_RC );if (IS_ERR (queue->cm_id)) {
@@ -736,8 +732,9 @@ static int nvme_rdma_alloc_io_queues(struct nvme_rdma_ctrl *ctrl)nvmf_set_io_queues (opts, nr_io_queues, ctrl->io_queues);for (i =1 ; i < ctrl->ctrl.queue_count; i++) { - ret =nvme_rdma_alloc_queue (ctrl, i, - ctrl->ctrl.sqsize +1 ); + ctrl->queues[i].ctrl = ctrl; + ctrl->queues[i].queue_size = ctrl->ctrl.sqsize +1 ; + ret =nvme_rdma_alloc_queue (&ctrl->queues[i]);if (ret) goto out_free_queues; }
@@ -783,7 +780,9 @@ static int nvme_rdma_configure_admin_queue(struct nvme_rdma_ctrl *ctrl,bool pi_capable = false;int error; - error =nvme_rdma_alloc_queue (ctrl,0 ,NVME_AQ_DEPTH ); + ctrl->queues[0 ].ctrl = ctrl; + ctrl->queues[0 ].queue_size =NVME_AQ_DEPTH ; + error =nvme_rdma_alloc_queue (&ctrl->queues[0 ]);if (error)return error;
Signature · 1 line
2.54.0
PATCH v4 2/2 nvme-rdma: parallelize I/O queue allocation and startup
In reply to Surabhi GogteRefactor nvme rdma I/O queue setup to use async API, combining allocation and startup into a single parallel operation per queue. This reduces connection and reconnection setup time when there are delays in establishing connections, which is especially important for high-core-count hosts.
Key changes:
- Use async API to facilitate parallel calls for io queue setup.
- Add nvme_rdma_setup_ctx for propagating errors from async workers.
- Remove nvme_rdma_alloc_io_queues() and nvme_rdma_start_io_queues(); their logic is folded into nvme_rdma_setup_io_queues() and nvme_rdma_configure_io_queues().
- Move queue count negotiation (nvme_set_queue_count, nvmf_set_io_queues) from the removed nvme_rdma_alloc_io_queues() into nvme_rdma_configure_io_queues().
Testing on a 64-core host with 64 IO-queues shows nvme-rdma connection time reduced from ~1.4s to 416ms.
--- drivers/nvme/host/rdma.c | 124 ++++++++++++++++++++++++--------------- 1 file changed, 77 insertions(+), 47 deletions(-)
drivers/nvme/host/rdma.c +77 -47
@@ -16,6 +16,7 @@#include <linux/types.h> #include <linux/list.h> #include <linux/mutex.h> +#include <linux/async.h> #include <linux/scatterlist.h> #include <linux/nvme.h> #include <linux/unaligned.h>
@@ -100,6 +101,11 @@ struct nvme_rdma_queue {struct mutex queue_lock; }; +struct nvme_rdma_setup_ctx { +struct nvme_rdma_queue *queue; +int *err; +}; +struct nvme_rdma_ctrl {/* read only in the hot path */ struct nvme_rdma_queue *queues;
@@ -690,60 +696,68 @@ static int nvme_rdma_start_queue(struct nvme_rdma_ctrl *ctrl, int idx)return ret; } -static int nvme_rdma_start_io_queues (struct nvme_rdma_ctrl *ctrl, -int first,int last) +static void nvme_rdma_setup_queue_async (void *data,async_cookie_t cookie) { -int i, ret =0 ; +struct nvme_rdma_setup_ctx *ctx = data; +struct nvme_rdma_queue *queue; +int ret; -for (i = first; i < last; i++) { - ret =nvme_rdma_start_queue (ctrl, i); -if (ret) - goto out_stop_queues; - } + queue = ctx->queue; + ret =nvme_rdma_alloc_queue (queue); +if (ret) + goto out_err; -return 0 ; + ret =nvme_rdma_start_queue (queue->ctrl,nvme_rdma_queue_idx (queue)); +if (ret) + goto out_err; -out_stop_queues: -for (i--; i >= first; i--) -nvme_rdma_stop_queue (&ctrl->queues[i]); -return ret; +return ; +out_err: +WRITE_ONCE (*ctx->err, ret); } -static int nvme_rdma_alloc_io_queues (struct nvme_rdma_ctrl *ctrl) +static int nvme_rdma_setup_io_queues (struct nvme_rdma_ctrl *ctrl, +unsigned int first,unsigned int last,size_t queue_size) { -struct nvmf_ctrl_options *opts = ctrl->ctrl.opts; -unsigned int nr_io_queues; -int i, ret; - - nr_io_queues =nvmf_nr_io_queues (opts); - ret =nvme_set_queue_count (&ctrl->ctrl, &nr_io_queues); -if (ret) -return ret; +ASYNC_DOMAIN_EXCLUSIVE (queue_domain); +struct nvme_rdma_setup_ctx *ctxs; +int nr_queues = last - first; +int err =0 , i, ret; -if (nr_io_queues ==0 ) { -dev_err (ctrl->ctrl.device, -"unable to set any I/O queues\n" ); + ctxs =kmalloc_objs (*ctxs, nr_queues); +if (!ctxs)return -ENOMEM ; - } - ctrl->ctrl.queue_count = nr_io_queues +1 ; -dev_info (ctrl->ctrl.device, -"creating %d I/O queues.\n" , nr_io_queues); - -nvmf_set_io_queues (opts, nr_io_queues, ctrl->io_queues); -for (i =1 ; i < ctrl->ctrl.queue_count; i++) { - ctrl->queues[i].ctrl = ctrl; - ctrl->queues[i].queue_size = ctrl->ctrl.sqsize +1 ; - ret =nvme_rdma_alloc_queue (&ctrl->queues[i]); -if (ret) - goto out_free_queues; +for (i =0 ; i < nr_queues; i++) { +struct nvme_rdma_queue *queue = &ctrl->queues[first + i]; + + queue->ctrl = ctrl; + queue->queue_size = queue_size; + + ctxs[i].queue = queue; + ctxs[i].err = &err; +async_schedule_domain (nvme_rdma_setup_queue_async, &ctxs[i], + &queue_domain); } -return 0 ; +async_synchronize_full_domain (&queue_domain); +kfree (ctxs); + ret =READ_ONCE (err); +if (ret) + goto out_free_queues; + +return 0 ; out_free_queues: -for (i--; i >=1 ; i--) -nvme_rdma_free_queue (&ctrl->queues[i]); +for (i =0 ; i < nr_queues; i++) { +struct nvme_rdma_queue *queue = + &ctrl->queues[first + i]; + +if (test_bit (NVME_RDMA_Q_LIVE , &queue->flags)) +nvme_rdma_stop_queue (queue); +if (test_bit (NVME_RDMA_Q_ALLOCATED , &queue->flags)) +nvme_rdma_free_queue (queue); + }return ret; }
@@ -862,12 +876,23 @@ static int nvme_rdma_configure_admin_queue(struct nvme_rdma_ctrl *ctrl,static int nvme_rdma_configure_io_queues (struct nvme_rdma_ctrl *ctrl,bool new) { +unsigned int nr_io_queues;int ret, nr_queues; - ret =nvme_rdma_alloc_io_queues (ctrl); + nr_io_queues =nvmf_nr_io_queues (ctrl->ctrl.opts); + ret =nvme_set_queue_count (&ctrl->ctrl, &nr_io_queues);if (ret)return ret; +if (nr_io_queues ==0 ) { +dev_err (ctrl->ctrl.device,"unable to set any I/O queues\n" ); +return -ENOMEM ; + } + + ctrl->ctrl.queue_count = nr_io_queues +1 ; +dev_info (ctrl->ctrl.device,"creating %d I/O queues.\n" , nr_io_queues); +nvmf_set_io_queues (ctrl->ctrl.opts, nr_io_queues, ctrl->io_queues); +if (new) { ret =nvme_rdma_alloc_tag_set (&ctrl->ctrl);if (ret)
@@ -880,7 +905,9 @@ static int nvme_rdma_configure_io_queues(struct nvme_rdma_ctrl *ctrl, bool new)* queue number might have changed. */ nr_queues =min (ctrl->tag_set.nr_hw_queues +1 , ctrl->ctrl.queue_count); - ret =nvme_rdma_start_io_queues (ctrl,1 , nr_queues); + ret =nvme_rdma_setup_io_queues (ctrl,1 , nr_queues, + ctrl->ctrl.sqsize +1 ); +if (ret) goto out_cleanup_tagset;
@@ -904,12 +931,15 @@ static int nvme_rdma_configure_io_queues(struct nvme_rdma_ctrl *ctrl, bool new)/* * If the number of queues has increased (reconnect case) -* start all new queues now. +* setup all new queues now. */ - ret =nvme_rdma_start_io_queues (ctrl, nr_queues, - ctrl->tag_set.nr_hw_queues +1 ); -if (ret) - goto out_wait_freeze_timed_out; +if (ctrl->tag_set.nr_hw_queues +1 > nr_queues) { + ret =nvme_rdma_setup_io_queues (ctrl, nr_queues, + ctrl->tag_set.nr_hw_queues +1 , + ctrl->ctrl.sqsize +1 ); +if (ret) + goto out_wait_freeze_timed_out; + }return 0 ;
Signature · 1 line
2.54.0
Re: [PATCH v4 2/2] nvme-rdma: parallelize I/O queue allocation and startup
Looks good:
Quoted 6 lines · Surabhi Gogte
This patch series parallelizes nvme-rdma connection and reconnection by setting up I/O queues in parallel instead of sequentially. Allocation and startup of each queue are combined into a single per-queue async work item, so per-queue connection latency overlaps across all queues. This matters most on high-core-count hosts with many I/O queues, where serial setup dominates connect time.
Thanks, applied to nvme-7.3.
Sagi Grimberg
maintainerQuoted 11 lines · Surabhi Gogte
This patch series parallelizes nvme-rdma connection and reconnection by setting up I/O queues in parallel instead of sequentially. Allocation and startup of each queue are combined into a single per-queue async work item, so per-queue connection latency overlaps across all queues. This matters most on high-core-count hosts with many I/O queues, where serial setup dominates connect time.
Patch 1 is a preparatory refactor:
nvme_rdma_alloc_queue()takes a queue pointer so allocation and startup can be folded into a single async worker.
Quoted 6 lines
Patch 2 contains the implementation for async setup of I/O queues.
Testing on a 64-core host with 64 I/O queues shows nvme-rdma connection time reduced from ~1.4s to 416ms.
Signed-off-by Surabhi Gogte <[email protected]>
Would it be possible to do the same for nvme-tcp?