nvme-rdma: parallelize I/O queue allocation and startup
PATCH v2 1/1 nvme-rdma: parallelize I/O queue allocation and startup
Refactor nvme rdma I/O queue setup to use async API, combining allocation and startup into a single parallel operation per queue. This reduces connection and reconnection setup time when there are delays in establishing connections, which is especially important for high-core-count hosts.
Key changes:
- Use async API to facilitate parallel calls for io queue setup.
- Add qsetup_err atomic to nvme_rdma_ctrl for collecting errors across parallel threads.
- Refactor nvme_rdma_alloc_queue() to accept a pre-initialized queue pointer instead of (ctrl, idx, queue_size), updating all call sites including the admin queue path.
- Remove nvme_rdma_alloc_io_queues() and nvme_rdma_start_io_queues(); their logic is folded into nvme_rdma_setup_io_queues() and nvme_rdma_configure_io_queues().
- Move queue count negotiation (nvme_set_queue_count, nvmf_set_io_queues) from the removed nvme_rdma_alloc_io_queues() into nvme_rdma_configure_io_queues().
Testing on a 64-core host with 64 IO-queues shows nvme-rdma connection time reduced from ~1.4s to 416ms.
---
Changes since v1:
- Replace dedicated nvme_setup_wq workqueue with async API implementation.
--- drivers/nvme/host/rdma.c | 122 ++++++++++++++++++++++----------------- 1 file changed, 68 insertions(+), 54 deletions(-)
drivers/nvme/host/rdma.c +68 -54
@@ -16,6 +16,7 @@#include <linux/types.h> #include <linux/list.h> #include <linux/mutex.h> +#include <linux/async.h> #include <linux/scatterlist.h> #include <linux/nvme.h> #include <linux/unaligned.h>
@@ -125,6 +126,7 @@ struct nvme_rdma_ctrl {struct nvme_ctrl ctrl;bool use_inline_data;u32 io_queues[HCTX_MAX_TYPES ]; +atomic_t qsetup_err; };static inline struct nvme_rdma_ctrl *to_rdma_ctrl (struct nvme_ctrl *ctrl)
@@ -566,16 +568,14 @@ static int nvme_rdma_create_queue_ib(struct nvme_rdma_queue *queue)return ret; } -static int nvme_rdma_alloc_queue (struct nvme_rdma_ctrl *ctrl, -int idx,size_t queue_size) +static int nvme_rdma_alloc_queue (struct nvme_rdma_queue *queue) { -struct nvme_rdma_queue *queue; +struct nvme_rdma_ctrl *ctrl = queue->ctrl; +int idx =nvme_rdma_queue_idx (queue);struct sockaddr *src_addr =NULL ;int ret; - queue = &ctrl->queues[idx];mutex_init (&queue->queue_lock); - queue->ctrl = ctrl;if (idx && ctrl->ctrl.max_integrity_segments) queue->pi_support = true;else
@@ -587,8 +587,6 @@ static int nvme_rdma_alloc_queue(struct nvme_rdma_ctrl *ctrl,else queue->cmnd_capsule_len =sizeof (struct nvme_command ); - queue->queue_size = queue_size; - queue->cm_id =rdma_create_id (&init_net, nvme_rdma_cm_handler, queue,RDMA_PS_TCP ,IB_QPT_RC );if (IS_ERR (queue->cm_id)) {
@@ -694,59 +692,57 @@ static int nvme_rdma_start_queue(struct nvme_rdma_ctrl *ctrl, int idx)return ret; } -static int nvme_rdma_start_io_queues (struct nvme_rdma_ctrl *ctrl, -int first,int last) +static void nvme_rdma_setup_queue_async (void *setup_queue,async_cookie_t cookie) { -int i, ret =0 ; +struct nvme_rdma_queue *queue = setup_queue; +int ret; -for (i = first; i < last; i++) { - ret =nvme_rdma_start_queue (ctrl, i); -if (ret) - goto out_stop_queues; - } + ret =nvme_rdma_alloc_queue (queue); +if (ret) + goto out_err; -return 0 ; + ret =nvme_rdma_start_queue (queue->ctrl,nvme_rdma_queue_idx (queue)); +if (ret) + goto out_err; -out_stop_queues: -for (i--; i >= first; i--) -nvme_rdma_stop_queue (&ctrl->queues[i]); -return ret; +return ; + +out_err: +atomic_cmpxchg (&queue->ctrl->qsetup_err,0 , ret); } -static int nvme_rdma_alloc_io_queues (struct nvme_rdma_ctrl *ctrl) +static int nvme_rdma_setup_io_queues (struct nvme_rdma_ctrl *ctrl,int first, +int last,size_t queue_size) { -struct nvmf_ctrl_options *opts = ctrl->ctrl.opts; -unsigned int nr_io_queues; +int nr_queues = last - first;int i, ret; +ASYNC_DOMAIN_EXCLUSIVE (setup_queue_domain); - nr_io_queues =nvmf_nr_io_queues (opts); - ret =nvme_set_queue_count (&ctrl->ctrl, &nr_io_queues); -if (ret) -return ret; - -if (nr_io_queues ==0 ) { -dev_err (ctrl->ctrl.device, -"unable to set any I/O queues\n" ); -return -ENOMEM ; - } +atomic_set (&ctrl->qsetup_err,0 ); - ctrl->ctrl.queue_count = nr_io_queues +1 ; -dev_info (ctrl->ctrl.device, -"creating %d I/O queues.\n" , nr_io_queues); +for (i =0 ; i < nr_queues; i++) { +struct nvme_rdma_queue *queue = &ctrl->queues[first + i]; -nvmf_set_io_queues (opts, nr_io_queues, ctrl->io_queues); -for (i =1 ; i < ctrl->ctrl.queue_count; i++) { - ret =nvme_rdma_alloc_queue (ctrl, i, - ctrl->ctrl.sqsize +1 ); -if (ret) - goto out_free_queues; + queue->ctrl = ctrl; + queue->queue_size = queue_size; +async_schedule_domain (nvme_rdma_setup_queue_async, queue, + &setup_queue_domain); } -return 0 ; +async_synchronize_full_domain (&setup_queue_domain); -out_free_queues: -for (i--; i >=1 ; i--) -nvme_rdma_free_queue (&ctrl->queues[i]); + ret =atomic_read (&ctrl->qsetup_err); +if (ret) { +for (i =0 ; i < nr_queues; i++) { +struct nvme_rdma_queue *queue = + &ctrl->queues[first + i]; + +if (test_bit (NVME_RDMA_Q_LIVE , &queue->flags)) +nvme_rdma_stop_queue (queue); +if (test_bit (NVME_RDMA_Q_ALLOCATED , &queue->flags)) +nvme_rdma_free_queue (queue); + } + }return ret; }
@@ -783,7 +779,9 @@ static int nvme_rdma_configure_admin_queue(struct nvme_rdma_ctrl *ctrl,bool pi_capable = false;int error; - error =nvme_rdma_alloc_queue (ctrl,0 ,NVME_AQ_DEPTH ); + ctrl->queues[0 ].ctrl = ctrl; + ctrl->queues[0 ].queue_size =NVME_AQ_DEPTH ; + error =nvme_rdma_alloc_queue (&ctrl->queues[0 ]);if (error)return error;
@@ -864,11 +862,22 @@ static int nvme_rdma_configure_admin_queue(struct nvme_rdma_ctrl *ctrl,static int nvme_rdma_configure_io_queues (struct nvme_rdma_ctrl *ctrl,bool new) {int ret, nr_queues; +unsigned int nr_io_queues; - ret =nvme_rdma_alloc_io_queues (ctrl); + nr_io_queues =nvmf_nr_io_queues (ctrl->ctrl.opts); + ret =nvme_set_queue_count (&ctrl->ctrl, &nr_io_queues);if (ret)return ret; +if (nr_io_queues ==0 ) { +dev_err (ctrl->ctrl.device,"unable to set any I/O queues\n" ); +return -ENOMEM ; + } + + ctrl->ctrl.queue_count = nr_io_queues +1 ; +dev_info (ctrl->ctrl.device,"creating %d I/O queues.\n" , nr_io_queues); +nvmf_set_io_queues (ctrl->ctrl.opts, nr_io_queues, ctrl->io_queues); +if (new) { ret =nvme_rdma_alloc_tag_set (&ctrl->ctrl);if (ret)
@@ -881,7 +890,9 @@ static int nvme_rdma_configure_io_queues(struct nvme_rdma_ctrl *ctrl, bool new)* queue number might have changed. */ nr_queues =min (ctrl->tag_set.nr_hw_queues +1 , ctrl->ctrl.queue_count); - ret =nvme_rdma_start_io_queues (ctrl,1 , nr_queues); + ret =nvme_rdma_setup_io_queues (ctrl,1 , nr_queues, + ctrl->ctrl.sqsize +1 ); +if (ret) goto out_cleanup_tagset;
@@ -905,12 +916,15 @@ static int nvme_rdma_configure_io_queues(struct nvme_rdma_ctrl *ctrl, bool new)/* * If the number of queues has increased (reconnect case) -* start all new queues now. +* setup all new queues now. */ - ret =nvme_rdma_start_io_queues (ctrl, nr_queues, - ctrl->tag_set.nr_hw_queues +1 ); -if (ret) - goto out_wait_freeze_timed_out; +if (ctrl->tag_set.nr_hw_queues +1 > nr_queues) { + ret =nvme_rdma_setup_io_queues (ctrl, nr_queues, + ctrl->tag_set.nr_hw_queues +1 , + ctrl->ctrl.sqsize +1 ); +if (ret) + goto out_wait_freeze_timed_out; + }return 0 ;
Signature · 1 line
2.54.0
Keith Busch
maintainerQuoted 5 lines · Surabhi Gogte
Refactor nvme rdma I/O queue setup to use async API, combining allocation and startup into a single parallel operation per queue. This reduces connection and reconnection setup time when there are delays in establishing connections, which is especially important for high-core-count hosts.
Mostly looks fine.
Quoted 9 lines
drivers/nvme/host/rdma.c +1
-0@@ -16,6 +16,7 @@#include <linux/types.h> #include <linux/list.h> #include <linux/mutex.h> +#include <linux/async.h> #include <linux/scatterlist.h> #include <linux/nvme.h> #include <linux/unaligned.h>
drivers/nvme/host/rdma.c struct nvme_rdma_ctrl { · line 130
Quoted 5 lines
drivers/nvme/host/rdma.c +1
-0@@ -125,6 +126,7 @@ struct nvme_rdma_ctrl {struct nvme_ctrl ctrl;bool use_inline_data;u32 io_queues[HCTX_MAX_TYPES ]; +atomic_t qsetup_err; };
This new field serves only to propogate an error from a local context, so I don't want to introduce a new field for it at this scope. I prefer you declare a special context struct for it to use with the async usage:
struct nvme_rdma_setup_ctx {
struct nvme_rdma_queue *queue;
int *err;
};And then make that the cookie passed to the async setup. I don't think it needs to be atomic here either: we really don't care if we see the first or last error, so forcing a cmpxchg for the first one is a bit overkill; you can just do READ/WRITE_ONCE instead and accept the race.
Quoted 11 lines · Keith Busch
Quoted 5 lines · Surabhi Gogte
Refactor nvme rdma I/O queue setup to use async API, combining allocation and startup into a single parallel operation per queue. This reduces connection and reconnection setup time when there are delays in establishing connections, which is especially important for high-core-count hosts.
Mostly looks fine.
Quoted 11 lines
drivers/nvme/host/rdma.c struct nvme_rdma_ctrl { · line 130
Quoted 6 lines
drivers/nvme/host/rdma.c +1
-0@@ -125,6 +126,7 @@ struct nvme_rdma_ctrl {struct nvme_ctrl ctrl;bool use_inline_data;u32 io_queues[HCTX_MAX_TYPES ]; +atomic_t qsetup_err; };This new field serves only to propogate an error from a local context, so I don't want to introduce a new field for it at this scope. I prefer you declare a special context struct for it to use with the async usage:
Implemented in v3.
Quoted 9 lines
struct nvme_rdma_setup_ctx { struct nvme_rdma_queue *queue; int *err; };And then make that the cookie passed to the async setup. I don't think it needs to be atomic here either: we really don't care if we see the first or last error, so forcing a cmpxchg for the first one is a bit overkill; you can just do READ/WRITE_ONCE instead and accept the race.
Yeah, that works too. added it in v3.
Christoph Hellwig
maintainerQuoted 1 line · Surabhi Gogte
+static void
nvme_rdma_setup_queue_async(void *setup_queue, async_cookie_t cookie)
Overly long line. Easily fixed by renaming setup_queue to data as in the async_schedule_domain prototype.
drivers/nvme/host/rdma.c static int nvme_rdma_start_queue(struct nvme_rdma_ctrl *ctrl, int idx) · line 715
Quoted 2 lines
drivers/nvme/host/rdma.c +2
-0+static int nvme_rdma_setup_io_queues (struct nvme_rdma_ctrl *ctrl,int first, +int last,size_t queue_size)
should first and last be unsigned?
Also please use two-tab indents for prototype continuations.
drivers/nvme/host/rdma.c static int nvme_rdma_start_queue(struct nvme_rdma_ctrl *ctrl, int idx) · line 735
Quoted 7 lines
drivers/nvme/host/rdma.c +3
-3+async_synchronize_full_domain (&setup_queue_domain); -out_free_queues: -for (i--; i >=1 ; i--) -nvme_rdma_free_queue (&ctrl->queues[i]); + ret =atomic_read (&ctrl->qsetup_err); +if (ret) {
Jump to an error label if there was an error to reduce indentation a bit below.
drivers/nvme/host/rdma.c static int nvme_rdma_configure_admin_queue(struct nvme_rdma_ctrl *ctrl, · line 784
Quoted 3 lines
drivers/nvme/host/rdma.c +3
-0+ ctrl->queues[0 ].ctrl = ctrl; + ctrl->queues[0 ].queue_size =NVME_AQ_DEPTH ; + error =nvme_rdma_alloc_queue (&ctrl->queues[0 ]);
The nvme_rdma_alloc_queue prototype change would be a nice prep patch to split out from the main change.
Quoted 7 lines · Christoph Hellwig
Quoted 1 line · Surabhi Gogte
+static void
nvme_rdma_setup_queue_async(void *setup_queue, async_cookie_t cookie)Overly long line. Easily fixed by renaming
setup_queueto data as in theasync_schedule_domainprototype.
done.
Quoted 7 lines
drivers/nvme/host/rdma.c static int nvme_rdma_start_queue(struct nvme_rdma_ctrl *ctrl, int idx) · line 715
Quoted 2 lines
drivers/nvme/host/rdma.c +2
-0+static int nvme_rdma_setup_io_queues (struct nvme_rdma_ctrl *ctrl,int first, +int last,size_t queue_size)should first and last be unsigned?
Also please use two-tab indents for prototype continuations.
converted to unsigned int. yeah using 2-tab indents in v3.
Quoted 11 lines
drivers/nvme/host/rdma.c static int nvme_rdma_start_queue(struct nvme_rdma_ctrl *ctrl, int idx) · line 735
Quoted 7 lines
drivers/nvme/host/rdma.c +3
-3+async_synchronize_full_domain (&setup_queue_domain); -out_free_queues: -for (i--; i >=1 ; i--) -nvme_rdma_free_queue (&ctrl->queues[i]); + ret =atomic_read (&ctrl->qsetup_err); +if (ret) {Jump to an error label if there was an error to reduce indentation a bit below.
done.
Quoted 7 lines
drivers/nvme/host/rdma.c static int nvme_rdma_configure_admin_queue(struct nvme_rdma_ctrl *ctrl, · line 784
Quoted 3 lines
drivers/nvme/host/rdma.c +3
-0+ ctrl->queues[0 ].ctrl = ctrl; + ctrl->queues[0 ].queue_size =NVME_AQ_DEPTH ; + error =nvme_rdma_alloc_queue (&ctrl->queues[0 ]);The
nvme_rdma_alloc_queueprototype change would be a nice prep patch to split out from the main change.
done. Separated the patches in v3.