Unifying fabrics drivers
v2 cover Unifying fabrics drivers
Second iteration of this idea. This time I moved all the state machien code to ctrl.
I've tested quickly tcp, rdma is only compile tested and coming up with good names for function is difficult.
changes:
v2:
- move the complete ctrl state machine to fabrics.c
v1:
- https://lore.kernel.org/linux-nvme/[email protected]/
Daniel Wagner (6):
nvme-rdma: stream line queue functions arguments
nvme-rdma: factor rdma specific queue init code out
nvme-tcp: move error and connect work to nvme_ctrl
nvme-rdma: use error and connect work from nvme_ctrl
nvme-tcp: move nvme ctrl state machine code to fabrics.c
nvme-rdma: use ctrl state machine from frabrics
drivers/nvme/host/fabrics.c | 434 +++++++++++++++++++++++++++++ drivers/nvme/host/fabrics.h | 21 ++ drivers/nvme/host/nvme.h | 6 + drivers/nvme/host/rdma.c | 535 +++++++----------------------------- drivers/nvme/host/tcp.c | 504 ++++----------------------------- 5 files changed, 606 insertions(+), 894 deletions(-)
Signature · 1 line
2.39.2
RFC v2 1/6 nvme-rdma: stream line queue functions arguments
In preparation to move common code from the fabrics driver to fabrics.c, we stream line the low level functions. This allows any common code just to pass in nvme subsystem global types, such as 'struct nvme_ctrl' instead of the driver specialized types 'struct nvme_rdma_ctrl'.
--- drivers/nvme/host/rdma.c | 62 ++++++++++++++++++++++++++-------------- 1 file changed, 40 insertions(+), 22 deletions(-)
drivers/nvme/host/rdma.c +40 -22
@@ -133,6 +133,11 @@ static inline struct nvme_rdma_ctrl *to_rdma_ctrl(struct nvme_ctrl *ctrl) return container_of(ctrl, struct nvme_rdma_ctrl, ctrl); } +static inline int nvme_rdma_queue_id (struct nvme_rdma_queue *queue) +{ +return queue - queue->ctrl->queues; +} +static LIST_HEAD (device_list);static DEFINE_MUTEX (device_list_mutex);
@@ -571,13 +576,19 @@ static int nvme_rdma_create_queue_ib(struct nvme_rdma_queue *queue)return ret; } -static int nvme_rdma_alloc_queue (struct nvme_rdma_ctrl *ctrl, -int idx,size_t queue_size) +static int nvme_rdma_alloc_queue (struct nvme_ctrl *nctrl,int idx) { +struct nvme_rdma_ctrl *ctrl =to_rdma_ctrl (nctrl);struct nvme_rdma_queue *queue;struct sockaddr *src_addr =NULL ; +size_t queue_size;int ret; +if (idx ==0 ) + queue_size =NVME_AQ_DEPTH ; +else + queue_size = ctrl->ctrl.sqsize +1 ; + queue = &ctrl->queues[idx];mutex_init (&queue->queue_lock); queue->ctrl = ctrl;
@@ -641,16 +652,22 @@ static void __nvme_rdma_stop_queue(struct nvme_rdma_queue *queue)ib_drain_qp (queue->qp); } -static void nvme_rdma_stop_queue (struct nvme_rdma_queue *queue) +static void nvme_rdma_stop_queue (struct nvme_ctrl *nctrl,int qid) { +struct nvme_rdma_ctrl *ctrl =to_rdma_ctrl (nctrl); +struct nvme_rdma_queue *queue = &ctrl->queues[qid]; +mutex_lock (&queue->queue_lock);if (test_and_clear_bit (NVME_RDMA_Q_LIVE , &queue->flags))__nvme_rdma_stop_queue (queue);mutex_unlock (&queue->queue_lock); } -static void nvme_rdma_free_queue (struct nvme_rdma_queue *queue) +static void nvme_rdma_free_queue (struct nvme_ctrl *nctrl,int qid) { +struct nvme_rdma_ctrl *ctrl =to_rdma_ctrl (nctrl); +struct nvme_rdma_queue *queue = &ctrl->queues[qid]; +if (!test_and_clear_bit (NVME_RDMA_Q_ALLOCATED , &queue->flags))return ;
@@ -664,7 +681,7 @@ static void nvme_rdma_free_io_queues(struct nvme_rdma_ctrl *ctrl)int i;for (i =1 ; i < ctrl->ctrl.queue_count; i++) -nvme_rdma_free_queue (&ctrl->queues[i]); +nvme_rdma_free_queue (&ctrl->ctrl, i); }static void nvme_rdma_stop_io_queues (struct nvme_rdma_ctrl *ctrl)
@@ -672,18 +689,19 @@ static void nvme_rdma_stop_io_queues(struct nvme_rdma_ctrl *ctrl)int i;for (i =1 ; i < ctrl->ctrl.queue_count; i++) -nvme_rdma_stop_queue (&ctrl->queues[i]); +nvme_rdma_stop_queue (&ctrl->ctrl, i); } -static int nvme_rdma_start_queue (struct nvme_rdma_ctrl *ctrl,int idx) +static int nvme_rdma_start_queue (struct nvme_ctrl *nctrl,int idx) { +struct nvme_rdma_ctrl *ctrl =to_rdma_ctrl (nctrl);struct nvme_rdma_queue *queue = &ctrl->queues[idx];int ret;if (idx) - ret =nvmf_connect_io_queue (&ctrl->ctrl, idx); + ret =nvmf_connect_io_queue (nctrl, idx);else - ret =nvmf_connect_admin_queue (&ctrl->ctrl); + ret =nvmf_connect_admin_queue (nctrl);if (!ret) {set_bit (NVME_RDMA_Q_LIVE , &queue->flags);
@@ -702,7 +720,7 @@ static int nvme_rdma_start_io_queues(struct nvme_rdma_ctrl *ctrl,int i, ret =0 ;for (i = first; i < last; i++) { - ret =nvme_rdma_start_queue (ctrl, i); + ret =nvme_rdma_start_queue (&ctrl->ctrl, i);if (ret) goto out_stop_queues; }
@@ -711,7 +729,7 @@ static int nvme_rdma_start_io_queues(struct nvme_rdma_ctrl *ctrl, out_stop_queues:for (i--; i >= first; i--) -nvme_rdma_stop_queue (&ctrl->queues[i]); +nvme_rdma_stop_queue (&ctrl->ctrl, i);return ret; }
@@ -773,8 +791,7 @@ static int nvme_rdma_alloc_io_queues(struct nvme_rdma_ctrl *ctrl) }for (i =1 ; i < ctrl->ctrl.queue_count; i++) { - ret =nvme_rdma_alloc_queue (ctrl, i, - ctrl->ctrl.sqsize +1 ); + ret =nvme_rdma_alloc_queue (&ctrl->ctrl, i);if (ret) goto out_free_queues; }
@@ -783,7 +800,7 @@ static int nvme_rdma_alloc_io_queues(struct nvme_rdma_ctrl *ctrl) out_free_queues:for (i--; i >=1 ; i--) -nvme_rdma_free_queue (&ctrl->queues[i]); +nvme_rdma_free_queue (&ctrl->ctrl, i);return ret; }
@@ -811,7 +828,7 @@ static void nvme_rdma_destroy_admin_queue(struct nvme_rdma_ctrl *ctrl) sizeof(struct nvme_command), DMA_TO_DEVICE); ctrl->async_event_sqe.data =NULL ; } -nvme_rdma_free_queue (&ctrl->queues[0 ]); +nvme_rdma_free_queue (&ctrl->ctrl,0 ); } static int nvme_rdma_configure_admin_queue(struct nvme_rdma_ctrl *ctrl,
@@ -820,7 +837,7 @@ static int nvme_rdma_configure_admin_queue(struct nvme_rdma_ctrl *ctrl,bool pi_capable = false;int error; - error =nvme_rdma_alloc_queue (ctrl,0 ,NVME_AQ_DEPTH ); + error =nvme_rdma_alloc_queue (&ctrl->ctrl,0 );if (error)return error;
@@ -855,7 +872,7 @@ static int nvme_rdma_configure_admin_queue(struct nvme_rdma_ctrl *ctrl, } - error =nvme_rdma_start_queue (ctrl,0 ); + error =nvme_rdma_start_queue (&ctrl->ctrl,0 );if (error) goto out_remove_admin_tag_set;
@@ -882,7 +899,7 @@ static int nvme_rdma_configure_admin_queue(struct nvme_rdma_ctrl *ctrl,nvme_quiesce_admin_queue (&ctrl->ctrl);blk_sync_queue (ctrl->ctrl.admin_q); out_stop_queue: -nvme_rdma_stop_queue (&ctrl->queues[0 ]); +nvme_rdma_stop_queue (&ctrl->ctrl,0 );nvme_cancel_admin_tagset (&ctrl->ctrl); out_remove_admin_tag_set:if (new)
@@ -894,7 +911,7 @@ static int nvme_rdma_configure_admin_queue(struct nvme_rdma_ctrl *ctrl, ctrl->async_event_sqe.data =NULL ; } out_free_queue: -nvme_rdma_free_queue (&ctrl->queues[0 ]); +nvme_rdma_free_queue (&ctrl->ctrl,0 );return error; }
@@ -967,7 +984,7 @@ static void nvme_rdma_teardown_admin_queue(struct nvme_rdma_ctrl *ctrl, {nvme_quiesce_admin_queue (&ctrl->ctrl);blk_sync_queue (ctrl->ctrl.admin_q); -nvme_rdma_stop_queue (&ctrl->queues[0 ]); +nvme_rdma_stop_queue (&ctrl->ctrl,0 );nvme_cancel_admin_tagset (&ctrl->ctrl);if (remove) {nvme_unquiesce_admin_queue (&ctrl->ctrl);
@@ -1118,7 +1135,7 @@ static int nvme_rdma_setup_ctrl(struct nvme_rdma_ctrl *ctrl, bool new) destroy_admin:nvme_quiesce_admin_queue (&ctrl->ctrl);blk_sync_queue (ctrl->ctrl.admin_q); -nvme_rdma_stop_queue (&ctrl->queues[0 ]); +nvme_rdma_stop_queue (&ctrl->ctrl,0 );nvme_cancel_admin_tagset (&ctrl->ctrl);if (new)nvme_remove_admin_tag_set (&ctrl->ctrl);
@@ -1965,9 +1982,10 @@ static int nvme_rdma_cm_handler(struct rdma_cm_id *cm_id,static void nvme_rdma_complete_timed_out (struct request *rq) {struct nvme_rdma_request *req =blk_mq_rq_to_pdu (rq); +struct nvme_ctrl *ctrl = &req->queue->ctrl->ctrl;struct nvme_rdma_queue *queue = req->queue; -nvme_rdma_stop_queue (queue); +nvme_rdma_stop_queue (ctrl,nvme_rdma_queue_id (queue));nvmf_complete_timed_out_request (rq); }
Signature · 1 line
2.39.2
RFC v2 2/6 nvme-rdma: factor rdma specific queue init code out
In reply to Daniel WagnerIn preparation to move common code from the fabrics driver to fabrics.c, move the rmda queue specific initialization code into a separate function.
--- drivers/nvme/host/rdma.c | 65 ++++++++++++++++++++++++++++------------ 1 file changed, 46 insertions(+), 19 deletions(-)
drivers/nvme/host/rdma.c +46 -19
@@ -831,15 +831,16 @@ static void nvme_rdma_destroy_admin_queue(struct nvme_rdma_ctrl *ctrl)nvme_rdma_free_queue (&ctrl->ctrl,0 ); } -static int nvme_rdma_configure_admin_queue (struct nvme_rdma_ctrl *ctrl, -bool new) +static int nvme_rdma_init_queue (struct nvme_ctrl *nctrl,int qid) { +struct nvme_rdma_ctrl *ctrl =to_rdma_ctrl (nctrl);bool pi_capable = false;int error; - error =nvme_rdma_alloc_queue (&ctrl->ctrl,0 ); -if (error) -return error; +if (qid !=0 ) +/* only admin queue needs additional work. */ +return 0 ; + ctrl->device = ctrl->queues[0 ].device; ctrl->ctrl.numa_node =ibdev_to_node (ctrl->device->dev);
@@ -859,6 +860,43 @@ static int nvme_rdma_configure_admin_queue(struct nvme_rdma_ctrl *ctrl,*/ error =nvme_rdma_alloc_qe (ctrl->device->dev, &ctrl->async_event_sqe,sizeof (struct nvme_command ),DMA_TO_DEVICE ); +if (error) +return error; + + ctrl->ctrl.max_segments = ctrl->max_fr_pages; + ctrl->ctrl.max_hw_sectors = ctrl->max_fr_pages << (ilog2 (SZ_4K ) -9 ); +if (pi_capable) + ctrl->ctrl.max_integrity_segments = ctrl->max_fr_pages; +else + ctrl->ctrl.max_integrity_segments =0 ; + +return 0 ; +} + +static void nvme_rdma_deinit_queue (struct nvme_ctrl *nctrl,int qid) +{ +struct nvme_rdma_ctrl *ctrl =to_rdma_ctrl (nctrl); + +if (qid !=0 ) +return ; + +if (ctrl->async_event_sqe.data) { +nvme_rdma_free_qe (ctrl->device->dev, &ctrl->async_event_sqe, +sizeof (struct nvme_command ),DMA_TO_DEVICE ); + ctrl->async_event_sqe.data =NULL ; + } +} + +static int nvme_rdma_configure_admin_queue (struct nvme_rdma_ctrl *ctrl, +bool new) +{ +int error; + + error =nvme_rdma_alloc_queue (&ctrl->ctrl,0 ); +if (error) +return error; + + error =nvme_rdma_init_queue (&ctrl->ctrl,0 );if (error) goto out_free_queue;
@@ -868,7 +906,7 @@ static int nvme_rdma_configure_admin_queue(struct nvme_rdma_ctrl *ctrl,sizeof (struct nvme_rdma_request ) + NVME_RDMA_DATA_SGL_SIZE);if (error) - goto out_free_async_qe; + goto out_deinit_admin_queue; }
@@ -880,13 +918,6 @@ static int nvme_rdma_configure_admin_queue(struct nvme_rdma_ctrl *ctrl,if (error) goto out_stop_queue; - ctrl->ctrl.max_segments = ctrl->max_fr_pages; - ctrl->ctrl.max_hw_sectors = ctrl->max_fr_pages << (ilog2 (SZ_4K ) -9 ); -if (pi_capable) - ctrl->ctrl.max_integrity_segments = ctrl->max_fr_pages; -else - ctrl->ctrl.max_integrity_segments =0 ; -nvme_unquiesce_admin_queue (&ctrl->ctrl); error =nvme_init_ctrl_finish (&ctrl->ctrl, false);
@@ -904,12 +935,8 @@ static int nvme_rdma_configure_admin_queue(struct nvme_rdma_ctrl *ctrl, out_remove_admin_tag_set:if (new)nvme_remove_admin_tag_set (&ctrl->ctrl); -out_free_async_qe: -if (ctrl->async_event_sqe.data) { -nvme_rdma_free_qe (ctrl->device->dev, &ctrl->async_event_sqe, -sizeof (struct nvme_command ),DMA_TO_DEVICE ); - ctrl->async_event_sqe.data =NULL ; - } +out_deinit_admin_queue: +nvme_rdma_deinit_queue (&ctrl->ctrl,0 ); out_free_queue:nvme_rdma_free_queue (&ctrl->ctrl,0 );return error;
Signature · 1 line
2.39.2
RFC v2 3/6 nvme-tcp: move error and connect work to nvme_ctrl
In reply to Daniel WagnerMove common data structures for fabrics to nvme_ctrl so that we are able to use them in fabrcis.c later.
--- drivers/nvme/host/nvme.h | 3 +++ drivers/nvme/host/tcp.c | 24 ++++++++++-------------- 2 files changed, 13 insertions(+), 14 deletions(-)
drivers/nvme/host/nvme.h +3 -0
@@ -339,6 +339,9 @@ struct nvme_ctrl {struct work_struct ana_work;#endif +struct work_struct err_work; +struct delayed_work connect_work; +#ifdef CONFIG_NVME_AUTH struct work_struct dhchap_auth_work;struct mutex dhchap_auth_mutex;
drivers/nvme/host/tcp.c +10 -14
@@ -165,8 +165,6 @@ struct nvme_tcp_ctrl {struct sockaddr_storage src_addr;struct nvme_ctrl ctrl; -struct work_struct err_work; -struct delayed_work connect_work;struct nvme_tcp_request async_req;u32 io_queues[HCTX_MAX_TYPES ]; };
@@ -526,7 +524,7 @@ static void nvme_tcp_error_recovery(struct nvme_ctrl *ctrl)return ;dev_warn (ctrl->device,"starting error recovery\n" ); -queue_work (nvme_reset_wq, &to_tcp_ctrl (ctrl)->err_work); +queue_work (nvme_reset_wq, &ctrl->err_work); } static int nvme_tcp_process_nvme_cqe(struct nvme_tcp_queue *queue,
@@ -2019,7 +2017,7 @@ static void nvme_tcp_reconnect_or_remove(struct nvme_ctrl *ctrl)if (nvmf_should_reconnect (ctrl)) {dev_info (ctrl->device,"Reconnecting in %d seconds...\n" , ctrl->opts->reconnect_delay); -queue_delayed_work (nvme_wq, &to_tcp_ctrl (ctrl)->connect_work, +queue_delayed_work (nvme_wq, &ctrl->connect_work, ctrl->opts->reconnect_delay *HZ ); }else {dev_info (ctrl->device,"Removing controller...\n" );
@@ -2101,9 +2099,8 @@ static int nvme_tcp_setup_ctrl(struct nvme_ctrl *ctrl, bool new)static void nvme_tcp_reconnect_ctrl_work (struct work_struct *work) { -struct nvme_tcp_ctrl *tcp_ctrl =container_of (to_delayed_work (work), - struct nvme_tcp_ctrl, connect_work); -struct nvme_ctrl *ctrl = &tcp_ctrl->ctrl; +struct nvme_ctrl *ctrl =container_of (to_delayed_work (work), + struct nvme_ctrl, connect_work); ++ctrl->nr_reconnects;
@@ -2125,9 +2122,8 @@ static void nvme_tcp_reconnect_ctrl_work(struct work_struct *work)static void nvme_tcp_error_recovery_work (struct work_struct *work) { -struct nvme_tcp_ctrl *tcp_ctrl =container_of (work, - struct nvme_tcp_ctrl, err_work); -struct nvme_ctrl *ctrl = &tcp_ctrl->ctrl; +struct nvme_ctrl *ctrl =container_of (work, + struct nvme_ctrl, err_work);nvme_stop_keep_alive (ctrl);flush_work (&ctrl->async_event_work);
@@ -2188,8 +2184,8 @@ static void nvme_reset_ctrl_work(struct work_struct *work)static void nvme_tcp_stop_ctrl (struct nvme_ctrl *ctrl) { -flush_work (&to_tcp_ctrl (ctrl)->err_work); -cancel_delayed_work_sync (&to_tcp_ctrl (ctrl)->connect_work); +flush_work (&ctrl->err_work); +cancel_delayed_work_sync (&ctrl->connect_work); }static void nvme_tcp_free_ctrl (struct nvme_ctrl *nctrl)
@@ -2569,9 +2565,9 @@ static struct nvme_ctrl *nvme_tcp_create_ctrl(struct device *dev, ctrl->ctrl.sqsize = opts->queue_size -1 ; ctrl->ctrl.kato = opts->kato; -INIT_DELAYED_WORK (&ctrl->connect_work, +INIT_DELAYED_WORK (&ctrl->ctrl.connect_work, nvme_tcp_reconnect_ctrl_work); -INIT_WORK (&ctrl->err_work, nvme_tcp_error_recovery_work); +INIT_WORK (&ctrl->ctrl.err_work, nvme_tcp_error_recovery_work);INIT_WORK (&ctrl->ctrl.reset_work, nvme_reset_ctrl_work);if (!(opts->mask &NVMF_OPT_TRSVCID )) {
Signature · 1 line
2.39.2
RFC v2 4/6 nvme-rdma: use error and connect work from nvme_ctrl
In reply to Daniel WagnerUse common data structures from nvme_ctrl so that we are able to use them in fabrics.c later.
--- drivers/nvme/host/rdma.c | 25 ++++++++++++------------- 1 file changed, 12 insertions(+), 13 deletions(-)
drivers/nvme/host/rdma.c +12 -13
@@ -107,12 +107,9 @@ struct nvme_rdma_ctrl {/* other member variables */ struct blk_mq_tag_set tag_set; -struct work_struct err_work;struct nvme_rdma_qe async_event_sqe; -struct delayed_work reconnect_work; -struct list_head list;struct blk_mq_tag_set admin_tag_set;
@@ -1041,8 +1038,8 @@ static void nvme_rdma_stop_ctrl(struct nvme_ctrl *nctrl) {struct nvme_rdma_ctrl *ctrl =to_rdma_ctrl (nctrl); -flush_work (&ctrl->err_work); -cancel_delayed_work_sync (&ctrl->reconnect_work); +flush_work (&ctrl->ctrl.err_work); +cancel_delayed_work_sync (&ctrl->ctrl.connect_work); }static void nvme_rdma_free_ctrl (struct nvme_ctrl *nctrl)
@@ -1074,7 +1071,7 @@ static void nvme_rdma_reconnect_or_remove(struct nvme_rdma_ctrl *ctrl)if (nvmf_should_reconnect (&ctrl->ctrl)) {dev_info (ctrl->ctrl.device,"Reconnecting in %d seconds...\n" , ctrl->ctrl.opts->reconnect_delay); -queue_delayed_work (nvme_wq, &ctrl->reconnect_work, +queue_delayed_work (nvme_wq, &ctrl->ctrl.connect_work, ctrl->ctrl.opts->reconnect_delay *HZ ); }else {nvme_delete_ctrl (&ctrl->ctrl);
@@ -1172,8 +1169,9 @@ static int nvme_rdma_setup_ctrl(struct nvme_rdma_ctrl *ctrl, bool new)static void nvme_rdma_reconnect_ctrl_work (struct work_struct *work) { -struct nvme_rdma_ctrl *ctrl =container_of (to_delayed_work (work), - struct nvme_rdma_ctrl, reconnect_work); +struct nvme_ctrl *nctrl =container_of (to_delayed_work (work), + struct nvme_ctrl, connect_work); +struct nvme_rdma_ctrl *ctrl =to_rdma_ctrl (nctrl); ++ctrl->ctrl.nr_reconnects;
@@ -1195,8 +1193,9 @@ static void nvme_rdma_reconnect_ctrl_work(struct work_struct *work)static void nvme_rdma_error_recovery_work (struct work_struct *work) { -struct nvme_rdma_ctrl *ctrl =container_of (work, - struct nvme_rdma_ctrl, err_work); +struct nvme_ctrl *nctrl =container_of (work, + struct nvme_ctrl, err_work); +struct nvme_rdma_ctrl *ctrl =to_rdma_ctrl (nctrl);nvme_stop_keep_alive (&ctrl->ctrl);flush_work (&ctrl->ctrl.async_event_work);
@@ -1222,7 +1221,7 @@ static void nvme_rdma_error_recovery(struct nvme_rdma_ctrl *ctrl)return ;dev_warn (ctrl->ctrl.device,"starting error recovery\n" ); -queue_work (nvme_reset_wq, &ctrl->err_work); +queue_work (nvme_reset_wq, &ctrl->ctrl.err_work); }static void nvme_rdma_end_request (struct nvme_rdma_request *req)
@@ -2376,9 +2375,9 @@ static struct nvme_ctrl *nvme_rdma_create_ctrl(struct device *dev, goto out_free_ctrl; } -INIT_DELAYED_WORK (&ctrl->reconnect_work, +INIT_DELAYED_WORK (&ctrl->ctrl.connect_work, nvme_rdma_reconnect_ctrl_work); -INIT_WORK (&ctrl->err_work, nvme_rdma_error_recovery_work); +INIT_WORK (&ctrl->ctrl.err_work, nvme_rdma_error_recovery_work);INIT_WORK (&ctrl->ctrl.reset_work, nvme_rdma_reset_ctrl_work); ctrl->ctrl.queue_count = opts->nr_io_queues + opts->nr_write_queues +
Signature · 1 line
2.39.2
RFC v2 5/6 nvme-tcp: move nvme ctrl state machine code to fabrics.c
In reply to Daniel WagnerThe two transports for tcp and rdma share a lot of o common code, which handles the controller states and is not transport specific.
The fabrics.c file contains helper functions to be used by the transport This limits what the common code in fabrics can do. Any transport specific allocation/initialization (e.g comand size for admin_tag_set) has either be parameterized via the function call or we introduce callbacks for parameterizing.
The third options is to introduce an intermediated type e.g. struct nvme_fabrics_ctrl, but that would mean a lot of code refactoring and restructuring just to avoid passing in arguments to functions.
The callback approach has another benefit, as it allows to add hooks to the generic code for the driver to do allocate additional transport specific resources.
--- drivers/nvme/host/fabrics.c | 434 +++++++++++++++++++++++++++++++ drivers/nvme/host/fabrics.h | 21 ++ drivers/nvme/host/nvme.h | 3 + drivers/nvme/host/rdma.c | 77 ++---- drivers/nvme/host/tcp.c | 498 ++++-------------------------------- 5 files changed, 526 insertions(+), 507 deletions(-)
drivers/nvme/host/fabrics.c +434 -0
@@ -1134,6 +1134,440 @@ nvmf_create_ctrl(struct device *dev, const char *buf)return ERR_PTR (ret); } +static int nvmf_start_io_queues (struct nvme_ctrl *ctrl, +int first,int last) +{ +int i, ret; + +for (i = first; i < last; i++) { + ret = ctrl->fabrics_ops->start_queue (ctrl, i); +if (ret) + goto out_stop_queues; + } + +return 0 ; + +out_stop_queues: +for (i--; i >= first; i--) + ctrl->fabrics_ops->stop_queue (ctrl, i); +return ret; +} + +static void nvmf_stop_io_queues (struct nvme_ctrl *ctrl) +{ +int i; + +for (i =1 ; i < ctrl->queue_count; i++) + ctrl->fabrics_ops->stop_queue (ctrl, i); +} + +static int __nvmf_alloc_io_queues (struct nvme_ctrl *ctrl) +{ +int i, ret; + +for (i =1 ; i < ctrl->queue_count; i++) { + ret = ctrl->fabrics_ops->alloc_queue (ctrl, i); +if (ret) + goto out_free_queues; + } + +return 0 ; + +out_free_queues: +for (i--; i >=1 ; i--) + ctrl->fabrics_ops->free_queue (ctrl, i); + +return ret; +} + +static int nvmf_alloc_io_queues (struct nvme_ctrl *ctrl) +{ +unsigned int nr_io_queues; +int ret; + + nr_io_queues = ctrl->fabrics_ops->nr_io_queues (ctrl); + ret =nvme_set_queue_count (ctrl, &nr_io_queues); +if (ret) +return ret; + +if (nr_io_queues ==0 ) { +dev_err (ctrl->device, +"unable to set any I/O queues\n" ); +return -ENOMEM ; + } + + ctrl->queue_count = nr_io_queues +1 ; +dev_info (ctrl->device, +"creating %d I/O queues.\n" , nr_io_queues); + + ctrl->fabrics_ops->set_io_queues (ctrl, nr_io_queues); + +return __nvmf_alloc_io_queues (ctrl); +} + +static void nvmf_free_io_queues (struct nvme_ctrl *ctrl) +{ +int i; + +for (i =1 ; i < ctrl->queue_count; i++) + ctrl->fabrics_ops->free_queue (ctrl, i); +} + +int nvmf_configure_io_queues (struct nvme_ctrl *ctrl,bool new) +{ +int ret, nr_queues; + + ret =nvmf_alloc_io_queues (ctrl); +if (ret) +return ret; + +if (new) { + ret = ctrl->fabrics_ops->alloc_tag_set (ctrl); +if (ret) + goto out_free_io_queues; + } + +/* +* Only start IO queues for which we have allocated the tagset +* and limitted it to the available queues. On reconnects, the +* queue number might have changed. +*/ + nr_queues =min (ctrl->tagset->nr_hw_queues +1 , ctrl->queue_count); + ret =nvmf_start_io_queues (ctrl,1 , nr_queues); +if (ret) + goto out_cleanup_connect_q; + +if (!new) { +nvme_unquiesce_io_queues (ctrl); +if (!nvme_wait_freeze_timeout (ctrl,NVME_IO_TIMEOUT )) { +/* +* If we timed out waiting for freeze we are likely to +* be stuck. Fail the controller initialization just +* to be safe. +*/ + ret = -ENODEV ; + goto out_wait_freeze_timed_out; + } +blk_mq_update_nr_hw_queues (ctrl->tagset, + ctrl->queue_count -1 ); +nvme_unfreeze (ctrl); + } + +/* +* If the number of queues has increased (reconnect case) +* start all new queues now. +*/ + ret =nvmf_start_io_queues (ctrl, nr_queues, + ctrl->tagset->nr_hw_queues +1 ); +if (ret) + goto out_wait_freeze_timed_out; + +return 0 ; + +out_wait_freeze_timed_out: +nvme_quiesce_io_queues (ctrl); +nvme_sync_io_queues (ctrl); +nvmf_stop_io_queues (ctrl); +out_cleanup_connect_q: +nvme_cancel_tagset (ctrl); +if (new) +nvme_remove_io_tag_set (ctrl); +out_free_io_queues: +nvmf_free_io_queues (ctrl); +return ret; +} +EXPORT_SYMBOL_GPL (nvmf_configure_io_queues); + +int nvmf_configure_admin_queue (struct nvme_ctrl *ctrl,bool new) +{ +int error; + + error = ctrl->fabrics_ops->alloc_queue (ctrl,0 ); +if (error) +return error; + + error = ctrl->fabrics_ops->init_queue (ctrl,0 ); +if (error) + goto out_free_queue; + +if (new) { + error = ctrl->fabrics_ops->alloc_admin_tag_set (ctrl); +if (error) + goto out_deinit_admin_queue; + + } + + error = ctrl->fabrics_ops->start_queue (ctrl,0 ); +if (error) + goto out_remove_admin_tag_set; + + error =nvme_enable_ctrl (ctrl); +if (error) + goto out_stop_queue; + +nvme_unquiesce_admin_queue (ctrl); + + error =nvme_init_ctrl_finish (ctrl, false); +if (error) + goto out_quiesce_queue; + +return 0 ; + +out_quiesce_queue: +nvme_quiesce_admin_queue (ctrl); +blk_sync_queue (ctrl->admin_q); +out_stop_queue: + ctrl->fabrics_ops->stop_queue (ctrl,0 ); +nvme_cancel_admin_tagset (ctrl); +out_remove_admin_tag_set: +if (new) +nvme_remove_admin_tag_set (ctrl); +out_deinit_admin_queue: + ctrl->fabrics_ops->deinit_queue (ctrl,0 ); +out_free_queue: + ctrl->fabrics_ops->free_queue (ctrl,0 ); +return error; +} +EXPORT_SYMBOL_GPL (nvmf_configure_admin_queue); + +static void nvmf_destroy_io_queues (struct nvme_ctrl *ctrl,bool remove) +{ +nvmf_stop_io_queues (ctrl); +if (remove) +nvme_remove_io_tag_set (ctrl); +nvmf_free_io_queues (ctrl); +} + +static void nvmf_destroy_admin_queue (struct nvme_ctrl *ctrl,bool remove) +{ + ctrl->fabrics_ops->stop_queue (ctrl,0 ); +if (remove) +nvme_remove_admin_tag_set (ctrl); + + ctrl->fabrics_ops->free_queue (ctrl,0 ); +} + +static void nvmf_teardown_admin_queue (struct nvme_ctrl *ctrl,bool remove) +{ +nvme_quiesce_admin_queue (ctrl); +blk_sync_queue (ctrl->admin_q); + ctrl->fabrics_ops->stop_queue (ctrl,0 ); +nvme_cancel_admin_tagset (ctrl); +if (remove) +nvme_unquiesce_admin_queue (ctrl); +nvmf_destroy_admin_queue (ctrl, remove); +} + +static void nvmf_teardown_io_queues (struct nvme_ctrl *ctrl,bool remove) +{ +if (ctrl->queue_count <=1 ) +return ; +nvme_quiesce_admin_queue (ctrl); +nvme_start_freeze (ctrl); +nvme_quiesce_io_queues (ctrl); +nvme_sync_io_queues (ctrl); +nvmf_stop_io_queues (ctrl); +nvme_cancel_tagset (ctrl); +if (remove) +nvme_unquiesce_io_queues (ctrl); +nvmf_destroy_io_queues (ctrl, remove); +} + +void nvmf_teardown_ctrl (struct nvme_ctrl *ctrl,bool shutdown) +{ +nvmf_teardown_io_queues (ctrl, shutdown); +nvme_quiesce_admin_queue (ctrl); +nvme_disable_ctrl (ctrl, shutdown); +nvmf_teardown_admin_queue (ctrl, shutdown); +} +EXPORT_SYMBOL_GPL (nvmf_teardown_ctrl); + +void nvmf_stop_ctrl (struct nvme_ctrl *ctrl) +{ +flush_work (&ctrl->err_work); +cancel_delayed_work_sync (&ctrl->connect_work); +} +EXPORT_SYMBOL_GPL (nvmf_stop_ctrl); + +int nvmf_setup_ctrl (struct nvme_ctrl *ctrl,bool new) +{ +struct nvmf_ctrl_options *opts = ctrl->opts; +int ret; + + ret =nvmf_configure_admin_queue (ctrl, new); +if (ret) +return ret; + +if (ctrl->icdoff) { + ret = -EOPNOTSUPP ; +dev_err (ctrl->device,"icdoff is not supported!\n" ); + goto destroy_admin; + } + +if (!nvme_ctrl_sgl_supported (ctrl)) { + ret = -EOPNOTSUPP ; +dev_err (ctrl->device,"Mandatory sgls are not supported!\n" ); + goto destroy_admin; + } + +if (opts->queue_size > ctrl->sqsize +1 ) +dev_warn (ctrl->device, +"queue_size %zu > ctrl sqsize %u, clamping down\n" , + opts->queue_size, ctrl->sqsize +1 ); + +if (ctrl->sqsize +1 > ctrl->maxcmd) { +dev_warn (ctrl->device, +"sqsize %u > ctrl maxcmd %u, clamping down\n" , + ctrl->sqsize +1 , ctrl->maxcmd); + ctrl->sqsize = ctrl->maxcmd -1 ; + } + +if (ctrl->queue_count >1 ) { + ret =nvmf_configure_io_queues (ctrl, new); +if (ret) + goto destroy_admin; + } + +if (!nvme_change_ctrl_state (ctrl,NVME_CTRL_LIVE )) { +/* +* state change failure is ok if we started ctrl delete, +* unless we're during creation of a new controller to +* avoid races with teardown flow. +*/ +WARN_ON_ONCE (ctrl->state !=NVME_CTRL_DELETING && + ctrl->state !=NVME_CTRL_DELETING_NOIO ); +WARN_ON_ONCE (new); + ret = -EINVAL ; + goto destroy_io; + } + +nvme_start_ctrl (ctrl); +return 0 ; + +destroy_io: +if (ctrl->queue_count >1 ) { +nvme_quiesce_io_queues (ctrl); +nvme_sync_io_queues (ctrl); +nvmf_stop_io_queues (ctrl); +nvme_cancel_tagset (ctrl); +nvmf_destroy_io_queues (ctrl, new); + } +destroy_admin: +nvme_quiesce_admin_queue (ctrl); +blk_sync_queue (ctrl->admin_q); + ctrl->fabrics_ops->stop_queue (ctrl,0 ); +nvme_cancel_admin_tagset (ctrl); +nvmf_destroy_admin_queue (ctrl, new); +return ret; +} +EXPORT_SYMBOL_GPL (nvmf_setup_ctrl); + +void nvmf_reconnect_or_remove (struct nvme_ctrl *ctrl) +{ +/* If we are resetting/deleting then do nothing */ +if (ctrl->state !=NVME_CTRL_CONNECTING ) { +WARN_ON_ONCE (ctrl->state ==NVME_CTRL_NEW || + ctrl->state ==NVME_CTRL_LIVE ); +return ; + } + +if (nvmf_should_reconnect (ctrl)) { +dev_info (ctrl->device,"Reconnecting in %d seconds...\n" , + ctrl->opts->reconnect_delay); +queue_delayed_work (nvme_wq, &ctrl->connect_work, + ctrl->opts->reconnect_delay *HZ ); + }else { +dev_info (ctrl->device,"Removing controller...\n" ); +nvme_delete_ctrl (ctrl); + } +} +EXPORT_SYMBOL_GPL (nvmf_reconnect_or_remove); + +void nvmf_error_recovery_work (struct work_struct *work) +{ +struct nvme_ctrl *ctrl =container_of (work, + struct nvme_ctrl, err_work); + +nvme_stop_keep_alive (ctrl); +flush_work (&ctrl->async_event_work); +nvmf_teardown_io_queues (ctrl, false); +/* unquiesce to fail fast pending requests */ +nvme_unquiesce_io_queues (ctrl); +nvmf_teardown_admin_queue (ctrl, false); +nvme_unquiesce_admin_queue (ctrl); +nvme_auth_stop (ctrl); + +if (!nvme_change_ctrl_state (ctrl,NVME_CTRL_CONNECTING )) { +/* state change failure is ok if we started ctrl delete */ +WARN_ON_ONCE (ctrl->state !=NVME_CTRL_DELETING && + ctrl->state !=NVME_CTRL_DELETING_NOIO ); +return ; + } + +nvmf_reconnect_or_remove (ctrl); +} +EXPORT_SYMBOL_GPL (nvmf_error_recovery_work); + +void nvmf_reset_ctrl_work (struct work_struct *work) +{ +struct nvme_ctrl *ctrl = + container_of(work, struct nvme_ctrl, reset_work); + +nvme_stop_ctrl (ctrl); +nvmf_teardown_ctrl (ctrl, false); + +if (!nvme_change_ctrl_state (ctrl,NVME_CTRL_CONNECTING )) { +/* state change failure is ok if we started ctrl delete */ +WARN_ON_ONCE (ctrl->state !=NVME_CTRL_DELETING && + ctrl->state !=NVME_CTRL_DELETING_NOIO ); +return ; + } + +if (nvmf_setup_ctrl (ctrl, false)) + goto out_fail; + +return ; + +out_fail: + ++ctrl->nr_reconnects; +nvmf_reconnect_or_remove (ctrl); +} +EXPORT_SYMBOL_GPL (nvmf_reset_ctrl_work); + +void nvmf_reconnect_ctrl_work (struct work_struct *work) +{ +struct nvme_ctrl *ctrl =container_of (to_delayed_work (work), + struct nvme_ctrl, connect_work); + + ++ctrl->nr_reconnects; + +if (nvmf_setup_ctrl (ctrl, false)) + goto requeue; + +dev_info (ctrl->device,"Successfully reconnected (%d attempt)\n" , + ctrl->nr_reconnects); + + ctrl->nr_reconnects =0 ; + +return ; + +requeue: +dev_info (ctrl->device,"Failed reconnect attempt %d\n" , + ctrl->nr_reconnects); +nvmf_reconnect_or_remove (ctrl); +} +EXPORT_SYMBOL_GPL (nvmf_reconnect_ctrl_work); + +void nvmf_error_recovery (struct nvme_ctrl *ctrl) +{ +if (!nvme_change_ctrl_state (ctrl,NVME_CTRL_RESETTING )) +return ; + +dev_warn (ctrl->device,"starting error recovery\n" ); +queue_work (nvme_reset_wq, &ctrl->err_work); +} +EXPORT_SYMBOL_GPL (nvmf_error_recovery); +static struct class *nvmf_class;static struct device *nvmf_device;static DEFINE_MUTEX (nvmf_dev_mutex);
drivers/nvme/host/fabrics.h +21 -0
@@ -172,6 +172,19 @@ struct nvmf_transport_ops { struct nvmf_ctrl_options *opts); }; +struct nvme_fabrics_ops { +int (*alloc_queue)(struct nvme_ctrl *ctrl,int qid); +void (*free_queue)(struct nvme_ctrl *ctrl,int qid); +int (*init_queue)(struct nvme_ctrl *ctrl,int qid); +void (*deinit_queue)(struct nvme_ctrl *ctrl,int qid); +int (*start_queue)(struct nvme_ctrl *ctrl,int qid); +void (*stop_queue)(struct nvme_ctrl *ctrl,int qid); +int (*alloc_admin_tag_set)(struct nvme_ctrl *ctrl); +int (*alloc_tag_set)(struct nvme_ctrl *ctrl); +unsigned int (*nr_io_queues)(struct nvme_ctrl *ctrl); +void (*set_io_queues)(struct nvme_ctrl *ctrl,unsigned int nr_io_queues); +}; +static inline bool nvmf_ctlr_matches_baseopts (struct nvme_ctrl *ctrl,struct nvmf_ctrl_options *opts)
@@ -214,5 +227,13 @@ int nvmf_get_address(struct nvme_ctrl *ctrl, char *buf, int size);bool nvmf_should_reconnect (struct nvme_ctrl *ctrl);bool nvmf_ip_options_match (struct nvme_ctrl *ctrl,struct nvmf_ctrl_options *opts); +int nvmf_setup_ctrl (struct nvme_ctrl *ctrl,bool new); +void nvmf_stop_ctrl (struct nvme_ctrl *ctrl); +void nvmf_teardown_ctrl (struct nvme_ctrl *ctrl,bool shutdown); +void nvmf_reset_ctrl_work (struct work_struct *work); +void nvmf_reconnect_or_remove (struct nvme_ctrl *ctrl); +void nvmf_error_recovery_work (struct work_struct *work); +void nvmf_reconnect_ctrl_work (struct work_struct *work); +void nvmf_error_recovery (struct nvme_ctrl *ctrl);#endif /* _NVME_FABRICS_H */
drivers/nvme/host/nvme.h +3 -0
@@ -244,6 +244,8 @@ enum nvme_ctrl_flags { NVME_CTRL_STOPPED = 3, }; +struct nvme_fabrics_ops ; +struct nvme_ctrl {bool comp_seen;enum nvme_ctrl_state state;
@@ -251,6 +253,7 @@ struct nvme_ctrl {spinlock_t lock;struct mutex scan_lock;const struct nvme_ctrl_ops *ops; +const struct nvme_fabrics_ops *fabrics_ops;struct request_queue *admin_q;struct request_queue *connect_q;struct request_queue *fabrics_q;
drivers/nvme/host/rdma.c +21 -56
106 lines not on this page · raw message
drivers/nvme/host/tcp.c +47 -451
596 lines not on this page · raw message
Signature · 1 line
2.39.2
RFC v2 6/6 nvme-rdma: use ctrl state machine from frabrics
In reply to Daniel WagnerUse common state machine code from fabrics.c.
--- drivers/nvme/host/rdma.c | 428 ++++----------------------------------- 1 file changed, 39 insertions(+), 389 deletions(-)
drivers/nvme/host/rdma.c +39 -389
@@ -665,6 +665,13 @@ static void nvme_rdma_free_queue(struct nvme_ctrl *nctrl, int qid)struct nvme_rdma_ctrl *ctrl =to_rdma_ctrl (nctrl);struct nvme_rdma_queue *queue = &ctrl->queues[qid]; +if (qid ==0 &&ctrl->async_event_sqe.data) { +cancel_work_sync (&ctrl->ctrl.async_event_work); +nvme_rdma_free_qe (ctrl->device->dev, &ctrl->async_event_sqe, +sizeof (struct nvme_command ),DMA_TO_DEVICE ); + ctrl->async_event_sqe.data =NULL ; + } +if (!test_and_clear_bit (NVME_RDMA_Q_ALLOCATED , &queue->flags))return ;
@@ -673,22 +680,6 @@ static void nvme_rdma_free_queue(struct nvme_ctrl *nctrl, int qid)mutex_destroy (&queue->queue_lock); } -static void nvme_rdma_free_io_queues (struct nvme_rdma_ctrl *ctrl) -{ -int i; - -for (i =1 ; i < ctrl->ctrl.queue_count; i++) -nvme_rdma_free_queue (&ctrl->ctrl, i); -} - -static void nvme_rdma_stop_io_queues (struct nvme_rdma_ctrl *ctrl) -{ -int i; - -for (i =1 ; i < ctrl->ctrl.queue_count; i++) -nvme_rdma_stop_queue (&ctrl->ctrl, i); -} -static int nvme_rdma_start_queue (struct nvme_ctrl *nctrl,int idx) {struct nvme_rdma_ctrl *ctrl =to_rdma_ctrl (nctrl);
@@ -711,53 +702,35 @@ static int nvme_rdma_start_queue(struct nvme_ctrl *nctrl, int idx)return ret; } -static int nvme_rdma_start_io_queues (struct nvme_rdma_ctrl *ctrl, -int first,int last) +static unsigned int nvme_rdma_nr_io_queues (struct nvme_ctrl *ctrl) { -int i, ret =0 ; +struct ib_device *ibdev =to_rdma_ctrl (ctrl)->device->dev; +struct nvmf_ctrl_options *opts = ctrl->opts; +unsigned int nr_io_queues; -for (i = first; i < last; i++) { - ret =nvme_rdma_start_queue (&ctrl->ctrl, i); -if (ret) - goto out_stop_queues; - } + nr_io_queues = min_t(unsigned int, ibdev->num_comp_vectors, +min (opts->nr_io_queues,num_online_cpus ())); + nr_io_queues += min_t(unsigned int, ibdev->num_comp_vectors, +min (opts->nr_write_queues,num_online_cpus ())); + nr_io_queues +=min (opts->nr_poll_queues,num_online_cpus ()); -return 0 ; - -out_stop_queues: -for (i--; i >= first; i--) -nvme_rdma_stop_queue (&ctrl->ctrl, i); -return ret; +return nr_io_queues; } -static int nvme_rdma_alloc_io_queues (struct nvme_rdma_ctrl *ctrl) +static void nvme_rdma_set_io_queues (struct nvme_ctrl *nctrl, +unsigned int nr_io_queues) { -struct nvmf_ctrl_options *opts = ctrl->ctrl.opts; +struct nvme_rdma_ctrl *ctrl =to_rdma_ctrl (nctrl);struct ib_device *ibdev = ctrl->device->dev; -unsigned int nr_io_queues, nr_default_queues; +struct nvmf_ctrl_options *opts = ctrl->ctrl.opts; +unsigned int nr_default_queues;unsigned int nr_read_queues, nr_poll_queues; -int i, ret; nr_read_queues = min_t(unsigned int, ibdev->num_comp_vectors,min (opts->nr_io_queues,num_online_cpus ())); nr_default_queues = min_t(unsigned int, ibdev->num_comp_vectors,min (opts->nr_write_queues,num_online_cpus ())); nr_poll_queues =min (opts->nr_poll_queues,num_online_cpus ()); - nr_io_queues = nr_read_queues + nr_default_queues + nr_poll_queues; - - ret =nvme_set_queue_count (&ctrl->ctrl, &nr_io_queues); -if (ret) -return ret; - -if (nr_io_queues ==0 ) { -dev_err (ctrl->ctrl.device, -"unable to set any I/O queues\n" ); -return -ENOMEM ; - } - - ctrl->ctrl.queue_count = nr_io_queues +1 ; -dev_info (ctrl->ctrl.device, -"creating %d I/O queues.\n" , nr_io_queues);if (opts->nr_write_queues && nr_read_queues < nr_io_queues) {/*
@@ -786,20 +759,6 @@ static int nvme_rdma_alloc_io_queues(struct nvme_rdma_ctrl *ctrl) ctrl->io_queues[HCTX_TYPE_POLL ] =min (nr_poll_queues, nr_io_queues); } - -for (i =1 ; i < ctrl->ctrl.queue_count; i++) { - ret =nvme_rdma_alloc_queue (&ctrl->ctrl, i); -if (ret) - goto out_free_queues; - } - -return 0 ; - -out_free_queues: -for (i--; i >=1 ; i--) -nvme_rdma_free_queue (&ctrl->ctrl, i); - -return ret; }static int nvme_rdma_alloc_tag_set (struct nvme_ctrl *ctrl)
@@ -817,17 +776,6 @@ static int nvme_rdma_alloc_tag_set(struct nvme_ctrl *ctrl) cmd_size); } -static void nvme_rdma_destroy_admin_queue (struct nvme_rdma_ctrl *ctrl) -{ -if (ctrl->async_event_sqe.data) { -cancel_work_sync (&ctrl->ctrl.async_event_work); -nvme_rdma_free_qe (ctrl->device->dev, &ctrl->async_event_sqe, -sizeof (struct nvme_command ),DMA_TO_DEVICE ); - ctrl->async_event_sqe.data =NULL ; - } -nvme_rdma_free_queue (&ctrl->ctrl,0 ); -} -static int nvme_rdma_init_queue (struct nvme_ctrl *nctrl,int qid) {struct nvme_rdma_ctrl *ctrl =to_rdma_ctrl (nctrl);
@@ -884,109 +832,6 @@ static void nvme_rdma_deinit_queue(struct nvme_ctrl *nctrl, int qid) } } -static int nvme_rdma_configure_io_queues (struct nvme_rdma_ctrl *ctrl,bool new) -{ -int ret, nr_queues; - - ret =nvme_rdma_alloc_io_queues (ctrl); -if (ret) -return ret; - -if (new) { - ret =nvme_rdma_alloc_tag_set (&ctrl->ctrl); -if (ret) - goto out_free_io_queues; - } - -/* -* Only start IO queues for which we have allocated the tagset -* and limitted it to the available queues. On reconnects, the -* queue number might have changed. -*/ - nr_queues =min (ctrl->tag_set.nr_hw_queues +1 , ctrl->ctrl.queue_count); - ret =nvme_rdma_start_io_queues (ctrl,1 , nr_queues); -if (ret) - goto out_cleanup_tagset; - -if (!new) { -nvme_unquiesce_io_queues (&ctrl->ctrl); -if (!nvme_wait_freeze_timeout (&ctrl->ctrl,NVME_IO_TIMEOUT )) { -/* -* If we timed out waiting for freeze we are likely to -* be stuck. Fail the controller initialization just -* to be safe. -*/ - ret = -ENODEV ; - goto out_wait_freeze_timed_out; - } -blk_mq_update_nr_hw_queues (ctrl->ctrl.tagset, - ctrl->ctrl.queue_count -1 ); -nvme_unfreeze (&ctrl->ctrl); - } - -/* -* If the number of queues has increased (reconnect case) -* start all new queues now. -*/ - ret =nvme_rdma_start_io_queues (ctrl, nr_queues, - ctrl->tag_set.nr_hw_queues +1 ); -if (ret) - goto out_wait_freeze_timed_out; - -return 0 ; - -out_wait_freeze_timed_out: -nvme_quiesce_io_queues (&ctrl->ctrl); -nvme_sync_io_queues (&ctrl->ctrl); -nvme_rdma_stop_io_queues (ctrl); -out_cleanup_tagset: -nvme_cancel_tagset (&ctrl->ctrl); -if (new) -nvme_remove_io_tag_set (&ctrl->ctrl); -out_free_io_queues: -nvme_rdma_free_io_queues (ctrl); -return ret; -} - -static void nvme_rdma_teardown_admin_queue (struct nvme_rdma_ctrl *ctrl, -bool remove) -{ -nvme_quiesce_admin_queue (&ctrl->ctrl); -blk_sync_queue (ctrl->ctrl.admin_q); -nvme_rdma_stop_queue (&ctrl->ctrl,0 ); -nvme_cancel_admin_tagset (&ctrl->ctrl); -if (remove) { -nvme_unquiesce_admin_queue (&ctrl->ctrl); -nvme_remove_admin_tag_set (&ctrl->ctrl); - } -nvme_rdma_destroy_admin_queue (ctrl); -} - -static void nvme_rdma_teardown_io_queues (struct nvme_rdma_ctrl *ctrl, -bool remove) -{ -if (ctrl->ctrl.queue_count >1 ) { -nvme_start_freeze (&ctrl->ctrl); -nvme_quiesce_io_queues (&ctrl->ctrl); -nvme_sync_io_queues (&ctrl->ctrl); -nvme_rdma_stop_io_queues (ctrl); -nvme_cancel_tagset (&ctrl->ctrl); -if (remove) { -nvme_unquiesce_io_queues (&ctrl->ctrl); -nvme_remove_io_tag_set (&ctrl->ctrl); - } -nvme_rdma_free_io_queues (ctrl); - } -} - -static void nvme_rdma_stop_ctrl (struct nvme_ctrl *nctrl) -{ -struct nvme_rdma_ctrl *ctrl =to_rdma_ctrl (nctrl); - -flush_work (&ctrl->ctrl.err_work); -cancel_delayed_work_sync (&ctrl->ctrl.connect_work); -} -static void nvme_rdma_free_ctrl (struct nvme_ctrl *nctrl) {struct nvme_rdma_ctrl *ctrl =to_rdma_ctrl (nctrl);
@@ -1004,25 +849,6 @@ static void nvme_rdma_free_ctrl(struct nvme_ctrl *nctrl)kfree (ctrl); } -static void nvme_rdma_reconnect_or_remove (struct nvme_rdma_ctrl *ctrl) -{ -/* If we are resetting/deleting then do nothing */ -if (ctrl->ctrl.state !=NVME_CTRL_CONNECTING ) { -WARN_ON_ONCE (ctrl->ctrl.state ==NVME_CTRL_NEW || - ctrl->ctrl.state ==NVME_CTRL_LIVE ); -return ; - } - -if (nvmf_should_reconnect (&ctrl->ctrl)) { -dev_info (ctrl->ctrl.device,"Reconnecting in %d seconds...\n" , - ctrl->ctrl.opts->reconnect_delay); -queue_delayed_work (nvme_wq, &ctrl->ctrl.connect_work, - ctrl->ctrl.opts->reconnect_delay *HZ ); - }else { -nvme_delete_ctrl (&ctrl->ctrl); - } -} -static int nvme_rdma_alloc_admin_tag_set (struct nvme_ctrl *ctrl) {
@@ -1032,152 +858,6 @@ static int nvme_rdma_alloc_admin_tag_set(struct nvme_ctrl *ctrl) NVME_RDMA_DATA_SGL_SIZE); } -static int nvme_rdma_setup_ctrl (struct nvme_rdma_ctrl *ctrl,bool new) -{ -int ret; -bool changed; - - ret =nvmf_configure_admin_queue (&ctrl->ctrl, new); -if (ret) -return ret; - -if (ctrl->ctrl.icdoff) { - ret = -EOPNOTSUPP ; -dev_err (ctrl->ctrl.device,"icdoff is not supported!\n" ); - goto destroy_admin; - } - -if (!(ctrl->ctrl.sgls & (1 <<2 ))) { - ret = -EOPNOTSUPP ; -dev_err (ctrl->ctrl.device, -"Mandatory keyed sgls are not supported!\n" ); - goto destroy_admin; - } - -if (ctrl->ctrl.opts->queue_size > ctrl->ctrl.sqsize +1 ) { -dev_warn (ctrl->ctrl.device, -"queue_size %zu > ctrl sqsize %u, clamping down\n" , - ctrl->ctrl.opts->queue_size, ctrl->ctrl.sqsize +1 ); - } - -if (ctrl->ctrl.sqsize +1 >NVME_RDMA_MAX_QUEUE_SIZE ) { -dev_warn (ctrl->ctrl.device, -"ctrl sqsize %u > max queue size %u, clamping down\n" , - ctrl->ctrl.sqsize +1 ,NVME_RDMA_MAX_QUEUE_SIZE ); - ctrl->ctrl.sqsize =NVME_RDMA_MAX_QUEUE_SIZE -1 ; - } - -if (ctrl->ctrl.sqsize +1 > ctrl->ctrl.maxcmd) { -dev_warn (ctrl->ctrl.device, -"sqsize %u > ctrl maxcmd %u, clamping down\n" , - ctrl->ctrl.sqsize +1 , ctrl->ctrl.maxcmd); - ctrl->ctrl.sqsize = ctrl->ctrl.maxcmd -1 ; - } - -if (ctrl->ctrl.sgls & (1 <<20 )) - ctrl->use_inline_data = true; - -if (ctrl->ctrl.queue_count >1 ) { - ret =nvme_rdma_configure_io_queues (ctrl, new); -if (ret) - goto destroy_admin; - } - - changed =nvme_change_ctrl_state (&ctrl->ctrl,NVME_CTRL_LIVE ); -if (!changed) { -/* -* state change failure is ok if we started ctrl delete, -* unless we're during creation of a new controller to -* avoid races with teardown flow. -*/ -WARN_ON_ONCE (ctrl->ctrl.state !=NVME_CTRL_DELETING && - ctrl->ctrl.state !=NVME_CTRL_DELETING_NOIO ); -WARN_ON_ONCE (new); - ret = -EINVAL ; - goto destroy_io; - } - -nvme_start_ctrl (&ctrl->ctrl); -return 0 ; - -destroy_io: -if (ctrl->ctrl.queue_count >1 ) { -nvme_quiesce_io_queues (&ctrl->ctrl); -nvme_sync_io_queues (&ctrl->ctrl); -nvme_rdma_stop_io_queues (ctrl); -nvme_cancel_tagset (&ctrl->ctrl); -if (new) -nvme_remove_io_tag_set (&ctrl->ctrl); -nvme_rdma_free_io_queues (ctrl); - } -destroy_admin: -nvme_quiesce_admin_queue (&ctrl->ctrl); -blk_sync_queue (ctrl->ctrl.admin_q); -nvme_rdma_stop_queue (&ctrl->ctrl,0 ); -nvme_cancel_admin_tagset (&ctrl->ctrl); -if (new) -nvme_remove_admin_tag_set (&ctrl->ctrl); -nvme_rdma_destroy_admin_queue (ctrl); -return ret; -} - -static void nvme_rdma_reconnect_ctrl_work (struct work_struct *work) -{ -struct nvme_ctrl *nctrl =container_of (to_delayed_work (work), - struct nvme_ctrl, connect_work); -struct nvme_rdma_ctrl *ctrl =to_rdma_ctrl (nctrl); - - ++ctrl->ctrl.nr_reconnects; - -if (nvme_rdma_setup_ctrl (ctrl, false)) - goto requeue; - -dev_info (ctrl->ctrl.device,"Successfully reconnected (%d attempts)\n" , - ctrl->ctrl.nr_reconnects); - - ctrl->ctrl.nr_reconnects =0 ; - -return ; - -requeue: -dev_info (ctrl->ctrl.device,"Failed reconnect attempt %d\n" , - ctrl->ctrl.nr_reconnects); -nvme_rdma_reconnect_or_remove (ctrl); -} - -static void nvme_rdma_error_recovery_work (struct work_struct *work) -{ -struct nvme_ctrl *nctrl =container_of (work, - struct nvme_ctrl, err_work); -struct nvme_rdma_ctrl *ctrl =to_rdma_ctrl (nctrl); - -nvme_stop_keep_alive (&ctrl->ctrl); -flush_work (&ctrl->ctrl.async_event_work); -nvme_rdma_teardown_io_queues (ctrl, false); -nvme_unquiesce_io_queues (&ctrl->ctrl); -nvme_rdma_teardown_admin_queue (ctrl, false); -nvme_unquiesce_admin_queue (&ctrl->ctrl); -nvme_auth_stop (&ctrl->ctrl); - -if (!nvme_change_ctrl_state (&ctrl->ctrl,NVME_CTRL_CONNECTING )) { -/* state change failure is ok if we started ctrl delete */ -WARN_ON_ONCE (ctrl->ctrl.state !=NVME_CTRL_DELETING && - ctrl->ctrl.state !=NVME_CTRL_DELETING_NOIO ); -return ; - } - -nvme_rdma_reconnect_or_remove (ctrl); -} - -static void nvme_rdma_error_recovery (struct nvme_rdma_ctrl *ctrl) -{ -if (!nvme_change_ctrl_state (&ctrl->ctrl,NVME_CTRL_RESETTING )) -return ; - -dev_warn (ctrl->ctrl.device,"starting error recovery\n" ); -queue_work (nvme_reset_wq, &ctrl->ctrl.err_work); -} -static void nvme_rdma_end_request (struct nvme_rdma_request *req) {struct request *rq =blk_mq_rq_from_pdu (req);
@@ -1199,7 +879,7 @@ static void nvme_rdma_wr_error(struct ib_cq *cq, struct ib_wc *wc,"%s for CQE 0x%p failed with status %s (%d)\n" , op, wc->wr_cqe, ib_wc_status_msg(wc->status), wc->status); -nvme_rdma_error_recovery (ctrl); +nvmf_error_recovery (&ctrl->ctrl); }static void nvme_rdma_memreg_done (struct ib_cq *cq,struct ib_wc *wc)
@@ -1718,7 +1398,7 @@ static void nvme_rdma_process_nvme_rsp(struct nvme_rdma_queue *queue,dev_err (queue->ctrl->ctrl.device,"got bad command_id %#x on QP %#x\n" , cqe->command_id, queue->qp->qp_num); -nvme_rdma_error_recovery (queue->ctrl); +nvmf_error_recovery (&queue->ctrl->ctrl);return ; } req =blk_mq_rq_to_pdu (rq);
@@ -1732,7 +1412,7 @@ static void nvme_rdma_process_nvme_rsp(struct nvme_rdma_queue *queue,dev_err (queue->ctrl->ctrl.device,"Bogus remote invalidation for rkey %#x\n" , req->mr ? req->mr->rkey :0 ); -nvme_rdma_error_recovery (queue->ctrl); +nvmf_error_recovery (&queue->ctrl->ctrl); } }else if (req->mr) {int ret;
@@ -1742,7 +1422,7 @@ static void nvme_rdma_process_nvme_rsp(struct nvme_rdma_queue *queue,dev_err (queue->ctrl->ctrl.device,"Queueing INV WR for rkey %#x failed (%d)\n" , req->mr->rkey, ret); -nvme_rdma_error_recovery (queue->ctrl); +nvmf_error_recovery (&queue->ctrl->ctrl); }/* the local invalidation completion will end the request */ return ;
@@ -1769,7 +1449,7 @@ static void nvme_rdma_recv_done(struct ib_cq *cq, struct ib_wc *wc)if (unlikely (wc->byte_len < len)) {dev_err (queue->ctrl->ctrl.device,"Unexpected nvme completion length(%d)\n" , wc->byte_len); -nvme_rdma_error_recovery (queue->ctrl); +nvmf_error_recovery (&queue->ctrl->ctrl);return ; }
@@ -1939,7 +1619,7 @@ static int nvme_rdma_cm_handler(struct rdma_cm_id *cm_id,case RDMA_CM_EVENT_TIMEWAIT_EXIT :dev_dbg (queue->ctrl->ctrl.device,"disconnect received - connection closed\n" ); -nvme_rdma_error_recovery (queue->ctrl); +nvmf_error_recovery (&queue->ctrl->ctrl);break ;case RDMA_CM_EVENT_DEVICE_REMOVAL :/* device removal is handled via the ib_client API */
@@ -1947,7 +1627,7 @@ static int nvme_rdma_cm_handler(struct rdma_cm_id *cm_id,default :dev_err (queue->ctrl->ctrl.device,"Unexpected RDMA CM event (%d)\n" , ev->event); -nvme_rdma_error_recovery (queue->ctrl); +nvmf_error_recovery (&queue->ctrl->ctrl);break ; }
@@ -2000,7 +1680,7 @@ static enum blk_eh_timer_return nvme_rdma_timeout(struct request *rq)* LIVE state should trigger the normal error recovery which will * handle completing this request. */ -nvme_rdma_error_recovery (ctrl); +nvmf_error_recovery (&ctrl->ctrl);return BLK_EH_RESET_TIMER ; }
@@ -2203,41 +1883,9 @@ static const struct blk_mq_ops nvme_rdma_admin_mq_ops = { .timeout = nvme_rdma_timeout, }; -static void nvme_rdma_shutdown_ctrl (struct nvme_rdma_ctrl *ctrl,bool shutdown) -{ -nvme_rdma_teardown_io_queues (ctrl, shutdown); -nvme_quiesce_admin_queue (&ctrl->ctrl); -nvme_disable_ctrl (&ctrl->ctrl, shutdown); -nvme_rdma_teardown_admin_queue (ctrl, shutdown); -} -static void nvme_rdma_delete_ctrl (struct nvme_ctrl *ctrl) { -nvme_rdma_shutdown_ctrl (to_rdma_ctrl (ctrl), true); -} - -static void nvme_rdma_reset_ctrl_work (struct work_struct *work) -{ -struct nvme_rdma_ctrl *ctrl = - container_of(work, struct nvme_rdma_ctrl, ctrl.reset_work); - -nvme_stop_ctrl (&ctrl->ctrl); -nvme_rdma_shutdown_ctrl (ctrl, false); - -if (!nvme_change_ctrl_state (&ctrl->ctrl,NVME_CTRL_CONNECTING )) { -/* state change failure should never happen */ -WARN_ON_ONCE (1 ); -return ; - } - -if (nvme_rdma_setup_ctrl (ctrl, false)) - goto out_fail; - -return ; - -out_fail: - ++ctrl->ctrl.nr_reconnects; -nvme_rdma_reconnect_or_remove (ctrl); +nvmf_teardown_ctrl (ctrl, true); }static const struct nvme_ctrl_ops nvme_rdma_ctrl_ops = {
@@ -2251,7 +1899,7 @@ static const struct nvme_ctrl_ops nvme_rdma_ctrl_ops = { .submit_async_event = nvme_rdma_submit_async_event, .delete_ctrl = nvme_rdma_delete_ctrl, .get_address = nvmf_get_address, - .stop_ctrl = nvme_rdma_stop_ctrl, + .stop_ctrl = nvmf_stop_ctrl, };/*
@@ -2291,6 +1939,9 @@ static struct nvme_fabrics_ops nvme_rdma_fabrics_ops = { .start_queue = nvme_rdma_start_queue, .stop_queue = nvme_rdma_stop_queue, .alloc_admin_tag_set = nvme_rdma_alloc_admin_tag_set, + .alloc_tag_set = nvme_rdma_alloc_tag_set, + .nr_io_queues = nvme_rdma_nr_io_queues, + .set_io_queues = nvme_rdma_set_io_queues, }; static struct nvme_ctrl *nvme_rdma_create_ctrl(struct device *dev,
@@ -2341,10 +1992,9 @@ static struct nvme_ctrl *nvme_rdma_create_ctrl(struct device *dev, }INIT_DELAYED_WORK (&ctrl->ctrl.connect_work, - nvme_rdma_reconnect_ctrl_work); -INIT_WORK (&ctrl->ctrl.err_work, nvme_rdma_error_recovery_work); -INIT_WORK (&ctrl->ctrl.reset_work, nvme_rdma_reset_ctrl_work); - + nvmf_reconnect_ctrl_work); +INIT_WORK (&ctrl->ctrl.err_work, nvmf_error_recovery_work); +INIT_WORK (&ctrl->ctrl.reset_work, nvmf_reset_ctrl_work); ctrl->ctrl.queue_count = opts->nr_io_queues + opts->nr_write_queues + opts->nr_poll_queues +1 ; ctrl->ctrl.sqsize = opts->queue_size -1 ;
@@ -2364,7 +2014,7 @@ static struct nvme_ctrl *nvme_rdma_create_ctrl(struct device *dev, changed =nvme_change_ctrl_state (&ctrl->ctrl,NVME_CTRL_CONNECTING );WARN_ON_ONCE (!changed); - ret =nvme_rdma_setup_ctrl (ctrl, true); + ret =nvmf_setup_ctrl (&ctrl->ctrl, true);if (ret) goto out_uninit_ctrl;
Signature · 1 line
2.39.2