Handle update hardware queues and queue freeze more carefully
Subsystems (2)
v4 cover Handle update hardware queues and queue freeze more carefully
Hi,
This update version makes sure the unfreeze call is done when recreating the queues. I was able to reproduce hanging I/Os when we go into error recovery mode for FC and TCP [1]. Unfortunatly, I don't have access to a RDMA setup to verify but as the code is identically to the TCP, RDMA looks to like to suffer from the same problem.
Thanks,
Daniel
[1] https://lore.kernel.org/linux-nvme/[email protected]/
https://lore.kernel.org/linux-nvme/[email protected]/
v1:
- https://lore.kernel.org/linux-nvme/[email protected]/
v2:
- https://lore.kernel.org/linux-nvme/[email protected]/
- reviewed tags collected
- added 'update hardware queues' for all transport
- added fix for fc hanger in nvme_wait_freeze_timeout
v3:
- https://lore.kernel.org/linux-nvme/[email protected]/
- dropped 'nvme-fc: Freeze queues before destroying them'
- added James' two patches
v4:
- added 'nvme-*: Unfreeze queues on reconnect'
- added Hannes' reviewed tags
Daniel Wagner (5):
nvme-fc: Update hardware queues before using them
nvme-rdma: Update number of hardware queues before using them
nvme-fc: Wait with a timeout for queue to freeze
nvme-tcp: Unfreeze queues on reconnect
nvme-rdma: Unfreeze queues on reconnect
Hannes Reinecke (1):
nvme-tcp: Update number of hardware queues before using them
James Smart (2):
nvme-fc: avoid race between time out and tear down
nvme-fc: fix controller reset hang during traffic
drivers/nvme/host/fc.c | 28 +++++++++++++++++++--------- drivers/nvme/host/rdma.c | 15 ++++++++------- drivers/nvme/host/tcp.c | 18 +++++++++--------- 3 files changed, 36 insertions(+), 25 deletions(-)
Signature · 1 line
2.29.2
PATCH v4 1/8 nvme-fc: Update hardware queues before using them
In case the number of hardware queues changes, do the update the tagset and ctx to hctx first before using the mapping to recreate and connnect the IO queues.
--- drivers/nvme/host/fc.c | 16 ++++++++-------- 1 file changed, 8 insertions(+), 8 deletions(-)
drivers/nvme/host/fc.c +8 -8
@@ -2951,14 +2951,6 @@ nvme_fc_recreate_io_queues(struct nvme_fc_ctrl *ctrl)if (ctrl->ctrl.queue_count ==1 )return 0 ; - ret =nvme_fc_create_hw_io_queues (ctrl, ctrl->ctrl.sqsize +1 ); -if (ret) - goto out_free_io_queues; - - ret =nvme_fc_connect_io_queues (ctrl, ctrl->ctrl.sqsize +1 ); -if (ret) - goto out_delete_hw_queues; -if (prior_ioq_cnt != nr_io_queues) { dev_info(ctrl->ctrl.device, "reconnect: revising io queue count from %d to %d\n",
@@ -2968,6 +2960,14 @@ nvme_fc_recreate_io_queues(struct nvme_fc_ctrl *ctrl)nvme_unfreeze (&ctrl->ctrl); } + ret =nvme_fc_create_hw_io_queues (ctrl, ctrl->ctrl.sqsize +1 ); +if (ret) + goto out_free_io_queues; + + ret =nvme_fc_connect_io_queues (ctrl, ctrl->ctrl.sqsize +1 ); +if (ret) + goto out_delete_hw_queues; +return 0 ; out_delete_hw_queues:
Signature · 1 line
2.29.2
PATCH v4 2/8 nvme-tcp: Update number of hardware queues before using them
In reply to Daniel WagnerFrom: Hannes Reinecke <[email protected]>
When the number of hardware queues changes during resetting we should update the tagset first before using it.
--- drivers/nvme/host/tcp.c | 14 ++++++-------- 1 file changed, 6 insertions(+), 8 deletions(-)
drivers/nvme/host/tcp.c +6 -8
@@ -1789,6 +1789,7 @@ static void nvme_tcp_destroy_io_queues(struct nvme_ctrl *ctrl, bool remove)static int nvme_tcp_configure_io_queues (struct nvme_ctrl *ctrl,bool new) {int ret; +u32 prior_q_cnt = ctrl->queue_count; ret =nvme_tcp_alloc_io_queues (ctrl);if (ret)
@@ -1806,14 +1807,7 @@ static int nvme_tcp_configure_io_queues(struct nvme_ctrl *ctrl, bool new) ret =PTR_ERR (ctrl->connect_q); goto out_free_tag_set; } - } - - ret =nvme_tcp_start_io_queues (ctrl); -if (ret) - goto out_cleanup_connect_q; - -if (!new) { -nvme_start_queues (ctrl); + }else if (prior_q_cnt != ctrl->queue_count) {if (!nvme_wait_freeze_timeout (ctrl,NVME_IO_TIMEOUT )) {/* * If we timed out waiting for freeze we are likely to
@@ -1828,6 +1822,10 @@ static int nvme_tcp_configure_io_queues(struct nvme_ctrl *ctrl, bool new)nvme_unfreeze (ctrl); } + ret =nvme_tcp_start_io_queues (ctrl); +if (ret) + goto out_cleanup_connect_q; +return 0 ; out_wait_freeze_timed_out:
Signature · 1 line
2.29.2
PATCH v4 3/8 nvme-rdma: Update number of hardware queues before using them
In reply to Daniel WagnerWhen the number of hardware queues changes during resetting we should update the tagset first before using it.
--- drivers/nvme/host/rdma.c | 13 ++++++------- 1 file changed, 6 insertions(+), 7 deletions(-)
drivers/nvme/host/rdma.c +6 -7
@@ -965,6 +965,7 @@ static void nvme_rdma_destroy_io_queues(struct nvme_rdma_ctrl *ctrl,static int nvme_rdma_configure_io_queues (struct nvme_rdma_ctrl *ctrl,bool new) {int ret; +u32 prior_q_cnt = ctrl->ctrl.queue_count; ret =nvme_rdma_alloc_io_queues (ctrl);if (ret)
@@ -982,13 +983,7 @@ static int nvme_rdma_configure_io_queues(struct nvme_rdma_ctrl *ctrl, bool new) ret =PTR_ERR (ctrl->ctrl.connect_q); goto out_free_tag_set; } - } - - ret =nvme_rdma_start_io_queues (ctrl); -if (ret) - goto out_cleanup_connect_q; - -if (!new) { + }else if (prior_q_cnt != ctrl->ctrl.queue_count) {nvme_start_queues (&ctrl->ctrl);if (!nvme_wait_freeze_timeout (&ctrl->ctrl,NVME_IO_TIMEOUT )) {/*
@@ -1004,6 +999,10 @@ static int nvme_rdma_configure_io_queues(struct nvme_rdma_ctrl *ctrl, bool new)nvme_unfreeze (&ctrl->ctrl); } + ret =nvme_rdma_start_io_queues (ctrl); +if (ret) + goto out_cleanup_connect_q; +return 0 ; out_wait_freeze_timed_out:
Signature · 1 line
2.29.2
PATCH v4 4/8 nvme-fc: Wait with a timeout for queue to freeze
In reply to Daniel WagnerDo not wait indifinitly for all queues to freeze. Instead use a timeout and abort the operation if we get stuck.
--- drivers/nvme/host/fc.c | 9 ++++++++- 1 file changed, 8 insertions(+), 1 deletion(-)
drivers/nvme/host/fc.c +8 -1
@@ -2955,7 +2955,14 @@ nvme_fc_recreate_io_queues(struct nvme_fc_ctrl *ctrl)dev_info (ctrl->ctrl.device,"reconnect: revising io queue count from %d to %d\n" , prior_ioq_cnt, nr_io_queues); -nvme_wait_freeze (&ctrl->ctrl); +if (!nvme_wait_freeze_timeout (&ctrl->ctrl,NVME_IO_TIMEOUT )) { +/* +* If we timed out waiting for freeze we are likely to +* be stuck. Fail the controller initialization just +* to be safe. +*/ +return -ENODEV ; + }blk_mq_update_nr_hw_queues (&ctrl->tag_set, nr_io_queues);nvme_unfreeze (&ctrl->ctrl); }
Signature · 1 line
2.29.2
PATCH v4 5/8 nvme-fc: avoid race between time out and tear down
In reply to Daniel WagnerFrom: James Smart <[email protected]>
To avoid race between time out and tear down, in tear down process, first we quiesce the queue, and then delete the timer and cancel the time out work for the queue.
This patch merges the admin and io sync ops into the queue teardown logic as shown in the RDMA patch 3017013dcc "nvme-rdma: avoid race between time out and tear down". There is no teardown_lock in nvme-fc.
--- drivers/nvme/host/fc.c | 2 ++ 1 file changed, 2 insertions(+)
drivers/nvme/host/fc.c +2 -0
@@ -2487,6 +2487,7 @@ __nvme_fc_abort_outstanding_ios(struct nvme_fc_ctrl *ctrl, bool start_queues)*/ if (ctrl->ctrl.queue_count >1 ) {nvme_stop_queues (&ctrl->ctrl); +nvme_sync_io_queues (&ctrl->ctrl);blk_mq_tagset_busy_iter (&ctrl->tag_set, nvme_fc_terminate_exchange, &ctrl->ctrl);blk_mq_tagset_wait_completed_request (&ctrl->tag_set);
@@ -2510,6 +2511,7 @@ __nvme_fc_abort_outstanding_ios(struct nvme_fc_ctrl *ctrl, bool start_queues)* clean up the admin queue. Same thing as above. */ blk_mq_quiesce_queue (ctrl->ctrl.admin_q); +blk_sync_queue (ctrl->ctrl.admin_q);blk_mq_tagset_busy_iter (&ctrl->admin_tag_set, nvme_fc_terminate_exchange, &ctrl->ctrl);blk_mq_tagset_wait_completed_request (&ctrl->admin_tag_set);
Signature · 1 line
2.29.2
PATCH v4 6/8 nvme-fc: fix controller reset hang during traffic
In reply to Daniel WagnerFrom: James Smart <[email protected]>
commit fe35ec58f0d3 ("block: update hctx map when use multiple maps") exposed an issue where we may hang trying to wait for queue freeze during I/O. We call blk_mq_update_nr_hw_queues which may attempt to freeze the queue. However we never started queue freeze when starting the reset, which means that we have inflight pending requests that entered the queue that we will not complete once the queue is quiesced.
So start a freeze before we quiesce the queue, and unfreeze the queue after we successfully connected the I/O queues (the unfreeze is already present in the code). blk_mq_update_nr_hw_queues will be called only after we are sure that the queue was already frozen.
This follows to how the pci driver handles resets.
This patch added logic introduced in commit 9f98772ba307 "nvme-rdma: fix controller reset hang during traffic".
[dwagner: call nvme_unfreeze() unconditionally in
nvme_fc_recreate_io_queues() to match the nvme_start_freeze()]
--- drivers/nvme/host/fc.c | 3 ++- 1 file changed, 2 insertions(+), 1 deletion(-)
drivers/nvme/host/fc.c +2 -1
@@ -2486,6 +2486,7 @@ __nvme_fc_abort_outstanding_ios(struct nvme_fc_ctrl *ctrl, bool start_queues)* (but with error status). */ if (ctrl->ctrl.queue_count >1 ) { +nvme_start_freeze (&ctrl->ctrl);nvme_stop_queues (&ctrl->ctrl);nvme_sync_io_queues (&ctrl->ctrl); blk_mq_tagset_busy_iter(&ctrl->tag_set,
@@ -2966,8 +2967,8 @@ nvme_fc_recreate_io_queues(struct nvme_fc_ctrl *ctrl)return -ENODEV ; }blk_mq_update_nr_hw_queues (&ctrl->tag_set, nr_io_queues); -nvme_unfreeze (&ctrl->ctrl); } +nvme_unfreeze (&ctrl->ctrl); ret =nvme_fc_create_hw_io_queues (ctrl, ctrl->ctrl.sqsize +1 );if (ret)
Signature · 1 line
2.29.2
PATCH v4 7/8 nvme-tcp: Unfreeze queues on reconnect
In reply to Daniel WagnerDuring the queue teardown in nvme_tcp_teardown_io_queues() freeze is called unconditionally. When we reconnect we need to pair the freeze with an unfreeze to avoid hanging I/Os. For newly created connection this is not needed.
--- drivers/nvme/host/tcp.c | 4 +++- 1 file changed, 3 insertions(+), 1 deletion(-)
drivers/nvme/host/tcp.c +3 -1
@@ -1819,9 +1819,11 @@ static int nvme_tcp_configure_io_queues(struct nvme_ctrl *ctrl, bool new) }blk_mq_update_nr_hw_queues (ctrl->tagset, ctrl->queue_count -1 ); -nvme_unfreeze (ctrl); } +if (!new) +nvme_unfreeze (ctrl); + ret =nvme_tcp_start_io_queues (ctrl);if (ret) goto out_cleanup_connect_q;
Signature · 1 line
2.29.2
PATCH v4 8/8 nvme-rdma: Unfreeze queues on reconnect
In reply to Daniel WagnerDuring the queue teardown in nvme_rdma_teardown_io_queues() freeze is called unconditionally. When we reconnect we need to pair the freeze with an unfreeze to avoid hanging I/Os. For newly created connection this is not needed.
--- drivers/nvme/host/rdma.c | 2 ++ 1 file changed, 2 insertions(+)
drivers/nvme/host/rdma.c +2 -0
@@ -901,6 +901,8 @@ static int nvme_rdma_configure_admin_queue(struct nvme_rdma_ctrl *ctrl, error =PTR_ERR (ctrl->ctrl.admin_q); goto out_cleanup_fabrics_q; } + }else { +nvme_unfreeze (&ctrl->ctrl); } error =nvme_rdma_start_queue (ctrl,0 );
Signature · 1 line
2.29.2