nvme/mpath: fix hang when disk goes live over reconnect
Subsystems (3)
PATCH 1/1 nvme/mpath: fix hang when disk goes live over reconnect
nvme_mpath_init_identify() invoked from nvme_init_identify() fetches a fresh ANA log from the ctrl.
This is essential to have an up to date path states for both existing namespaces and for those scan_work may discover once the ctrl is up.
This happens in the following cases:
1) A new ctrl is being connected.
2) An existing ctrl is successfully reconnected.
3) An existing ctrl is being reset.
While in (1) ctrl->namespaces is empty, (2 & 3) may have namespaces, and nvme_read_ana_log() may call nvme_update_ns_ana_state().
This result in a hang when the ANA state of an existing namespace changes and makes the disk live: nvme_mpath_set_live() issues IO to the namespace through the ctrl, which does NOT have IO queues yet. See sample hang below.
Solution:
- move nvme_read_ana_log() call from nvme_mpath_init_identify() into a separate function, nvme_mpath_update().
- call this function in nvme_start_ctrl() - here the ctrl is ready.
Downside: in case nvme_read_ana_log() fails, we do not find this out
in nvme_mpath_init_identify().
Alernative solution:
nvme_get_log() in nvme_mpath_init_identify(), but don't parse. Then nvme_start_ctrl() can just queue_work(nvme_wq, &ctrl->ana_work), along with scan_work.
The ctrl->ana_log_buf contents is valid for scan_work,
and ana_work will re-fetch and parse it.
Downside: fetching an ANA log from the ctrl twice.
Sample failure:
nvme nvme0: starting error recovery
nvme nvme0: Reconnecting in 10 seconds...
block nvme0n6: no usable path - requeuing I/O
INFO: task kworker/u8:3:312 blocked for more than 122 seconds.
Tainted: G E 5.14.5-1.el7.elrepo.x86_64 #1
Workqueue: nvme-wq nvme_tcp_reconnect_ctrl_work [nvme_tcp]
Call Trace:
__schedule+0x2a2/0x7e0
schedule+0x4e/0xb0
io_schedule+0x16/0x40
wait_on_page_bit_common+0x15c/0x3e0
do_read_cache_page+0x1e0/0x410
read_cache_page+0x12/0x20
read_part_sector+0x46/0x100
read_lba+0x121/0x240
efi_partition+0x1d2/0x6a0
bdev_disk_changed.part.0+0x1df/0x430
bdev_disk_changed+0x18/0x20
blkdev_get_whole+0x77/0xe0
blkdev_get_by_dev+0xd2/0x3a0
__device_add_disk+0x1ed/0x310
device_add_disk+0x13/0x20
nvme_mpath_set_live+0x138/0x1b0 [nvme_core]
nvme_update_ns_ana_state+0x2b/0x30 [nvme_core]
nvme_update_ana_state+0xca/0xe0 [nvme_core]
nvme_parse_ana_log+0xac/0x170 [nvme_core]
nvme_read_ana_log+0x7d/0xe0 [nvme_core]
nvme_mpath_init_identify+0x105/0x150 [nvme_core]
nvme_init_identify+0x2df/0x4d0 [nvme_core]
nvme_init_ctrl_finish+0x8d/0x3b0 [nvme_core]
nvme_tcp_setup_ctrl+0x337/0x390 [nvme_tcp]
nvme_tcp_reconnect_ctrl_work+0x24/0x40 [nvme_tcp]
process_one_work+0x1bd/0x360
worker_thread+0x50/0x3d0--- drivers/nvme/host/core.c | 1 + drivers/nvme/host/multipath.c | 9 ++++++--- drivers/nvme/host/nvme.h | 4 ++++ 3 files changed, 11 insertions(+), 3 deletions(-)
drivers/nvme/host/core.c +1 -0
@@ -4325,6 +4325,7 @@ void nvme_start_ctrl(struct nvme_ctrl *ctrl)nvme_enable_aen (ctrl);if (ctrl->queue_count >1 ) { +nvme_mpath_update (ctrl);nvme_queue_scan (ctrl);nvme_start_queues (ctrl); }
drivers/nvme/host/multipath.c +6 -3
@@ -850,9 +850,7 @@ int nvme_mpath_init_identify(struct nvme_ctrl *ctrl, struct nvme_id_ctrl *id)return -ENOMEM ; } ctrl->ana_log_size = ana_log_size; - error =nvme_read_ana_log (ctrl); -if (error) - goto out_uninit; +return 0 ; out_uninit:
@@ -860,6 +858,11 @@ int nvme_mpath_init_identify(struct nvme_ctrl *ctrl, struct nvme_id_ctrl *id)return error; } +void nvme_mpath_update (struct nvme_ctrl *ctrl) +{ +nvme_read_ana_log (ctrl); +} +void nvme_mpath_uninit (struct nvme_ctrl *ctrl) {kfree (ctrl->ana_log_buf);
drivers/nvme/host/nvme.h +4 -0
@@ -746,6 +746,7 @@ void nvme_mpath_add_disk(struct nvme_ns *ns, struct nvme_id_ns *id);void nvme_mpath_remove_disk (struct nvme_ns_head *head);int nvme_mpath_init_identify (struct nvme_ctrl *ctrl,struct nvme_id_ctrl *id);void nvme_mpath_init_ctrl (struct nvme_ctrl *ctrl); +void nvme_mpath_update (struct nvme_ctrl *ctrl);void nvme_mpath_uninit (struct nvme_ctrl *ctrl);void nvme_mpath_stop (struct nvme_ctrl *ctrl);bool nvme_mpath_clear_current_path (struct nvme_ns *ns);
@@ -820,6 +821,9 @@ static inline int nvme_mpath_init_identify(struct nvme_ctrl *ctrl, "Please enable CONFIG_NVME_MULTIPATH for full support of multi-port devices.\n");return 0 ; } +static void nvme_mpath_update (struct nvme_ctrl *ctrl) +{ +}static inline void nvme_mpath_uninit (struct nvme_ctrl *ctrl) { }
Signature · 7 lines
2.25.1 _______________________________________________ Linux-nvme mailing list [email protected] http://lists.infradead.org/mailman/listinfo/linux-nvme
Sagi Grimberg
maintainerQuoted 83 lines · Anton Eidelman
nvme_mpath_init_identify()invoked fromnvme_init_identify()fetches a fresh ANA log from the ctrl.
This is essential to have an up to date path states for both existing namespaces and for thosescan_workmay discover once the ctrl is up.This happens in the following cases:
1) A new ctrl is being connected.
2) An existing ctrl is successfully reconnected.
3) An existing ctrl is being reset.While in (1)
ctrl->namespacesis empty, (2 & 3) may have namespaces, andnvme_read_ana_log()may callnvme_update_ns_ana_state().This result in a hang when the ANA state of an existing namespace changes and makes the disk live:
nvme_mpath_set_live()issues IO to the namespace through the ctrl, which does NOT have IO queues yet. See sample hang below.Solution:
- movenvme_read_ana_log()call fromnvme_mpath_init_identify()into a separate function,nvme_mpath_update().
- call this function innvme_start_ctrl()- here the ctrl is ready.Downside: in case
nvme_read_ana_log()fails, we do not find this out
innvme_mpath_init_identify().Alernative solution:
nvme_get_log()innvme_mpath_init_identify(), but don't parse. Thennvme_start_ctrl()can justqueue_work(nvme_wq, &ctrl->ana_work), along withscan_work.
Thectrl->ana_log_bufcontents is valid forscan_work, andana_workwill re-fetch and parse it.
Downside: fetching an ANA log from the ctrl twice.Sample failure:
nvme nvme0: starting error recovery nvme nvme0: Reconnecting in 10 seconds... block nvme0n6: no usable path - requeuing I/O INFO: task kworker/u8:3:312 blocked for more than 122 seconds. Tainted: G E 5.14.5-1.el7.elrepo.x86_64 #1 Workqueue: nvme-wq nvme_tcp_reconnect_ctrl_work [nvme_tcp] Call Trace: __schedule+0x2a2/0x7e0 schedule+0x4e/0xb0 io_schedule+0x16/0x40 wait_on_page_bit_common+0x15c/0x3e0 do_read_cache_page+0x1e0/0x410 read_cache_page+0x12/0x20 read_part_sector+0x46/0x100 read_lba+0x121/0x240 efi_partition+0x1d2/0x6a0 bdev_disk_changed.part.0+0x1df/0x430 bdev_disk_changed+0x18/0x20 blkdev_get_whole+0x77/0xe0 blkdev_get_by_dev+0xd2/0x3a0 __device_add_disk+0x1ed/0x310 device_add_disk+0x13/0x20 nvme_mpath_set_live+0x138/0x1b0 [nvme_core] nvme_update_ns_ana_state+0x2b/0x30 [nvme_core] nvme_update_ana_state+0xca/0xe0 [nvme_core] nvme_parse_ana_log+0xac/0x170 [nvme_core] nvme_read_ana_log+0x7d/0xe0 [nvme_core] nvme_mpath_init_identify+0x105/0x150 [nvme_core] nvme_init_identify+0x2df/0x4d0 [nvme_core] nvme_init_ctrl_finish+0x8d/0x3b0 [nvme_core] nvme_tcp_setup_ctrl+0x337/0x390 [nvme_tcp] nvme_tcp_reconnect_ctrl_work+0x24/0x40 [nvme_tcp] process_one_work+0x1bd/0x360 worker_thread+0x50/0x3d0Signed-off-by Anton Eidelman <[email protected]>--- drivers/nvme/host/core.c | 1 + drivers/nvme/host/multipath.c | 9 ++++++--- drivers/nvme/host/nvme.h | 4 ++++ 3 files changed, 11 insertions(+), 3 deletions(-)drivers/nvme/host/core.c +0
-0
drivers/nvme/host/core.c void nvme_start_ctrl(struct nvme_ctrl *ctrl) · line 4328
Quoted 4 lines
drivers/nvme/host/core.c +1
-0@@ -4325,6 +4325,7 @@ void nvme_start_ctrl(struct nvme_ctrl *ctrl)nvme_enable_aen (ctrl);if (ctrl->queue_count >1 ) { +nvme_mpath_update (ctrl);
Does this ordering now becomes subtle as it needs to run before we queue the ns scan?
drivers/nvme/host/core.c void nvme_start_ctrl(struct nvme_ctrl *ctrl) · line 4331
Quoted 3 lines
drivers/nvme/host/core.c +0
-0nvme_queue_scan (ctrl);nvme_start_queues (ctrl); }
_______________________________________________
Linux-nvme mailing list
[email protected]
http://lists.infradead.org/mailman/listinfo/linux-nvme
Well, this works for me (passes a test that gets into the described scenario deteministically), probably because we nvme_start_queues() earlier in nvme_tcp_configure_io_queues().
Same is correct for rdma transport.
Quoted 81 lines · Sagi Grimberg
Quoted 78 lines · Anton Eidelman
nvme_mpath_init_identify()invoked fromnvme_init_identify()fetches a fresh ANA log from the ctrl.
This is essential to have an up to date path states for both existing namespaces and for thosescan_workmay discover once the ctrl is up.This happens in the following cases:
1) A new ctrl is being connected.
2) An existing ctrl is successfully reconnected.
3) An existing ctrl is being reset.While in (1)
ctrl->namespacesis empty, (2 & 3) may have namespaces, andnvme_read_ana_log()may callnvme_update_ns_ana_state().This result in a hang when the ANA state of an existing namespace changes and makes the disk live:
nvme_mpath_set_live()issues IO to the namespace through the ctrl, which does NOT have IO queues yet. See sample hang below.Solution:
- movenvme_read_ana_log()call fromnvme_mpath_init_identify()into a separate function,nvme_mpath_update().
- call this function innvme_start_ctrl()- here the ctrl is ready.Downside: in case
nvme_read_ana_log()fails, we do not find this out
innvme_mpath_init_identify().Alernative solution:
nvme_get_log()innvme_mpath_init_identify(), but don't parse. Thennvme_start_ctrl()can justqueue_work(nvme_wq, &ctrl->ana_work), along withscan_work.
Thectrl->ana_log_bufcontents is valid forscan_work, andana_workwill re-fetch and parse it.
Downside: fetching an ANA log from the ctrl twice.Sample failure:
nvme nvme0: starting error recovery nvme nvme0: Reconnecting in 10 seconds... block nvme0n6: no usable path - requeuing I/O INFO: task kworker/u8:3:312 blocked for more than 122 seconds. Tainted: G E 5.14.5-1.el7.elrepo.x86_64 #1 Workqueue: nvme-wq nvme_tcp_reconnect_ctrl_work [nvme_tcp] Call Trace: __schedule+0x2a2/0x7e0 schedule+0x4e/0xb0 io_schedule+0x16/0x40 wait_on_page_bit_common+0x15c/0x3e0 do_read_cache_page+0x1e0/0x410 read_cache_page+0x12/0x20 read_part_sector+0x46/0x100 read_lba+0x121/0x240 efi_partition+0x1d2/0x6a0 bdev_disk_changed.part.0+0x1df/0x430 bdev_disk_changed+0x18/0x20 blkdev_get_whole+0x77/0xe0 blkdev_get_by_dev+0xd2/0x3a0 __device_add_disk+0x1ed/0x310 device_add_disk+0x13/0x20 nvme_mpath_set_live+0x138/0x1b0 [nvme_core] nvme_update_ns_ana_state+0x2b/0x30 [nvme_core] nvme_update_ana_state+0xca/0xe0 [nvme_core] nvme_parse_ana_log+0xac/0x170 [nvme_core] nvme_read_ana_log+0x7d/0xe0 [nvme_core] nvme_mpath_init_identify+0x105/0x150 [nvme_core] nvme_init_identify+0x2df/0x4d0 [nvme_core] nvme_init_ctrl_finish+0x8d/0x3b0 [nvme_core] nvme_tcp_setup_ctrl+0x337/0x390 [nvme_tcp] nvme_tcp_reconnect_ctrl_work+0x24/0x40 [nvme_tcp] process_one_work+0x1bd/0x360 worker_thread+0x50/0x3d0Signed-off-by Anton Eidelman <[email protected]>--- drivers/nvme/host/core.c | 1 + drivers/nvme/host/multipath.c | 9 ++++++--- drivers/nvme/host/nvme.h | 4 ++++ 3 files changed, 11 insertions(+), 3 deletions(-)
Quoted 15 lines
drivers/nvme/host/core.c void nvme_start_ctrl(struct nvme_ctrl *ctrl) · line 4327
Quoted 8 lines
drivers/nvme/host/core.c +1
-0@@ -4325,6 +4325,7 @@ void nvme_start_ctrl(struct nvme_ctrl *ctrl)nvme_enable_aen (ctrl);if (ctrl->queue_count >1 ) { +nvme_mpath_update (ctrl);Does this ordering now becomes subtle as it needs to run before we queue the ns scan?
Quoted 3 lines
drivers/nvme/host/core.c +0
-0nvme_queue_scan (ctrl);nvme_start_queues (ctrl); }
_______________________________________________
Linux-nvme mailing list
[email protected]
http://lists.infradead.org/mailman/listinfo/linux-nvme
Christoph Hellwig
maintainerdrivers/nvme/host/multipath.c int nvme_mpath_init_identify(struct nvme_ctrl *ctrl, struct nvme_id_ctrl *id) · line 864
Quoted 4 lines · Anton Eidelman
drivers/nvme/host/multipath.c +4
-0+void nvme_mpath_update (struct nvme_ctrl *ctrl) +{ +nvme_read_ana_log (ctrl); +}
What is the point of this wrapper vs just calling nvme_read_ana_log directly? Also shouldn;t we propagate the error here and through nvme_start_ctrl?
_______________________________________________
Linux-nvme mailing list
[email protected]
http://lists.infradead.org/mailman/listinfo/linux-nvme
Quoted 6 lines
Well, this works for me (passes a test that gets into the described scenario deteministically), probably because we
nvme_start_queues()earlier innvme_tcp_configure_io_queues().
Same is correct for rdma transport.
OK, looks good to me:
_______________________________________________
Linux-nvme mailing list
[email protected]
http://lists.infradead.org/mailman/listinfo/linux-nvme
The wrapper is to:
- follow the naming convention for mpath functions called from core
- allow the implementation to have more logic in future
As for error propagation, nvme_start_ctrl is void. Any error in nvme_read_ana_log() will have the same effect as when it is invoked from nvme_ana_work(), i.e. will be ignored.
Quoted 9 lines · Christoph Hellwig
drivers/nvme/host/multipath.c int nvme_mpath_init_identify(struct nvme_ctrl *ctrl, struct nvme_id_ctrl *id) · line 864
Quoted 4 lines · Anton Eidelman
drivers/nvme/host/multipath.c +4
-0+void nvme_mpath_update (struct nvme_ctrl *ctrl) +{ +nvme_read_ana_log (ctrl); +}What is the point of this wrapper vs just calling
nvme_read_ana_logdirectly? Also shouldn;t we propagate the error here and throughnvme_start_ctrl?
_______________________________________________
Linux-nvme mailing list
[email protected]
http://lists.infradead.org/mailman/listinfo/linux-nvme
Christoph Hellwig
maintainerQuoted 5 lines · Anton Eidelman
The wrapper is to:
- follow the naming convention for mpath functions called from core
- allow the implementation to have more logic in futureAs for error propagation,
nvme_start_ctrlis void.
Which of course can be changed, like all software.
Quoted 2 lines
Any error in
nvme_read_ana_log()will have the same effect as when it is invoked fromnvme_ana_work(), i.e. will be ignored.
We need to return negative errors during initialization otherwise we're going to run into teardown problems.
_______________________________________________
Linux-nvme mailing list
[email protected]
http://lists.infradead.org/mailman/listinfo/linux-nvme
Quoted 7 lines · Christoph Hellwig
Quoted 5 lines · Anton Eidelman
The wrapper is to:
- follow the naming convention for mpath functions called from core
- allow the implementation to have more logic in futureAs for error propagation,
nvme_start_ctrlis void.
Quoted 7 lines
Which of course can be changed, like all software.
Quoted 2 lines
Any error in
nvme_read_ana_log()will have the same effect as when it is invoked fromnvme_ana_work(), i.e. will be ignored.We need to return negative errors during initialization otherwise we're going to run into teardown problems.
How do we proceed with this fix?
I believe error propagation here is not wanted because:
1) A failure to fetch or parse the ANA log should not be considered an error in ctrl initialization.
2) Such error will not cause problems in teardown.
3) The same failure is possible in ANA work and we do not take any action in such case.
4) Adding support for failure to nvme_start_ctrl() adds complexity and does not look useful due to the above 1-3.
_______________________________________________
Linux-nvme mailing list
[email protected]
http://lists.infradead.org/mailman/listinfo/linux-nvme
Christoph Hellwig
maintainerQuoted 1 line · Anton Eidelman
How do we proceed with this fix?
Please resend with the suggested updates.
Quoted 3 lines
I believe error propagation here is not wanted because:
1) A failure to fetch or parse the ANA log should not be considered an error in ctrl initialization.
We must handle error that are due to a controller failing to initialize.
Quoted 3 lines
2) Such error will not cause problems in teardown.
3) The same failure is possible in ANA work and we do not take any action in such case.
Failing in a workqueue is different from failing in the initialization path.
Quoted 2 lines
4) Adding support for failure to
nvme_start_ctrl()adds complexity and does not look useful due to the above 1-3.
---end quoted text---
_______________________________________________
Linux-nvme mailing list
[email protected]
http://lists.infradead.org/mailman/listinfo/linux-nvme
Sagi Grimberg
maintainerQuoted 9 lines
Quoted 2 lines
How do we proceed with this fix?
Please resend with the suggested updates.
Quoted 3 lines
I believe error propagation here is not wanted because:
1) A failure to fetch or parse the ANA log should not be considered an error in ctrl initialization.
Quoted 11 lines
We must handle error that are due to a controller failing to initialize.
Quoted 3 lines
2) Such error will not cause problems in teardown.
3) The same failure is possible in ANA work and we do not take any action in such case.Failing in a workqueue is different from failing in the initialization path.
Quoted 2 lines
4) Adding support for failure to
nvme_start_ctrl()adds complexity and does not look useful due to the above 1-3.
The feedback here is that this patch is changing the functionality as before we failed initialization and now we don't.
What is the issue in propagating the error and then modify the call-sites? Shouldn't it be simple enough to do? --
drivers/nvme/host/fc.c +5 -2
@@ -3158,8 +3158,11 @@ nvme_fc_create_association(struct nvme_fc_ctrl *ctrl) ctrl->ctrl.nr_reconnects =0 ; -if (changed) -nvme_start_ctrl (&ctrl->ctrl); +if (changed) { + ret =nvme_start_ctrl (&ctrl->ctrl); +if (ret) + goto out_term_aen_ops; + }return 0 ;/* Success */
drivers/nvme/host/pci.c +4 -1
@@ -2827,7 +2827,10 @@ static void nvme_reset_work(struct work_struct *work) &nvme_pci_attr_group)) dev->attrs_added = true; -nvme_start_ctrl (&dev->ctrl); + ret =nvme_start_ctrl (&dev->ctrl); +if (ret) + goto out_unlock; +return ; out_unlock:
drivers/nvme/host/rdma.c +4 -1
@@ -1141,7 +1141,10 @@ static int nvme_rdma_setup_ctrl(struct nvme_rdma_ctrl *ctrl, bool new) goto destroy_io; } -nvme_start_ctrl (&ctrl->ctrl); + ret =nvme_start_ctrl (&ctrl->ctrl); +if (ret) + goto destroy_io; +return 0 ; destroy_io:
drivers/nvme/host/tcp.c +0 -0
@@ -923,6 +923,11 @@ static void nvme_tcp_fail_request(struct nvme_tcp_request *req) nvme_tcp_end_request(blk_mq_rq_from_pdu(req),
NVME_SC_HOST_PATH_ERROR);
}
@@ -2045,7 +2052,10 @@ static int nvme_tcp_setup_ctrl(struct nvme_ctrl *ctrl, bool new)
goto destroy_io;
}- nvme_start_ctrl(ctrl);
+ ret = nvme_start_ctrl(ctrl);
+ if (ret)
+ goto destroy_io;
+
return 0; destroy_io:
drivers/nvme/target/loop.c +3 -2
@@ -490,7 +490,9 @@ static void nvme_loop_reset_ctrl_work(struct work_struct *work)if (!nvme_change_ctrl_state (&ctrl->ctrl,NVME_CTRL_LIVE ))WARN_ON_ONCE (1 ); -nvme_start_ctrl (&ctrl->ctrl); + ret =nvme_start_ctrl (&ctrl->ctrl); +if (ret) + goto out_destroy_io;return ; --
_______________________________________________
Linux-nvme mailing list
[email protected]
http://lists.infradead.org/mailman/listinfo/linux-nvme
Looking into making nvme_start_ctrl capable of returning an error. This is a bit more complicated IMHO, because the in most transports the ctrl is already in LIVE state when nvme_start_ctrl is invoked, so we need to bail out carefully.
Quoted 124 lines · Sagi Grimberg
Quoted 108 lines · Sagi Grimberg
Quoted 9 lines
Quoted 2 lines
How do we proceed with this fix?
Please resend with the suggested updates.
Quoted 3 lines
I believe error propagation here is not wanted because:
1) A failure to fetch or parse the ANA log should not be considered an error in ctrl initialization.Quoted 11 lines
We must handle error that are due to a controller failing to initialize.
Quoted 3 lines
2) Such error will not cause problems in teardown.
3) The same failure is possible in ANA work and we do not take any action in such case.Failing in a workqueue is different from failing in the initialization path.
Quoted 2 lines
4) Adding support for failure to
nvme_start_ctrl()adds complexity and does not look useful due to the above 1-3.The feedback here is that this patch is changing the functionality as before we failed initialization and now we don't.
What is the issue in propagating the error and then modify the call-sites? Shouldn't it be simple enough to do?
drivers/nvme/host/fc.c +5
-2@@ -3158,8 +3158,11 @@ nvme_fc_create_association(struct nvme_fc_ctrl *ctrl) ctrl->ctrl.nr_reconnects = 0; - if (changed) - nvme_start_ctrl(&ctrl->ctrl); + if (changed) { + ret = nvme_start_ctrl(&ctrl->ctrl); + if (ret) + goto out_term_aen_ops; + } return 0; /* Success */drivers/nvme/host/pci.c +4
-1@@ -2827,7 +2827,10 @@ static void nvme_reset_work(struct work_struct *work) &nvme_pci_attr_group)) dev->attrs_added = true; - nvme_start_ctrl(&dev->ctrl); + ret = nvme_start_ctrl(&dev->ctrl); + if (ret) + goto out_unlock; + return;
out_unlock:drivers/nvme/host/rdma.c +4
-1@@ -1141,7 +1141,10 @@ static int nvme_rdma_setup_ctrl(struct nvme_rdma_ctrl *ctrl, bool new) goto destroy_io; } - nvme_start_ctrl(&ctrl->ctrl); + ret = nvme_start_ctrl(&ctrl->ctrl); + if (ret) + goto destroy_io; + return 0;destroy_io:
drivers/nvme/host/tcp.c +4
-1@@ -923,6 +923,11 @@ static void nvme_tcp_fail_request(struct nvme_tcp_request *req) nvme_tcp_end_request(blk_mq_rq_from_pdu(req),NVME_SC_HOST_PATH_ERROR ); }@@ -2045,7 +2052,10 @@ static int nvme_tcp_setup_ctrl(struct nvme_ctrl *ctrl, bool new) goto destroy_io; } - nvme_start_ctrl(ctrl); + ret = nvme_start_ctrl(ctrl); + if (ret) + goto destroy_io; + return 0;destroy_io:
drivers/nvme/target/loop.c +0
-0drivers/nvme/target/loop.c static void nvme_loop_reset_ctrl_work(struct · line 498
Quoted 10 lines
drivers/nvme/target/loop.c +3
-1@@ -490,7 +490,9 @@ static void nvme_loop_reset_ctrl_work(struct work_struct *work) if (!nvme_change_ctrl_state(&ctrl->ctrl, NVME_CTRL_LIVE)) WARN_ON_ONCE(1); - nvme_start_ctrl(&ctrl->ctrl); + ret = nvme_start_ctrl(&ctrl->ctrl); + if (ret) + goto out_destroy_io; return;So Anton, can this suggestion work? I think Hannes reported the same issue that this patch addresses.
v2 cover nvme/mpath: fix hang when disk goes live over reconnect
In reply to Anton EidelmanRevising the original v1 patch submitted on Sep 18, 2021, which deferred nvme_read_ana_log() until nvme_start_ctrl() and raised concerns since the former might fail, while the latter does not currently return an error. I did not see appropriate to introduce error handling in the nvme_start_ctrl() call-sites.
Instead, this patch keeps the nvme_read_ana_log() in its original place inside nvme_mpath_init_identify(),
but defers updating of the existing ns path state
until the ctrl IO queues are enabled in nvme_start_ctrl().
This way the handling of failures to retrieve the ANA log remains intact, while the mpath update is deferred until it's safe to do.
Anton Eidelman (1):
nvme/mpath: fix hang when disk goes live over reconnect
drivers/nvme/host/core.c | 1 + drivers/nvme/host/multipath.c | 12 +++++++++++- drivers/nvme/host/nvme.h | 4 ++++ 3 files changed, 16 insertions(+), 1 deletion(-)
Signature · 1 line
2.25.1
PATCH v2 1/1 nvme/mpath: fix hang when disk goes live over reconnect
nvme_mpath_init_identify() invoked from nvme_init_identify() fetches a fresh ANA log from the ctrl.
This is essential to have an up to date path states for both existing namespaces and for those scan_work may discover once the ctrl is up.
This happens in the following cases:
1) A new ctrl is being connected.
2) An existing ctrl is successfully reconnected.
3) An existing ctrl is being reset.
While in (1) ctrl->namespaces is empty, (2 & 3) may have namespaces, and nvme_read_ana_log() may call nvme_update_ns_ana_state().
This result in a hang when the ANA state of an existing namespace changes and makes the disk live: nvme_mpath_set_live() issues IO to the namespace through the ctrl,
which does NOT have IO queues yet.
See sample hang below.
Solution:
- nvme_update_ns_ana_state() to call set_live only if ctrl is live
- nvme_read_ana_log() call from nvme_mpath_init_identify() therefore only fetches and parses the ANA log;
any erros in this process will fail the ctrl setup as appropriate;
- a separate function nvme_mpath_update()
is called in nvme_start_ctrl();
this parses the ANA log without fetching it.
At this point the ctrl is live,
therefore, disks can be set live normally.
Sample failure:
nvme nvme0: starting error recovery
nvme nvme0: Reconnecting in 10 seconds...
block nvme0n6: no usable path - requeuing I/O
INFO: task kworker/u8:3:312 blocked for more than 122 seconds.
Tainted: G E 5.14.5-1.el7.elrepo.x86_64 #1
Workqueue: nvme-wq nvme_tcp_reconnect_ctrl_work [nvme_tcp]
Call Trace:
__schedule+0x2a2/0x7e0
schedule+0x4e/0xb0
io_schedule+0x16/0x40
wait_on_page_bit_common+0x15c/0x3e0
do_read_cache_page+0x1e0/0x410
read_cache_page+0x12/0x20
read_part_sector+0x46/0x100
read_lba+0x121/0x240
efi_partition+0x1d2/0x6a0
bdev_disk_changed.part.0+0x1df/0x430
bdev_disk_changed+0x18/0x20
blkdev_get_whole+0x77/0xe0
blkdev_get_by_dev+0xd2/0x3a0
__device_add_disk+0x1ed/0x310
device_add_disk+0x13/0x20
nvme_mpath_set_live+0x138/0x1b0 [nvme_core]
nvme_update_ns_ana_state+0x2b/0x30 [nvme_core]
nvme_update_ana_state+0xca/0xe0 [nvme_core]
nvme_parse_ana_log+0xac/0x170 [nvme_core]
nvme_read_ana_log+0x7d/0xe0 [nvme_core]
nvme_mpath_init_identify+0x105/0x150 [nvme_core]
nvme_init_identify+0x2df/0x4d0 [nvme_core]
nvme_init_ctrl_finish+0x8d/0x3b0 [nvme_core]
nvme_tcp_setup_ctrl+0x337/0x390 [nvme_tcp]
nvme_tcp_reconnect_ctrl_work+0x24/0x40 [nvme_tcp]
process_one_work+0x1bd/0x360
worker_thread+0x50/0x3d0--- drivers/nvme/host/core.c | 1 + drivers/nvme/host/multipath.c | 12 +++++++++++- drivers/nvme/host/nvme.h | 4 ++++ 3 files changed, 16 insertions(+), 1 deletion(-)
drivers/nvme/host/core.c +1 -0
@@ -4509,6 +4509,7 @@ void nvme_start_ctrl(struct nvme_ctrl *ctrl)if (ctrl->queue_count >1 ) {nvme_queue_scan (ctrl);nvme_start_queues (ctrl); +nvme_mpath_update (ctrl); }nvme_change_uevent (ctrl,"NVME_EVENT=connected" );
drivers/nvme/host/multipath.c +11 -1
@@ -613,7 +613,8 @@ static void nvme_update_ns_ana_state(struct nvme_ana_group_desc *desc, ns->ana_state = desc->state;clear_bit (NVME_NS_ANA_PENDING , &ns->flags); -if (nvme_state_is_live (ns->ana_state)) +if (nvme_state_is_live (ns->ana_state) && + (ns->ctrl->state ==NVME_CTRL_LIVE ))nvme_mpath_set_live (ns); }
@@ -700,6 +701,15 @@ static void nvme_ana_work(struct work_struct *work)nvme_read_ana_log (ctrl); } +void nvme_mpath_update (struct nvme_ctrl *ctrl) +{ +u32 nr_change_groups =0 ; + +mutex_lock (&ctrl->ana_lock); +nvme_parse_ana_log (ctrl, &nr_change_groups, nvme_update_ana_state); +mutex_unlock (&ctrl->ana_lock); +} +static void nvme_anatt_timeout (struct timer_list *t) {struct nvme_ctrl *ctrl =from_timer (ctrl, t, anatt_timer);
drivers/nvme/host/nvme.h +4 -0
@@ -781,6 +781,7 @@ void nvme_mpath_add_disk(struct nvme_ns *ns, struct nvme_id_ns *id);void nvme_mpath_remove_disk (struct nvme_ns_head *head);int nvme_mpath_init_identify (struct nvme_ctrl *ctrl,struct nvme_id_ctrl *id);void nvme_mpath_init_ctrl (struct nvme_ctrl *ctrl); +void nvme_mpath_update (struct nvme_ctrl *ctrl);void nvme_mpath_uninit (struct nvme_ctrl *ctrl);void nvme_mpath_stop (struct nvme_ctrl *ctrl);bool nvme_mpath_clear_current_path (struct nvme_ns *ns);
@@ -852,6 +853,9 @@ static inline int nvme_mpath_init_identify(struct nvme_ctrl *ctrl, "Please enable CONFIG_NVME_MULTIPATH for full support of multi-port devices.\n");return 0 ; } +void nvme_mpath_update (struct nvme_ctrl *ctrl) +{ +}static inline void nvme_mpath_uninit (struct nvme_ctrl *ctrl) { }
Signature · 1 line
2.25.1
Sagi Grimberg
maintainerQuoted 11 lines · Anton Eidelman
nvme_mpath_init_identify()invoked fromnvme_init_identify()fetches a fresh ANA log from the ctrl.
This is essential to have an up to date path states for both existing namespaces and for thosescan_workmay discover once the ctrl is up.This happens in the following cases:
1) A new ctrl is being connected.
2) An existing ctrl is successfully reconnected.
3) An existing ctrl is being reset.
Quoted 7 lines
While in (1)
ctrl->namespacesis empty, (2 & 3) may have namespaces, andnvme_read_ana_log()may callnvme_update_ns_ana_state().This result in a hang when the ANA state of an existing namespace changes and makes the disk live:
nvme_mpath_set_live()issues IO to the namespace through the ctrl,
which does NOT have IO queues yet.
Given that setting the path as live triggers I/O, it makes perfect sense to do it only after we have I/O queues that can accept this I/O.
Quoted 74 lines
See sample hang below.
Solution:
-nvme_update_ns_ana_state()to callset_liveonly if ctrl is live
-nvme_read_ana_log()call fromnvme_mpath_init_identify()therefore only fetches and parses the ANA log;
any erros in this process will fail the ctrl setup as appropriate;
- a separate functionnvme_mpath_update()
is called innvme_start_ctrl();
this parses the ANA log without fetching it.
At this point the ctrl is live,
therefore, disks can be set live normally.Sample failure:
nvme nvme0: starting error recovery nvme nvme0: Reconnecting in 10 seconds... block nvme0n6: no usable path - requeuing I/O INFO: task kworker/u8:3:312 blocked for more than 122 seconds. Tainted: G E 5.14.5-1.el7.elrepo.x86_64 #1 Workqueue: nvme-wq nvme_tcp_reconnect_ctrl_work [nvme_tcp] Call Trace: __schedule+0x2a2/0x7e0 schedule+0x4e/0xb0 io_schedule+0x16/0x40 wait_on_page_bit_common+0x15c/0x3e0 do_read_cache_page+0x1e0/0x410 read_cache_page+0x12/0x20 read_part_sector+0x46/0x100 read_lba+0x121/0x240 efi_partition+0x1d2/0x6a0 bdev_disk_changed.part.0+0x1df/0x430 bdev_disk_changed+0x18/0x20 blkdev_get_whole+0x77/0xe0 blkdev_get_by_dev+0xd2/0x3a0 __device_add_disk+0x1ed/0x310 device_add_disk+0x13/0x20 nvme_mpath_set_live+0x138/0x1b0 [nvme_core] nvme_update_ns_ana_state+0x2b/0x30 [nvme_core] nvme_update_ana_state+0xca/0xe0 [nvme_core] nvme_parse_ana_log+0xac/0x170 [nvme_core] nvme_read_ana_log+0x7d/0xe0 [nvme_core] nvme_mpath_init_identify+0x105/0x150 [nvme_core] nvme_init_identify+0x2df/0x4d0 [nvme_core] nvme_init_ctrl_finish+0x8d/0x3b0 [nvme_core] nvme_tcp_setup_ctrl+0x337/0x390 [nvme_tcp] nvme_tcp_reconnect_ctrl_work+0x24/0x40 [nvme_tcp] process_one_work+0x1bd/0x360 worker_thread+0x50/0x3d0Signed-off-by Anton Eidelman <[email protected]>--- drivers/nvme/host/core.c | 1 + drivers/nvme/host/multipath.c | 12 +++++++++++- drivers/nvme/host/nvme.h | 4 ++++ 3 files changed, 16 insertions(+), 1 deletion(-)drivers/nvme/host/core.c +1
-0@@ -4509,6 +4509,7 @@ void nvme_start_ctrl(struct nvme_ctrl *ctrl)if (ctrl->queue_count >1 ) {nvme_queue_scan (ctrl);nvme_start_queues (ctrl); +nvme_mpath_update (ctrl); }nvme_change_uevent (ctrl,"NVME_EVENT=connected" );drivers/nvme/host/multipath.c +0
-0
drivers/nvme/host/multipath.c static void nvme_update_ns_ana_state(struct nvme_ana_group_desc *desc, · line 615
Quoted 3 lines
drivers/nvme/host/multipath.c +0
-0@@ -613,7 +613,8 @@ static void nvme_update_ns_ana_state(struct nvme_ana_group_desc *desc, ns->ana_state = desc->state;clear_bit (NVME_NS_ANA_PENDING , &ns->flags);
I think we need a comment here explaining this. something like:
/*
* nvme_mpath_set_live() will trigger I/O to the mpath
* device node and in turn to this path device, however we
* cannot accept this I/O if the ctrl is not live, this may
* deadlock if called from the nvme_mpath_init_identify and
* the controller will never complete initialization, preventing
* I/O from completing. For this we will reprocess the ANA log
* page again once the controller will be fully initialized. */
drivers/nvme/host/multipath.c static void nvme_update_ns_ana_state(struct nvme_ana_group_desc *desc, · line 617
Quoted 3 lines
drivers/nvme/host/multipath.c +2
-1-if (nvme_state_is_live (ns->ana_state)) +if (nvme_state_is_live (ns->ana_state) && + (ns->ctrl->state ==NVME_CTRL_LIVE ))
No need for the brackets I think, and
Alignment should be:
if (nvme_state_is_live(ns->ana_state) && ns->ctrl->state == NVME_CTRL_LIVE)
Quoted 41 lines
drivers/nvme/host/multipath.c +9
-0nvme_mpath_set_live (ns); }@@ -700,6 +701,15 @@ static void nvme_ana_work(struct work_struct *work)nvme_read_ana_log (ctrl); } +void nvme_mpath_update (struct nvme_ctrl *ctrl) +{ +u32 nr_change_groups =0 ; + +mutex_lock (&ctrl->ana_lock); +nvme_parse_ana_log (ctrl, &nr_change_groups, nvme_update_ana_state); +mutex_unlock (&ctrl->ana_lock); +} +static void nvme_anatt_timeout (struct timer_list *t) {struct nvme_ctrl *ctrl =from_timer (ctrl, t, anatt_timer);drivers/nvme/host/nvme.h +4
-0@@ -781,6 +781,7 @@ void nvme_mpath_add_disk(struct nvme_ns *ns, struct nvme_id_ns *id);void nvme_mpath_remove_disk (struct nvme_ns_head *head);int nvme_mpath_init_identify (struct nvme_ctrl *ctrl,struct nvme_id_ctrl *id);void nvme_mpath_init_ctrl (struct nvme_ctrl *ctrl); +void nvme_mpath_update (struct nvme_ctrl *ctrl);void nvme_mpath_uninit (struct nvme_ctrl *ctrl);void nvme_mpath_stop (struct nvme_ctrl *ctrl);bool nvme_mpath_clear_current_path (struct nvme_ns *ns);@@ -852,6 +853,9 @@ static inline int nvme_mpath_init_identify(struct nvme_ctrl *ctrl, "Please enable CONFIG_NVME_MULTIPATH for full support of multi-port devices.\n");return 0 ; } +void nvme_mpath_update (struct nvme_ctrl *ctrl) +{ +}static inline void nvme_mpath_uninit (struct nvme_ctrl *ctrl) { }
v3 cover nvme/mpath: fix hang when disk goes live over reconnect
In reply to Anton EidelmanFixed cosmetics.
Added a comment in nvme_update_ns_ana_state() explaining the condition for nvme_mpath_set_live() call.
Anton Eidelman (1):
nvme/mpath: fix hang when disk goes live over reconnect
drivers/nvme/host/core.c | 1 + drivers/nvme/host/multipath.c | 23 +++++++++++++++++++++-- drivers/nvme/host/nvme.h | 4 ++++ 3 files changed, 26 insertions(+), 2 deletions(-)
Signature · 1 line
2.25.1
PATCH v3 1/1 nvme/mpath: fix hang when disk goes live over reconnect
nvme_mpath_init_identify() invoked from nvme_init_identify() fetches a fresh ANA log from the ctrl.
This is essential to have an up to date path states for both existing namespaces and for those scan_work may discover once the ctrl is up.
This happens in the following cases:
1) A new ctrl is being connected.
2) An existing ctrl is successfully reconnected.
3) An existing ctrl is being reset.
While in (1) ctrl->namespaces is empty, (2 & 3) may have namespaces, and nvme_read_ana_log() may call nvme_update_ns_ana_state().
This result in a hang when the ANA state of an existing namespace changes and makes the disk live: nvme_mpath_set_live() issues IO to the namespace through the ctrl,
which does NOT have IO queues yet.
See sample hang below.
Solution:
- nvme_update_ns_ana_state() to call set_live only if ctrl is live
- nvme_read_ana_log() call from nvme_mpath_init_identify() therefore only fetches and parses the ANA log;
any erros in this process will fail the ctrl setup as appropriate;
- a separate function nvme_mpath_update()
is called in nvme_start_ctrl();
this parses the ANA log without fetching it.
At this point the ctrl is live,
therefore, disks can be set live normally.
Sample failure:
nvme nvme0: starting error recovery
nvme nvme0: Reconnecting in 10 seconds...
block nvme0n6: no usable path - requeuing I/O
INFO: task kworker/u8:3:312 blocked for more than 122 seconds.
Tainted: G E 5.14.5-1.el7.elrepo.x86_64 #1
Workqueue: nvme-wq nvme_tcp_reconnect_ctrl_work [nvme_tcp]
Call Trace:
__schedule+0x2a2/0x7e0
schedule+0x4e/0xb0
io_schedule+0x16/0x40
wait_on_page_bit_common+0x15c/0x3e0
do_read_cache_page+0x1e0/0x410
read_cache_page+0x12/0x20
read_part_sector+0x46/0x100
read_lba+0x121/0x240
efi_partition+0x1d2/0x6a0
bdev_disk_changed.part.0+0x1df/0x430
bdev_disk_changed+0x18/0x20
blkdev_get_whole+0x77/0xe0
blkdev_get_by_dev+0xd2/0x3a0
__device_add_disk+0x1ed/0x310
device_add_disk+0x13/0x20
nvme_mpath_set_live+0x138/0x1b0 [nvme_core]
nvme_update_ns_ana_state+0x2b/0x30 [nvme_core]
nvme_update_ana_state+0xca/0xe0 [nvme_core]
nvme_parse_ana_log+0xac/0x170 [nvme_core]
nvme_read_ana_log+0x7d/0xe0 [nvme_core]
nvme_mpath_init_identify+0x105/0x150 [nvme_core]
nvme_init_identify+0x2df/0x4d0 [nvme_core]
nvme_init_ctrl_finish+0x8d/0x3b0 [nvme_core]
nvme_tcp_setup_ctrl+0x337/0x390 [nvme_tcp]
nvme_tcp_reconnect_ctrl_work+0x24/0x40 [nvme_tcp]
process_one_work+0x1bd/0x360
worker_thread+0x50/0x3d0--- drivers/nvme/host/core.c | 1 + drivers/nvme/host/multipath.c | 23 +++++++++++++++++++++-- drivers/nvme/host/nvme.h | 4 ++++ 3 files changed, 26 insertions(+), 2 deletions(-)
drivers/nvme/host/core.c +1 -0
@@ -4509,6 +4509,7 @@ void nvme_start_ctrl(struct nvme_ctrl *ctrl)if (ctrl->queue_count >1 ) {nvme_queue_scan (ctrl);nvme_start_queues (ctrl); +nvme_mpath_update (ctrl); }nvme_change_uevent (ctrl,"NVME_EVENT=connected" );
drivers/nvme/host/multipath.c +21 -2
@@ -612,8 +612,18 @@ static void nvme_update_ns_ana_state(struct nvme_ana_group_desc *desc, ns->ana_grpid =le32_to_cpu (desc->grpid); ns->ana_state = desc->state;clear_bit (NVME_NS_ANA_PENDING , &ns->flags); - -if (nvme_state_is_live (ns->ana_state)) +/* +* nvme_mpath_set_live() will trigger I/O to the mpath +* device node and in turn to this path device, however we +* cannot accept this I/O if the ctrl is not live. +* This may deadlock if called from the nvme_mpath_init_identify() +* and the ctrl will never complete initialization, +* preventing I/O from completing. +* For this case we will reprocess the ANA log page +* in nvme_mpath_update() once the ctrl ready. +*/ +if (nvme_state_is_live (ns->ana_state) && + ns->ctrl->state ==NVME_CTRL_LIVE )nvme_mpath_set_live (ns); }
@@ -700,6 +710,15 @@ static void nvme_ana_work(struct work_struct *work)nvme_read_ana_log (ctrl); } +void nvme_mpath_update (struct nvme_ctrl *ctrl) +{ +u32 nr_change_groups =0 ; + +mutex_lock (&ctrl->ana_lock); +nvme_parse_ana_log (ctrl, &nr_change_groups, nvme_update_ana_state); +mutex_unlock (&ctrl->ana_lock); +} +static void nvme_anatt_timeout (struct timer_list *t) {struct nvme_ctrl *ctrl =from_timer (ctrl, t, anatt_timer);
drivers/nvme/host/nvme.h +4 -0
@@ -781,6 +781,7 @@ void nvme_mpath_add_disk(struct nvme_ns *ns, struct nvme_id_ns *id);void nvme_mpath_remove_disk (struct nvme_ns_head *head);int nvme_mpath_init_identify (struct nvme_ctrl *ctrl,struct nvme_id_ctrl *id);void nvme_mpath_init_ctrl (struct nvme_ctrl *ctrl); +void nvme_mpath_update (struct nvme_ctrl *ctrl);void nvme_mpath_uninit (struct nvme_ctrl *ctrl);void nvme_mpath_stop (struct nvme_ctrl *ctrl);bool nvme_mpath_clear_current_path (struct nvme_ns *ns);
@@ -852,6 +853,9 @@ static inline int nvme_mpath_init_identify(struct nvme_ctrl *ctrl, "Please enable CONFIG_NVME_MULTIPATH for full support of multi-port devices.\n");return 0 ; } +void nvme_mpath_update (struct nvme_ctrl *ctrl) +{ +}static inline void nvme_mpath_uninit (struct nvme_ctrl *ctrl) { }
Signature · 1 line
2.25.1
This looks good to me Anton,
Thanks,
applied to nvme-5.18.
v4 cover nvme/mpath: fix hang when disk goes live over reconnect
Fixed:
1) Build error with CONFIG_NVME_MULTIPATH=n
- missing "static inline" for the nvme_mpath_update() stub.
2) Crash when CONFIG_NVME_MULTIPATH=y but
either multipath is off or the ctrl is not ANA-enabled.
Anton Eidelman (1):
nvme/mpath: fix hang when disk goes live over reconnect
drivers/nvme/host/core.c | 1 + drivers/nvme/host/multipath.c | 26 ++++++++++++++++++++++++-- drivers/nvme/host/nvme.h | 4 ++++ 3 files changed, 29 insertions(+), 2 deletions(-)
Signature · 1 line
2.25.1
PATCH v4 1/1 nvme/mpath: fix hang when disk goes live over reconnect
nvme_mpath_init_identify() invoked from nvme_init_identify() fetches a fresh ANA log from the ctrl.
This is essential to have an up to date path states for both existing namespaces and for those scan_work may discover once the ctrl is up.
This happens in the following cases:
1) A new ctrl is being connected.
2) An existing ctrl is successfully reconnected.
3) An existing ctrl is being reset.
While in (1) ctrl->namespaces is empty, (2 & 3) may have namespaces, and nvme_read_ana_log() may call nvme_update_ns_ana_state().
This result in a hang when the ANA state of an existing namespace changes and makes the disk live: nvme_mpath_set_live() issues IO to the namespace through the ctrl,
which does NOT have IO queues yet.
See sample hang below.
Solution:
- nvme_update_ns_ana_state() to call set_live only if ctrl is live
- nvme_read_ana_log() call from nvme_mpath_init_identify() therefore only fetches and parses the ANA log;
any erros in this process will fail the ctrl setup as appropriate;
- a separate function nvme_mpath_update()
is called in nvme_start_ctrl();
this parses the ANA log without fetching it.
At this point the ctrl is live,
therefore, disks can be set live normally.
Sample failure:
nvme nvme0: starting error recovery
nvme nvme0: Reconnecting in 10 seconds...
block nvme0n6: no usable path - requeuing I/O
INFO: task kworker/u8:3:312 blocked for more than 122 seconds.
Tainted: G E 5.14.5-1.el7.elrepo.x86_64 #1
Workqueue: nvme-wq nvme_tcp_reconnect_ctrl_work [nvme_tcp]
Call Trace:
__schedule+0x2a2/0x7e0
schedule+0x4e/0xb0
io_schedule+0x16/0x40
wait_on_page_bit_common+0x15c/0x3e0
do_read_cache_page+0x1e0/0x410
read_cache_page+0x12/0x20
read_part_sector+0x46/0x100
read_lba+0x121/0x240
efi_partition+0x1d2/0x6a0
bdev_disk_changed.part.0+0x1df/0x430
bdev_disk_changed+0x18/0x20
blkdev_get_whole+0x77/0xe0
blkdev_get_by_dev+0xd2/0x3a0
__device_add_disk+0x1ed/0x310
device_add_disk+0x13/0x20
nvme_mpath_set_live+0x138/0x1b0 [nvme_core]
nvme_update_ns_ana_state+0x2b/0x30 [nvme_core]
nvme_update_ana_state+0xca/0xe0 [nvme_core]
nvme_parse_ana_log+0xac/0x170 [nvme_core]
nvme_read_ana_log+0x7d/0xe0 [nvme_core]
nvme_mpath_init_identify+0x105/0x150 [nvme_core]
nvme_init_identify+0x2df/0x4d0 [nvme_core]
nvme_init_ctrl_finish+0x8d/0x3b0 [nvme_core]
nvme_tcp_setup_ctrl+0x337/0x390 [nvme_tcp]
nvme_tcp_reconnect_ctrl_work+0x24/0x40 [nvme_tcp]
process_one_work+0x1bd/0x360
worker_thread+0x50/0x3d0--- drivers/nvme/host/core.c | 1 + drivers/nvme/host/multipath.c | 26 ++++++++++++++++++++++++-- drivers/nvme/host/nvme.h | 4 ++++ 3 files changed, 29 insertions(+), 2 deletions(-)
drivers/nvme/host/core.c +1 -0
@@ -4511,6 +4511,7 @@ void nvme_start_ctrl(struct nvme_ctrl *ctrl)if (ctrl->queue_count >1 ) {nvme_queue_scan (ctrl);nvme_start_queues (ctrl); +nvme_mpath_update (ctrl); }nvme_change_uevent (ctrl,"NVME_EVENT=connected" );
drivers/nvme/host/multipath.c +24 -2
@@ -612,8 +612,18 @@ static void nvme_update_ns_ana_state(struct nvme_ana_group_desc *desc, ns->ana_grpid =le32_to_cpu (desc->grpid); ns->ana_state = desc->state;clear_bit (NVME_NS_ANA_PENDING , &ns->flags); - -if (nvme_state_is_live (ns->ana_state)) +/* +* nvme_mpath_set_live() will trigger I/O to the mpath +* device node and in turn to this path device, however we +* cannot accept this I/O if the ctrl is not live. +* This may deadlock if called from the nvme_mpath_init_identify() +* and the ctrl will never complete initialization, +* preventing I/O from completing. +* For this case we will reprocess the ANA log page +* in nvme_mpath_update() once the ctrl ready. +*/ +if (nvme_state_is_live (ns->ana_state) && + ns->ctrl->state ==NVME_CTRL_LIVE )nvme_mpath_set_live (ns); }
@@ -700,6 +710,18 @@ static void nvme_ana_work(struct work_struct *work)nvme_read_ana_log (ctrl); } +void nvme_mpath_update (struct nvme_ctrl *ctrl) +{ +u32 nr_change_groups =0 ; + +if (!ctrl->ana_log_buf) +return ; + +mutex_lock (&ctrl->ana_lock); +nvme_parse_ana_log (ctrl, &nr_change_groups, nvme_update_ana_state); +mutex_unlock (&ctrl->ana_lock); +} +static void nvme_anatt_timeout (struct timer_list *t) {struct nvme_ctrl *ctrl =from_timer (ctrl, t, anatt_timer);
drivers/nvme/host/nvme.h +4 -0
@@ -781,6 +781,7 @@ void nvme_mpath_add_disk(struct nvme_ns *ns, struct nvme_id_ns *id);void nvme_mpath_remove_disk (struct nvme_ns_head *head);int nvme_mpath_init_identify (struct nvme_ctrl *ctrl,struct nvme_id_ctrl *id);void nvme_mpath_init_ctrl (struct nvme_ctrl *ctrl); +void nvme_mpath_update (struct nvme_ctrl *ctrl);void nvme_mpath_uninit (struct nvme_ctrl *ctrl);void nvme_mpath_stop (struct nvme_ctrl *ctrl);bool nvme_mpath_clear_current_path (struct nvme_ns *ns);
@@ -852,6 +853,9 @@ static inline int nvme_mpath_init_identify(struct nvme_ctrl *ctrl, "Please enable CONFIG_NVME_MULTIPATH for full support of multi-port devices.\n");return 0 ; } +static inline void nvme_mpath_update (struct nvme_ctrl *ctrl) +{ +}static inline void nvme_mpath_uninit (struct nvme_ctrl *ctrl) { }
Signature · 1 line
2.25.1
Thanks,
applied to nvme-5.18.