nvme: invalidate paths during rescan
PATCH v2 1/1 nvme: invalidate paths during rescan
From: Hannes Reinecke <[email protected]>
When triggering a rescan due to a namespace resize we will be receiving AENs on every controller, triggering a rescan of all attached namespaces. If multipath is active only the current path and the ns_head disk will be updated, the other paths will still refer to the old size until AENs for the remaining controllers are received.
If I/O comes in before that it might be routed to one of the old paths, triggering an I/O failure with 'access beyond end of device'. With this patch the old paths are skipped from multipath path selection until the controller serving these paths has been rescanned.
---
v2:
- removed churn from failed rebase.
v1:
- https://lore.kernel.org/linux-nvme/[email protected]/
drivers/nvme/host/core.c | 2 ++ drivers/nvme/host/multipath.c | 16 ++++++++++++++++ drivers/nvme/host/nvme.h | 5 +++++ 3 files changed, 23 insertions(+)
drivers/nvme/host/core.c +2 -0
@@ -1877,6 +1877,7 @@ static int nvme_update_ns_info(struct nvme_ns *ns, struct nvme_id_ns *id) goto out_unfreeze; } +clear_bit (NVME_NS_INVALIDATED , &ns->flags);blk_mq_unfreeze_queue (ns->disk->queue);if (blk_queue_is_zoned (ns->queue)) {
@@ -1888,6 +1889,7 @@ static int nvme_update_ns_info(struct nvme_ns *ns, struct nvme_id_ns *id)if (nvme_ns_head_multipath (ns->head)) {blk_mq_freeze_queue (ns->head->disk->queue);nvme_update_disk_info (ns->head->disk, ns, id); +nvme_mpath_invalidate_paths (ns);blk_stack_limits (&ns->head->disk->queue->limits, &ns->queue->limits,0 );blk_queue_update_readahead (ns->head->disk->queue);
drivers/nvme/host/multipath.c +16 -0
@@ -147,6 +147,21 @@ void nvme_mpath_clear_ctrl_paths(struct nvme_ctrl *ctrl)mutex_unlock (&ctrl->scan_lock); } +void nvme_mpath_invalidate_paths (struct nvme_ns *ns) +{ +struct nvme_ns_head *head = ns->head; +sector_t capacity =get_capacity (head->disk); +int node; + +for_each_node (node) +rcu_assign_pointer (head->current_path[node],NULL ); + +list_for_each_entry_rcu (ns, &head->list, siblings) { +if (capacity !=get_capacity (ns->disk)) +set_bit (NVME_NS_INVALIDATED , &ns->flags); + } +} +static bool nvme_path_is_disabled (struct nvme_ns *ns) {/*
@@ -158,6 +173,7 @@ static bool nvme_path_is_disabled(struct nvme_ns *ns) ns->ctrl->state != NVME_CTRL_DELETING)return true;if (test_bit (NVME_NS_ANA_PENDING , &ns->flags) || +test_bit (NVME_NS_INVALIDATED , &ns->flags) ||test_bit (NVME_NS_REMOVING , &ns->flags))return true;return false;
drivers/nvme/host/nvme.h +5 -0
@@ -467,6 +467,7 @@ struct nvme_ns {#define NVME_NS_DEAD 1#define NVME_NS_ANA_PENDING 2#define NVME_NS_FORCE_RO 3 +#define NVME_NS_INVALIDATED 4struct cdev cdev;struct device cdev_device;
@@ -715,6 +716,7 @@ void nvme_mpath_init_ctrl(struct nvme_ctrl *ctrl);void nvme_mpath_uninit (struct nvme_ctrl *ctrl);void nvme_mpath_stop (struct nvme_ctrl *ctrl);bool nvme_mpath_clear_current_path (struct nvme_ns *ns); +void nvme_mpath_invalidate_paths (struct nvme_ns *ns);void nvme_mpath_clear_ctrl_paths (struct nvme_ctrl *ctrl);void nvme_mpath_shutdown_disk (struct nvme_ns_head *head);
@@ -762,6 +764,9 @@ static inline bool nvme_mpath_clear_current_path(struct nvme_ns *ns) {return false; } +static inline void nvme_mpath_invalidate_paths (struct nvme_ns *ns) +{ +}static inline void nvme_mpath_clear_ctrl_paths (struct nvme_ctrl *ctrl) { }
Signature · 7 lines
2.29.2 _______________________________________________ Linux-nvme mailing list [email protected] http://lists.infradead.org/mailman/listinfo/linux-nvme
Sagi Grimberg
maintainerQuoted 52 lines · Daniel Wagner
From: Hannes Reinecke <[email protected]>
When triggering a rescan due to a namespace resize we will be receiving AENs on every controller, triggering a rescan of all attached namespaces. If multipath is active only the current path and the
ns_headdisk will be updated, the other paths will still refer to the old size until AENs for the remaining controllers are received.If I/O comes in before that it might be routed to one of the old paths, triggering an I/O failure with 'access beyond end of device'. With this patch the old paths are skipped from multipath path selection until the controller serving these paths has been rescanned.
Signed-off-by Hannes Reinecke <[email protected]>Tested-by Daniel Wagner <[email protected]>Signed-off-by Daniel Wagner <[email protected]>---
v2:
- removed churn from failed rebase.
v1:
- https://lore.kernel.org/linux-nvme/[email protected]/drivers/nvme/host/core.c | 2 ++ drivers/nvme/host/multipath.c | 16 ++++++++++++++++ drivers/nvme/host/nvme.h | 5 +++++ 3 files changed, 23 insertions(+)drivers/nvme/host/core.c +2
-0@@ -1877,6 +1877,7 @@ static int nvme_update_ns_info(struct nvme_ns *ns, struct nvme_id_ns *id) goto out_unfreeze; } +clear_bit (NVME_NS_INVALIDATED , &ns->flags);blk_mq_unfreeze_queue (ns->disk->queue);if (blk_queue_is_zoned (ns->queue)) {@@ -1888,6 +1889,7 @@ static int nvme_update_ns_info(struct nvme_ns *ns, struct nvme_id_ns *id)if (nvme_ns_head_multipath (ns->head)) {blk_mq_freeze_queue (ns->head->disk->queue);nvme_update_disk_info (ns->head->disk, ns, id); +nvme_mpath_invalidate_paths (ns);blk_stack_limits (&ns->head->disk->queue->limits, &ns->queue->limits,0 );blk_queue_update_readahead (ns->head->disk->queue);drivers/nvme/host/multipath.c +0
-0
drivers/nvme/host/multipath.c void nvme_mpath_clear_ctrl_paths(struct nvme_ctrl *ctrl) · line 150
Quoted 4 lines
drivers/nvme/host/multipath.c +1
-0@@ -147,6 +147,21 @@ void nvme_mpath_clear_ctrl_paths(struct nvme_ctrl *ctrl)mutex_unlock (&ctrl->scan_lock); } +void nvme_mpath_invalidate_paths (struct nvme_ns *ns)
invalidate is not a great name for this very specific case here.
drivers/nvme/host/multipath.c void nvme_mpath_clear_ctrl_paths(struct nvme_ctrl *ctrl) · line 157
Quoted 7 lines
drivers/nvme/host/multipath.c +7
-0+{ +struct nvme_ns_head *head = ns->head; +sector_t capacity =get_capacity (head->disk); +int node; + +for_each_node (node) +rcu_assign_pointer (head->current_path[node],NULL );
Here the ns can get re-selected as the current path.
drivers/nvme/host/multipath.c void nvme_mpath_clear_ctrl_paths(struct nvme_ctrl *ctrl) · line 161
Quoted 4 lines
drivers/nvme/host/multipath.c +4
-0+ +list_for_each_entry_rcu (ns, &head->list, siblings) { +if (capacity !=get_capacity (ns->disk)) +set_bit (NVME_NS_INVALIDATED , &ns->flags);
maybe instead of invalidated reverse the polarity with NVME_NS_READY? clear it here and set it in update_ns_info?
_______________________________________________
Linux-nvme mailing list
[email protected]
http://lists.infradead.org/mailman/listinfo/linux-nvme
Hi Sagi,
Quoted 3 lines · Sagi Grimberg
Quoted 1 line
+void
nvme_mpath_invalidate_paths(struct nvme_ns *ns)invalidate is not a great name for this very specific case here.
Okay, I opted for nvme_mpath_revalidate_paths() as we already have nvme_revalidate_zones() which does also capacity checks too.
Quoted 9 lines
drivers/nvme/host/multipath.c void nvme_mpath_clear_ctrl_paths(struct nvme_ctrl *ctrl) · line 157
Quoted 7 lines
drivers/nvme/host/multipath.c +7
-0+{ +struct nvme_ns_head *head = ns->head; +sector_t capacity =get_capacity (head->disk); +int node; + +for_each_node (node) +rcu_assign_pointer (head->current_path[node],NULL );Here the ns can get re-selected as the current path.
This is on purpose. By clearing all current_path the next nvme_find_path() will select a valid current_path which might be the same path.
Quoted 7 lines
drivers/nvme/host/multipath.c void nvme_mpath_clear_ctrl_paths(struct nvme_ctrl *ctrl) · line 161
Quoted 4 lines
drivers/nvme/host/multipath.c +4
-0+ +list_for_each_entry_rcu (ns, &head->list, siblings) { +if (capacity !=get_capacity (ns->disk)) +set_bit (NVME_NS_INVALIDATED , &ns->flags);maybe instead of invalidated reverse the polarity with
NVME_NS_READY? clear it here and set it inupdate_ns_info?
Will do, see next version.
Thanks for the feedback!
Daniel
_______________________________________________
Linux-nvme mailing list
[email protected]
http://lists.infradead.org/mailman/listinfo/linux-nvme