nvme-fc: marginal path handling
Subsystems (3)
PATCH v6 2/6 nvme-fc: marginal path handling
From: Hannes Reinecke <[email protected]>
FPIN LI (link integrity) messages are received when the attached fabric detects hardware errors. In response to these messages I/O should be directed away from the affected ports, and only used if the 'optimized' paths are unavailable.
To handle this a new controller flag 'NVME_CTRL_MARGINAL' is added which will cause the multipath scheduler to skip these paths when checking for 'optimized' paths. They are, however, still eligible for non-optimized path selected. The flag is cleared upon reset as then the faulty hardware might be replaced.
--- drivers/nvme/host/core.c | 1 + drivers/nvme/host/fc.c | 4 ++++ drivers/nvme/host/multipath.c | 17 +++++++++++------ drivers/nvme/host/nvme.h | 6 ++++++ 4 files changed, 22 insertions(+), 6 deletions(-)
drivers/nvme/host/core.c +1 -0
@@ -5040,6 +5040,7 @@ int nvme_init_ctrl(struct nvme_ctrl *ctrl, struct device *dev,WRITE_ONCE (ctrl->state,NVME_CTRL_NEW ); ctrl->passthru_err_log_enabled = false;clear_bit (NVME_CTRL_FAILFAST_EXPIRED , &ctrl->flags); +clear_bit (NVME_CTRL_MARGINAL , &ctrl->flags);spin_lock_init (&ctrl->lock);mutex_init (&ctrl->namespaces_lock);
drivers/nvme/host/fc.c +4 -0
@@ -786,6 +786,10 @@ nvme_fc_ctrl_connectivity_loss(struct nvme_fc_ctrl *ctrl) "Reconnect", ctrl->cnum);set_bit (ASSOC_FAILED , &ctrl->flags); + +/* clear 'marginal' flag as controller will be reset */ +clear_bit (NVME_CTRL_MARGINAL , &ctrl->flags); +nvme_reset_ctrl (&ctrl->ctrl); }
drivers/nvme/host/multipath.c +11 -6
@@ -324,11 +324,14 @@ static struct nvme_ns *__nvme_find_path(struct nvme_ns_head *head, int node)switch (ns->ana_state) {case NVME_ANA_OPTIMIZED : -if (distance < found_distance) { - found_distance = distance; - found = ns; +if (!nvme_ctrl_is_marginal (ns->ctrl)) { +if (distance < found_distance) { + found_distance = distance; + found = ns; + } +break ; } -break ; + fallthrough;case NVME_ANA_NONOPTIMIZED :if (distance < fallback_distance) { fallback_distance = distance;
@@ -381,7 +384,8 @@ static struct nvme_ns *nvme_round_robin_path(struct nvme_ns_head *head)if (ns->ana_state ==NVME_ANA_OPTIMIZED ) { found = ns; - goto out; +if (!nvme_ctrl_is_marginal (ns->ctrl)) + goto out; }if (ns->ana_state ==NVME_ANA_NONOPTIMIZED ) found = ns;
@@ -445,7 +449,8 @@ static struct nvme_ns *nvme_queue_depth_path(struct nvme_ns_head *head)static inline bool nvme_path_is_optimized (struct nvme_ns *ns) {return nvme_ctrl_state (ns->ctrl) ==NVME_CTRL_LIVE && - ns->ana_state ==NVME_ANA_OPTIMIZED ; + ns->ana_state ==NVME_ANA_OPTIMIZED && + !nvme_ctrl_is_marginal (ns->ctrl); }static struct nvme_ns *nvme_numa_path (struct nvme_ns_head *head)
drivers/nvme/host/nvme.h +6 -0
@@ -275,6 +275,7 @@ enum nvme_ctrl_flags {NVME_CTRL_SKIP_ID_CNS_CS =4 ,NVME_CTRL_DIRTY_CAPABILITY =5 ,NVME_CTRL_FROZEN =6 , + NVME_CTRL_MARGINAL = 7, }; struct nvme_ctrl {
@@ -417,6 +418,11 @@ static inline enum nvme_ctrl_state nvme_ctrl_state(struct nvme_ctrl *ctrl)return READ_ONCE (ctrl->state); } +static inline bool nvme_ctrl_is_marginal (struct nvme_ctrl *ctrl) +{ +return test_bit (NVME_CTRL_MARGINAL , &ctrl->flags); +} +enum nvme_iopolicy {NVME_IOPOLICY_NUMA ,NVME_IOPOLICY_RR ,
Signature · 1 line
2.49.0
Quoted 115 lines · Bryan Gurney
From: Hannes Reinecke <[email protected]>
FPIN LI (link integrity) messages are received when the attached fabric detects hardware errors. In response to these messages I/O should be directed away from the affected ports, and only used if the 'optimized' paths are unavailable.
To handle this a new controller flag 'NVME_CTRL_MARGINAL' is added which will cause the multipath scheduler to skip these paths when checking for 'optimized' paths. They are, however, still eligible for non-optimized path selected. The flag is cleared upon reset as then the faulty hardware might be replaced.Signed-off-by Hannes Reinecke <[email protected]>Tested-by Bryan Gurney <[email protected]>--- drivers/nvme/host/core.c | 1 + drivers/nvme/host/fc.c | 4 ++++ drivers/nvme/host/multipath.c | 17 +++++++++++------ drivers/nvme/host/nvme.h | 6 ++++++ 4 files changed, 22 insertions(+), 6 deletions(-)drivers/nvme/host/core.c +1
-0@@ -5040,6 +5040,7 @@ int nvme_init_ctrl(struct nvme_ctrl *ctrl, struct device *dev,WRITE_ONCE (ctrl->state,NVME_CTRL_NEW ); ctrl->passthru_err_log_enabled = false;clear_bit (NVME_CTRL_FAILFAST_EXPIRED , &ctrl->flags); +clear_bit (NVME_CTRL_MARGINAL , &ctrl->flags);spin_lock_init (&ctrl->lock);mutex_init (&ctrl->namespaces_lock);drivers/nvme/host/fc.c +4
-0@@ -786,6 +786,10 @@ nvme_fc_ctrl_connectivity_loss(struct nvme_fc_ctrl *ctrl) "Reconnect", ctrl->cnum);set_bit (ASSOC_FAILED , &ctrl->flags); + +/* clear 'marginal' flag as controller will be reset */ +clear_bit (NVME_CTRL_MARGINAL , &ctrl->flags); +nvme_reset_ctrl (&ctrl->ctrl); }drivers/nvme/host/multipath.c +11
-6@@ -324,11 +324,14 @@ static struct nvme_ns *__nvme_find_path(struct nvme_ns_head *head, int node)switch (ns->ana_state) {case NVME_ANA_OPTIMIZED : -if (distance < found_distance) { - found_distance = distance; - found = ns; +if (!nvme_ctrl_is_marginal (ns->ctrl)) { +if (distance < found_distance) { + found_distance = distance; + found = ns; + } +break ; } -break ; + fallthrough;case NVME_ANA_NONOPTIMIZED :if (distance < fallback_distance) { fallback_distance = distance;@@ -381,7 +384,8 @@ static struct nvme_ns *nvme_round_robin_path(struct nvme_ns_head *head)if (ns->ana_state ==NVME_ANA_OPTIMIZED ) { found = ns; - goto out; +if (!nvme_ctrl_is_marginal (ns->ctrl)) + goto out; }if (ns->ana_state ==NVME_ANA_NONOPTIMIZED ) found = ns;@@ -445,7 +449,8 @@ static struct nvme_ns *nvme_queue_depth_path(struct nvme_ns_head *head)static inline bool nvme_path_is_optimized (struct nvme_ns *ns) {return nvme_ctrl_state (ns->ctrl) ==NVME_CTRL_LIVE && - ns->ana_state ==NVME_ANA_OPTIMIZED ; + ns->ana_state ==NVME_ANA_OPTIMIZED && + !nvme_ctrl_is_marginal (ns->ctrl); }static struct nvme_ns *nvme_numa_path (struct nvme_ns_head *head)drivers/nvme/host/nvme.h +6
-0@@ -275,6 +275,7 @@ enum nvme_ctrl_flags {NVME_CTRL_SKIP_ID_CNS_CS =4 ,NVME_CTRL_DIRTY_CAPABILITY =5 ,NVME_CTRL_FROZEN =6 , + NVME_CTRL_MARGINAL = 7, }; struct nvme_ctrl {@@ -417,6 +418,11 @@ static inline enum nvme_ctrl_state nvme_ctrl_state(struct nvme_ctrl *ctrl)return READ_ONCE (ctrl->state); } +static inline bool nvme_ctrl_is_marginal (struct nvme_ctrl *ctrl) +{ +return test_bit (NVME_CTRL_MARGINAL , &ctrl->flags); +} +enum nvme_iopolicy {NVME_IOPOLICY_NUMA ,NVME_IOPOLICY_RR ,