Skip to contentSkip to conversation

nvme-fc: marginal path handling

PATCH Superseded Bryan Gurney 2 participants 2 messages Jun 20, 2025 – Jun 23, 2025

Subsystems (3)

v6 of 9 Superseded Latest is v9 →

lore
Share & tools
lore
#0 lore raw

PATCH v6 2/6 nvme-fc: marginal path handling

From: Hannes Reinecke <[email protected]>

FPIN LI (link integrity) messages are received when the attached fabric detects hardware errors. In response to these messages I/O should be directed away from the affected ports, and only used if the 'optimized' paths are unavailable.
To handle this a new controller flag 'NVME_CTRL_MARGINAL' is added which will cause the multipath scheduler to skip these paths when checking for 'optimized' paths. They are, however, still eligible for non-optimized path selected. The flag is cleared upon reset as then the faulty hardware might be replaced.

---
 drivers/nvme/host/core.c      |  1 +
 drivers/nvme/host/fc.c        |  4 ++++
 drivers/nvme/host/multipath.c | 17 +++++++++++------
 drivers/nvme/host/nvme.h      |  6 ++++++
 4 files changed, 22 insertions(+), 6 deletions(-)
drivers/nvme/host/core.c +1 -0
index 3da5ac71a9b0..ac03ef7baab9 100644
@@ -5040,6 +5040,7 @@ int nvme_init_ctrl(struct nvme_ctrl *ctrl, struct device *dev,
 	WRITE_ONCE(ctrl->state, NVME_CTRL_NEW);
 	ctrl->passthru_err_log_enabled = false;
 	clear_bit(NVME_CTRL_FAILFAST_EXPIRED, &ctrl->flags);
+	clear_bit(NVME_CTRL_MARGINAL, &ctrl->flags);
 	spin_lock_init(&ctrl->lock);
 	mutex_init(&ctrl->namespaces_lock);
 
drivers/nvme/host/fc.c +4 -0
index 014b387f1e8b..7e81c815bb83 100644
@@ -786,6 +786,10 @@ nvme_fc_ctrl_connectivity_loss(struct nvme_fc_ctrl *ctrl)
 		"Reconnect", ctrl->cnum);
 
 	set_bit(ASSOC_FAILED, &ctrl->flags);
+
+	/* clear 'marginal' flag as controller will be reset */
+	clear_bit(NVME_CTRL_MARGINAL, &ctrl->flags);
+
 	nvme_reset_ctrl(&ctrl->ctrl);
 }
 
drivers/nvme/host/multipath.c +11 -6
index 1062467595f3..003954985675 100644
@@ -324,11 +324,14 @@ static struct nvme_ns *__nvme_find_path(struct nvme_ns_head *head, int node)
 
 		switch (ns->ana_state) {
 		case NVME_ANA_OPTIMIZED:
-			if (distance < found_distance) {
-				found_distance = distance;
-				found = ns;
+			if (!nvme_ctrl_is_marginal(ns->ctrl)) {
+				if (distance < found_distance) {
+					found_distance = distance;
+					found = ns;
+				}
+				break;
 			}
-			break;
+			fallthrough;
 		case NVME_ANA_NONOPTIMIZED:
 			if (distance < fallback_distance) {
 				fallback_distance = distance;
@@ -381,7 +384,8 @@ static struct nvme_ns *nvme_round_robin_path(struct nvme_ns_head *head)
 
 		if (ns->ana_state == NVME_ANA_OPTIMIZED) {
 			found = ns;
-			goto out;
+			if (!nvme_ctrl_is_marginal(ns->ctrl))
+				goto out;
 		}
 		if (ns->ana_state == NVME_ANA_NONOPTIMIZED)
 			found = ns;
@@ -445,7 +449,8 @@ static struct nvme_ns *nvme_queue_depth_path(struct nvme_ns_head *head)
 static inline bool nvme_path_is_optimized(struct nvme_ns *ns)
 {
 	return nvme_ctrl_state(ns->ctrl) == NVME_CTRL_LIVE &&
-		ns->ana_state == NVME_ANA_OPTIMIZED;
+		ns->ana_state == NVME_ANA_OPTIMIZED &&
+		!nvme_ctrl_is_marginal(ns->ctrl);
 }
 
 static struct nvme_ns *nvme_numa_path(struct nvme_ns_head *head)
drivers/nvme/host/nvme.h +6 -0
index 7df2ea21851f..71a5c5f87db6 100644
@@ -275,6 +275,7 @@ enum nvme_ctrl_flags {
 	NVME_CTRL_SKIP_ID_CNS_CS	= 4,
 	NVME_CTRL_DIRTY_CAPABILITY	= 5,
 	NVME_CTRL_FROZEN		= 6,
+	NVME_CTRL_MARGINAL		= 7,
 };
 
 struct nvme_ctrl {
@@ -417,6 +418,11 @@ static inline enum nvme_ctrl_state nvme_ctrl_state(struct nvme_ctrl *ctrl)
 	return READ_ONCE(ctrl->state);
 }
 
+static inline bool nvme_ctrl_is_marginal(struct nvme_ctrl *ctrl)
+{
+	return test_bit(NVME_CTRL_MARGINAL, &ctrl->flags);
+}
+
 enum nvme_iopolicy {
 	NVME_IOPOLICY_NUMA,
 	NVME_IOPOLICY_RR,
Signature · 1 line
2.49.0
Reviewed-by on v6 2/6
#1 lore raw
Quoted 115 lines · Bryan Gurney

From: Hannes Reinecke <[email protected]>

FPIN LI (link integrity) messages are received when the attached fabric detects hardware errors. In response to these messages I/O should be directed away from the affected ports, and only used if the 'optimized' paths are unavailable.
To handle this a new controller flag 'NVME_CTRL_MARGINAL' is added which will cause the multipath scheduler to skip these paths when checking for 'optimized' paths. They are, however, still eligible for non-optimized path selected. The flag is cleared upon reset as then the faulty hardware might be replaced.

---
  drivers/nvme/host/core.c      |  1 +
  drivers/nvme/host/fc.c        |  4 ++++
  drivers/nvme/host/multipath.c | 17 +++++++++++------
  drivers/nvme/host/nvme.h      |  6 ++++++
  4 files changed, 22 insertions(+), 6 deletions(-)
drivers/nvme/host/core.c +1 -0
index 3da5ac71a9b0..ac03ef7baab9 100644
@@ -5040,6 +5040,7 @@ int nvme_init_ctrl(struct nvme_ctrl *ctrl, struct device *dev,
  	WRITE_ONCE(ctrl->state, NVME_CTRL_NEW);
  	ctrl->passthru_err_log_enabled = false;
  	clear_bit(NVME_CTRL_FAILFAST_EXPIRED, &ctrl->flags);
+	clear_bit(NVME_CTRL_MARGINAL, &ctrl->flags);
  	spin_lock_init(&ctrl->lock);
  	mutex_init(&ctrl->namespaces_lock);
 
drivers/nvme/host/fc.c +4 -0
index 014b387f1e8b..7e81c815bb83 100644
@@ -786,6 +786,10 @@ nvme_fc_ctrl_connectivity_loss(struct nvme_fc_ctrl *ctrl)
  		"Reconnect", ctrl->cnum);
 
  	set_bit(ASSOC_FAILED, &ctrl->flags);
+
+	/* clear 'marginal' flag as controller will be reset */
+	clear_bit(NVME_CTRL_MARGINAL, &ctrl->flags);
+
  	nvme_reset_ctrl(&ctrl->ctrl);
  }
 
drivers/nvme/host/multipath.c +11 -6
index 1062467595f3..003954985675 100644
@@ -324,11 +324,14 @@ static struct nvme_ns *__nvme_find_path(struct nvme_ns_head *head, int node)
 
  		switch (ns->ana_state) {
  		case NVME_ANA_OPTIMIZED:
-			if (distance < found_distance) {
-				found_distance = distance;
-				found = ns;
+			if (!nvme_ctrl_is_marginal(ns->ctrl)) {
+				if (distance < found_distance) {
+					found_distance = distance;
+					found = ns;
+				}
+				break;
  			}
-			break;
+			fallthrough;
  		case NVME_ANA_NONOPTIMIZED:
  			if (distance < fallback_distance) {
  				fallback_distance = distance;
@@ -381,7 +384,8 @@ static struct nvme_ns *nvme_round_robin_path(struct nvme_ns_head *head)
 
  		if (ns->ana_state == NVME_ANA_OPTIMIZED) {
  			found = ns;
-			goto out;
+			if (!nvme_ctrl_is_marginal(ns->ctrl))
+				goto out;
  		}
  		if (ns->ana_state == NVME_ANA_NONOPTIMIZED)
  			found = ns;
@@ -445,7 +449,8 @@ static struct nvme_ns *nvme_queue_depth_path(struct nvme_ns_head *head)
  static inline bool nvme_path_is_optimized(struct nvme_ns *ns)
  {
  	return nvme_ctrl_state(ns->ctrl) == NVME_CTRL_LIVE &&
-		ns->ana_state == NVME_ANA_OPTIMIZED;
+		ns->ana_state == NVME_ANA_OPTIMIZED &&
+		!nvme_ctrl_is_marginal(ns->ctrl);
  }
 
  static struct nvme_ns *nvme_numa_path(struct nvme_ns_head *head)
drivers/nvme/host/nvme.h +6 -0
index 7df2ea21851f..71a5c5f87db6 100644
@@ -275,6 +275,7 @@ enum nvme_ctrl_flags {
  	NVME_CTRL_SKIP_ID_CNS_CS	= 4,
  	NVME_CTRL_DIRTY_CAPABILITY	= 5,
  	NVME_CTRL_FROZEN		= 6,
+	NVME_CTRL_MARGINAL		= 7,
  };
 
  struct nvme_ctrl {
@@ -417,6 +418,11 @@ static inline enum nvme_ctrl_state nvme_ctrl_state(struct nvme_ctrl *ctrl)
  	return READ_ONCE(ctrl->state);
  }
 
+static inline bool nvme_ctrl_is_marginal(struct nvme_ctrl *ctrl)
+{
+	return test_bit(NVME_CTRL_MARGINAL, &ctrl->flags);
+}
+
  enum nvme_iopolicy {
  	NVME_IOPOLICY_NUMA,
  	NVME_IOPOLICY_RR,