Skip to contentSkip to conversation

nvme-core: add ctrl state transition debug helper

PATCH In review Chaitanya Kulkarni 5 participants 11 messages Feb 12, 2024 – Mar 7, 2024 NVMe

lore
Share & tools
lore
#0 lore raw

PATCH 1/2 nvme-core: add ctrl state transition debug helper

NVMe controller state machine has total 7 states and 13 state transition arcs. Debugging NVMeOF problems in the field is not straight-froward, since it involves complex combination of connect/reconnect/kato/timeout handlers etc scenarios.
We already have a helper in sysfs.c that reads the controller state, but one has to constantly read the state in order understand the complete state transition which is very inconvenient.

It is often helpful to know full state transition of controller when dealing with NVMeOF issues at the time of analyzing the trace.

Add a helper that allows us to decode and print each controller state transition :-

blktests (master) # dmesg -c | grep nvme_change_ctrl_state blktests (master) # nvme_trtype=tcp ./check nvme/048
nvme/048 (Test queue count changes on reconnect) [passed]

    runtime  6.264s  ...  5.240s
blktests (master) # dmesg -c | grep nvme_change_ctrl_state
[17080.988689] nvme nvme0: nvme_change_ctrl_state new -> connecting
[17081.006623] nvme nvme0: nvme_change_ctrl_state connecting -> live
[17081.038313] nvme nvme0: nvme_change_ctrl_state live -> resetting
[17081.040730] nvme nvme0: nvme_change_ctrl_state resetting -> connecting
[17083.056750] nvme nvme0: nvme_change_ctrl_state connecting -> live
[17083.075906] nvme nvme0: nvme_change_ctrl_state live -> resetting
[17083.076112] nvme nvme0: nvme_change_ctrl_state resetting -> connecting
[17085.105270] nvme nvme0: nvme_change_ctrl_state connecting -> live
[17086.126484] nvme nvme0: nvme_change_ctrl_state live -> deleting
[17086.126506] nvme nvme0: nvme_change_ctrl_state deleting -> deleting (no IO)

blktests (master) #

---

 drivers/nvme/host/core.c | 18 ++++++++++++++++++
 1 file changed, 18 insertions(+)
drivers/nvme/host/core.c +18 -0
index 20c0c141fcc0..daeb2409f989 100644
@@ -544,6 +544,21 @@ void nvme_cancel_admin_tagset(struct nvme_ctrl *ctrl)
 }
 EXPORT_SYMBOL_GPL(nvme_cancel_admin_tagset);
 
+static const char *nvme_ctrl_state_str(enum nvme_ctrl_state st)
+{
+	static const char *const str[] = {
+		[NVME_CTRL_NEW]			= "new",
+		[NVME_CTRL_LIVE]		= "live",
+		[NVME_CTRL_RESETTING]		= "resetting",
+		[NVME_CTRL_CONNECTING]		= "connecting",
+		[NVME_CTRL_DELETING]		= "deleting",
+		[NVME_CTRL_DELETING_NOIO]	= "deleting (no IO)",
+		[NVME_CTRL_DEAD]		= "dead",
+	};
+
+	return st < ARRAY_SIZE(str) && str[st] ? str[st] : "unknown state";
+}
+
 bool nvme_change_ctrl_state(struct nvme_ctrl *ctrl,
 		enum nvme_ctrl_state new_state)
 {
@@ -621,6 +636,9 @@ bool nvme_change_ctrl_state(struct nvme_ctrl *ctrl,
 	}
 
 	if (changed) {
+		dev_dbg(ctrl->device, "%s %s -> %s\n", __func__,
+			nvme_ctrl_state_str(old_state),
+			nvme_ctrl_state_str(new_state));
 		WRITE_ONCE(ctrl->state, new_state);
 		wake_up_all(&ctrl->state_wq);
 	}
Signature · 1 line
2.40.0
#1 lore raw

PATCH 2/2 nvme: export and use previously added helper

Export nvme_ctrl_state_str() and use it in the host/sysfs.c:
nvme_sysfs_show_state().

---
 drivers/nvme/host/core.c  |  2 +-
 drivers/nvme/host/nvme.h  |  1 +
 drivers/nvme/host/sysfs.c | 16 ++--------------
 3 files changed, 4 insertions(+), 15 deletions(-)
drivers/nvme/host/core.c +1 -1
index daeb2409f989..acde9e377c00 100644
@@ -544,7 +544,7 @@ void nvme_cancel_admin_tagset(struct nvme_ctrl *ctrl)
 }
 EXPORT_SYMBOL_GPL(nvme_cancel_admin_tagset);
 
-static const char *nvme_ctrl_state_str(enum nvme_ctrl_state st)
+const char *nvme_ctrl_state_str(enum nvme_ctrl_state st)
 {
 	static const char *const str[] = {
 		[NVME_CTRL_NEW]			= "new",
drivers/nvme/host/nvme.h +1 -0
index 3ca486fe927c..3584c3861d6c 100644
@@ -1094,6 +1094,7 @@ void nvme_passthru_end(struct nvme_ctrl *ctrl, struct nvme_ns *ns, u32 effects,
 struct nvme_ctrl *nvme_ctrl_from_file(struct file *file);
 struct nvme_ns *nvme_find_get_ns(struct nvme_ctrl *ctrl, unsigned nsid);
 void nvme_put_ns(struct nvme_ns *ns);
+const char *nvme_ctrl_state_str(enum nvme_ctrl_state st);
 
 static inline bool nvme_multi_css(struct nvme_ctrl *ctrl)
 {
drivers/nvme/host/sysfs.c +2 -14
index f2832f70e7e0..a067feaba5ae 100644
@@ -368,21 +368,9 @@ static ssize_t nvme_sysfs_show_state(struct device *dev,
 				     char *buf)
 {
 	struct nvme_ctrl *ctrl = dev_get_drvdata(dev);
-	unsigned state = (unsigned)nvme_ctrl_state(ctrl);
-	static const char *const state_name[] = {
-		[NVME_CTRL_NEW]		= "new",
-		[NVME_CTRL_LIVE]	= "live",
-		[NVME_CTRL_RESETTING]	= "resetting",
-		[NVME_CTRL_CONNECTING]	= "connecting",
-		[NVME_CTRL_DELETING]	= "deleting",
-		[NVME_CTRL_DELETING_NOIO]= "deleting (no IO)",
-		[NVME_CTRL_DEAD]	= "dead",
-	};
-
-	if (state < ARRAY_SIZE(state_name) && state_name[state])
-		return sysfs_emit(buf, "%s\n", state_name[state]);
 
-	return sysfs_emit(buf, "unknown state\n");
+	return sysfs_emit(buf, "%s\n",
+			  nvme_ctrl_state_str(nvme_ctrl_state(ctrl)));
 }
 
 static DEVICE_ATTR(state, S_IRUGO, nvme_sysfs_show_state, NULL);
Signature · 1 line
2.40.0
Reviewed-by on v1 2/2
#2 lore raw

Re: [PATCH 2/2] nvme: export and use previously added helper

Quoted 67 lines · Chaitanya Kulkarni

Export nvme_ctrl_state_str() and use it in the host/sysfs.c:
nvme_sysfs_show_state().

---

drivers/nvme/host/core.c  |  2 +-
drivers/nvme/host/nvme.h  |  1 +
drivers/nvme/host/sysfs.c | 16 ++--------------
3 files changed, 4 insertions(+), 15 deletions(-)
drivers/nvme/host/core.c +1 -1
index daeb2409f989..acde9e377c00 100644
@@ -544,7 +544,7 @@ void nvme_cancel_admin_tagset(struct nvme_ctrl *ctrl)
 }
 EXPORT_SYMBOL_GPL(nvme_cancel_admin_tagset);
 
-static const char *nvme_ctrl_state_str(enum nvme_ctrl_state st)
+const char *nvme_ctrl_state_str(enum nvme_ctrl_state st)
 {
 	static const char *const str[] = {
 		[NVME_CTRL_NEW]			= "new",
drivers/nvme/host/nvme.h +1 -0
index 3ca486fe927c..3584c3861d6c 100644
@@ -1094,6 +1094,7 @@ void nvme_passthru_end(struct nvme_ctrl *ctrl, struct nvme_ns *ns, u32 effects,
 struct nvme_ctrl *nvme_ctrl_from_file(struct file *file);
 struct nvme_ns *nvme_find_get_ns(struct nvme_ctrl *ctrl, unsigned nsid);
 void nvme_put_ns(struct nvme_ns *ns);
+const char *nvme_ctrl_state_str(enum nvme_ctrl_state st);
 
 static inline bool nvme_multi_css(struct nvme_ctrl *ctrl)
 {
drivers/nvme/host/sysfs.c +2 -14
index f2832f70e7e0..a067feaba5ae 100644
@@ -368,21 +368,9 @@ static ssize_t nvme_sysfs_show_state(struct device *dev,
 				     char *buf)
 {
 	struct nvme_ctrl *ctrl = dev_get_drvdata(dev);
-	unsigned state = (unsigned)nvme_ctrl_state(ctrl);
-	static const char *const state_name[] = {
-		[NVME_CTRL_NEW]		= "new",
-		[NVME_CTRL_LIVE]	= "live",
-		[NVME_CTRL_RESETTING]	= "resetting",
-		[NVME_CTRL_CONNECTING]	= "connecting",
-		[NVME_CTRL_DELETING]	= "deleting",
-		[NVME_CTRL_DELETING_NOIO]= "deleting (no IO)",
-		[NVME_CTRL_DEAD]	= "dead",
-	};
-
-	if (state < ARRAY_SIZE(state_name) && state_name[state])
-		return sysfs_emit(buf, "%s\n", state_name[state]);
 
-	return sysfs_emit(buf, "unknown state\n");
+	return sysfs_emit(buf, "%s\n",
+			  nvme_ctrl_state_str(nvme_ctrl_state(ctrl)));
 }
 
 static DEVICE_ATTR(state, S_IRUGO, nvme_sysfs_show_state, NULL);
Signature · 1 line
2.40.0
#4 lore raw
In reply to Chaitanya Kulkarni
Quoted 77 lines · Chaitanya Kulkarni

NVMe controller state machine has total 7 states and 13 state transition arcs. Debugging NVMeOF problems in the field is not straight-froward, since it involves complex combination of connect/reconnect/kato/timeout handlers etc scenarios.
We already have a helper in sysfs.c that reads the controller state, but one has to constantly read the state in order understand the complete state transition which is very inconvenient.

It is often helpful to know full state transition of controller when dealing with NVMeOF issues at the time of analyzing the trace.

Add a helper that allows us to decode and print each controller state transition :-

blktests (master) # dmesg -c | grep nvme_change_ctrl_state blktests (master) # nvme_trtype=tcp ./check nvme/048
nvme/048 (Test queue count changes on reconnect) [passed]
runtime 6.264s ... 5.240s
blktests (master) # dmesg -c | grep nvme_change_ctrl_state

[17080.988689] nvme nvme0: nvme_change_ctrl_state new -> connecting
[17081.006623] nvme nvme0: nvme_change_ctrl_state connecting -> live
[17081.038313] nvme nvme0: nvme_change_ctrl_state live -> resetting
[17081.040730] nvme nvme0: nvme_change_ctrl_state resetting -> connecting
[17083.056750] nvme nvme0: nvme_change_ctrl_state connecting -> live
[17083.075906] nvme nvme0: nvme_change_ctrl_state live -> resetting
[17083.076112] nvme nvme0: nvme_change_ctrl_state resetting -> connecting
[17085.105270] nvme nvme0: nvme_change_ctrl_state connecting -> live
[17086.126484] nvme nvme0: nvme_change_ctrl_state live -> deleting
[17086.126506] nvme nvme0: nvme_change_ctrl_state deleting -> deleting (no IO)

blktests (master) #

---

drivers/nvme/host/core.c | 18 ++++++++++++++++++

1 file changed, 18 insertions(+)
drivers/nvme/host/core.c +18 -0
index 20c0c141fcc0..daeb2409f989 100644
@@ -544,6 +544,21 @@ void nvme_cancel_admin_tagset(struct nvme_ctrl *ctrl)
 }
 EXPORT_SYMBOL_GPL(nvme_cancel_admin_tagset);
 
+static const char *nvme_ctrl_state_str(enum nvme_ctrl_state st)
+{
+	static const char *const str[] = {
+		[NVME_CTRL_NEW]			= "new",
+		[NVME_CTRL_LIVE]		= "live",
+		[NVME_CTRL_RESETTING]		= "resetting",
+		[NVME_CTRL_CONNECTING]		= "connecting",
+		[NVME_CTRL_DELETING]		= "deleting",
+		[NVME_CTRL_DELETING_NOIO]	= "deleting (no IO)",
+		[NVME_CTRL_DEAD]		= "dead",
+	};
+
+	return st < ARRAY_SIZE(str) && str[st] ? str[st] : "unknown state";
+}
+
 bool nvme_change_ctrl_state(struct nvme_ctrl *ctrl,
 		enum nvme_ctrl_state new_state)
 {
@@ -621,6 +636,9 @@ bool nvme_change_ctrl_state(struct nvme_ctrl *ctrl,
 	}
 
 	if (changed) {
+		dev_dbg(ctrl->device, "%s %s -> %s\n", __func__,
+			nvme_ctrl_state_str(old_state),
+			nvme_ctrl_state_str(new_state));
 		WRITE_ONCE(ctrl->state, new_state);
 		wake_up_all(&ctrl->state_wq);
 	}
Signature · 1 line
2.40.0
#5 lore raw
In reply to Chaitanya Kulkarni
Quoted 29 lines · Chaitanya Kulkarni

NVMe controller state machine has total 7 states and 13 state transition arcs. Debugging NVMeOF problems in the field is not straight-froward, since it involves complex combination of connect/reconnect/kato/timeout handlers etc scenarios.
We already have a helper in sysfs.c that reads the controller state, but one has to constantly read the state in order understand the complete state transition which is very inconvenient.

It is often helpful to know full state transition of controller when dealing with NVMeOF issues at the time of analyzing the trace.

Add a helper that allows us to decode and print each controller state transition :-

blktests (master) # dmesg -c | grep nvme_change_ctrl_state blktests (master) # nvme_trtype=tcp ./check nvme/048
nvme/048 (Test queue count changes on reconnect) [passed]

     runtime  6.264s  ...  5.240s
blktests (master) # dmesg -c | grep nvme_change_ctrl_state
[17080.988689] nvme nvme0: nvme_change_ctrl_state new -> connecting
[17081.006623] nvme nvme0: nvme_change_ctrl_state connecting -> live
[17081.038313] nvme nvme0: nvme_change_ctrl_state live -> resetting
[17081.040730] nvme nvme0: nvme_change_ctrl_state resetting -> connecting
[17083.056750] nvme nvme0: nvme_change_ctrl_state connecting -> live
[17083.075906] nvme nvme0: nvme_change_ctrl_state live -> resetting
[17083.076112] nvme nvme0: nvme_change_ctrl_state resetting -> connecting
[17085.105270] nvme nvme0: nvme_change_ctrl_state connecting -> live
[17086.126484] nvme nvme0: nvme_change_ctrl_state live -> deleting
[17086.126506] nvme nvme0: nvme_change_ctrl_state deleting -> deleting (no IO)
Quoted 5 lines

blktests (master) #

---

gentle ping ...

-ck

#6 lore raw
Quoted 32 lines · Chaitanya Kulkarni
Quoted 31 lines · Chaitanya Kulkarni

NVMe controller state machine has total 7 states and 13 state transition arcs. Debugging NVMeOF problems in the field is not straight-froward, since it involves complex combination of connect/reconnect/kato/timeout handlers etc scenarios.
We already have a helper in sysfs.c that reads the controller state, but one has to constantly read the state in order understand the complete state transition which is very inconvenient.

It is often helpful to know full state transition of controller when dealing with NVMeOF issues at the time of analyzing the trace.

Add a helper that allows us to decode and print each controller state transition :-

blktests (master) # dmesg -c | grep nvme_change_ctrl_state blktests (master) # nvme_trtype=tcp ./check nvme/048
nvme/048 (Test queue count changes on reconnect) [passed]

      runtime  6.264s  ...  5.240s
blktests (master) # dmesg -c | grep nvme_change_ctrl_state
[17080.988689] nvme nvme0: nvme_change_ctrl_state new -> connecting
[17081.006623] nvme nvme0: nvme_change_ctrl_state connecting -> live
[17081.038313] nvme nvme0: nvme_change_ctrl_state live -> resetting
[17081.040730] nvme nvme0: nvme_change_ctrl_state resetting -> connecting
[17083.056750] nvme nvme0: nvme_change_ctrl_state connecting -> live
[17083.075906] nvme nvme0: nvme_change_ctrl_state live -> resetting
[17083.076112] nvme nvme0: nvme_change_ctrl_state resetting -> connecting
[17085.105270] nvme nvme0: nvme_change_ctrl_state connecting -> live
[17086.126484] nvme nvme0: nvme_change_ctrl_state live -> deleting
[17086.126506] nvme nvme0: nvme_change_ctrl_state deleting -> deleting (no IO)

blktests (master) #

Quoted 6 lines
Quoted 3 lines

---

gentle ping ...

-ck

gentle ping ...

-ck

Quoted 2 lines
maintainer
#7 lore raw
In reply to Chaitanya Kulkarni

drivers/nvme/host/core.c bool nvme_change_ctrl_state(struct nvme_ctrl *ctrl, · line 644

Quoted 10 lines · Chaitanya Kulkarni
drivers/nvme/host/core.c +3 -0
@@ -621,6 +636,9 @@ bool nvme_change_ctrl_state(struct nvme_ctrl *ctrl,
 	}
 
 	if (changed) {
+		dev_dbg(ctrl->device, "%s %s -> %s\n", __func__,
+			nvme_ctrl_state_str(old_state),
+			nvme_ctrl_state_str(new_state));
 		WRITE_ONCE(ctrl->state, new_state);
 		wake_up_all(&ctrl->state_wq);
 	}

Turing the system logs up to debug level might get you a whole lot of unrelated stuff. Could a new trace_event get you what you need instead? There's more fine grain control on those.

#8 lore raw
Quoted 4 lines · Keith Busch
Quoted 3 lines · Chaitanya Kulkarni
(quoted diff) +0 -0
@@ -621,6 +636,9 @@ bool nvme_change_ctrl_state(struct nvme_ctrl *ctrl,
  	}
 
Quoted 10 lines

drivers/nvme/host/core.c bool nvme_change_ctrl_state(struct nvme_ctrl *ctrl, · line 644

Quoted 7 lines
drivers/nvme/host/core.c +3 -0
  	if (changed) {
+		dev_dbg(ctrl->device, "%s %s -> %s\n", __func__,
+			nvme_ctrl_state_str(old_state),
+			nvme_ctrl_state_str(new_state));
  		WRITE_ONCE(ctrl->state, new_state);
  		wake_up_all(&ctrl->state_wq);
  	}

Turing the system logs up to debug level might get you a whole lot of unrelated stuff. Could a new trace_event get you what you need instead? There's more fine grain control on those.

totally agree with you, I have patches to add few more tracepoints I'll add v2 to that series and post soon, thanks for the suggestion ...

-ck

maintainer
#9 lore raw
In reply to Keith Busch
Quoted 4 lines · Keith Busch
Quoted 3 lines · Chaitanya Kulkarni
(quoted diff) +0 -0
@@ -621,6 +636,9 @@ bool nvme_change_ctrl_state(struct nvme_ctrl *ctrl,
  	}
 
Quoted 11 lines

drivers/nvme/host/core.c bool nvme_change_ctrl_state(struct nvme_ctrl *ctrl, · line 644

Quoted 7 lines
drivers/nvme/host/core.c +3 -0
  	if (changed) {
+		dev_dbg(ctrl->device, "%s %s -> %s\n", __func__,
+			nvme_ctrl_state_str(old_state),
+			nvme_ctrl_state_str(new_state));
  		WRITE_ONCE(ctrl->state, new_state);
  		wake_up_all(&ctrl->state_wq);
  	}

Turing the system logs up to debug level might get you a whole lot of unrelated stuff. Could a new trace_event get you what you need instead? There's more fine grain control on those.

I actually think its useful here as is, dynamic debug can also turn on debug per function.