nvme-core: add ctrl state transition debug helper
PATCH 1/2 nvme-core: add ctrl state transition debug helper
NVMe controller state machine has total 7 states and 13 state transition arcs. Debugging NVMeOF problems in the field is not straight-froward, since it involves complex combination of connect/reconnect/kato/timeout handlers etc scenarios.
We already have a helper in sysfs.c that reads the controller state, but one has to constantly read the state in order understand the complete state transition which is very inconvenient.
It is often helpful to know full state transition of controller when dealing with NVMeOF issues at the time of analyzing the trace.
Add a helper that allows us to decode and print each controller state transition :-
blktests (master) # dmesg -c | grep nvme_change_ctrl_state blktests (master) # nvme_trtype=tcp ./check nvme/048
nvme/048 (Test queue count changes on reconnect) [passed]
runtime 6.264s ... 5.240s blktests (master) # dmesg -c | grep nvme_change_ctrl_state [17080.988689] nvme nvme0: nvme_change_ctrl_state new -> connecting [17081.006623] nvme nvme0: nvme_change_ctrl_state connecting -> live [17081.038313] nvme nvme0: nvme_change_ctrl_state live -> resetting [17081.040730] nvme nvme0: nvme_change_ctrl_state resetting -> connecting [17083.056750] nvme nvme0: nvme_change_ctrl_state connecting -> live [17083.075906] nvme nvme0: nvme_change_ctrl_state live -> resetting [17083.076112] nvme nvme0: nvme_change_ctrl_state resetting -> connecting [17085.105270] nvme nvme0: nvme_change_ctrl_state connecting -> live [17086.126484] nvme nvme0: nvme_change_ctrl_state live -> deleting [17086.126506] nvme nvme0: nvme_change_ctrl_state deleting -> deleting (no IO)
blktests (master) #
---
drivers/nvme/host/core.c | 18 ++++++++++++++++++ 1 file changed, 18 insertions(+)
drivers/nvme/host/core.c +18 -0
@@ -544,6 +544,21 @@ void nvme_cancel_admin_tagset(struct nvme_ctrl *ctrl) }EXPORT_SYMBOL_GPL (nvme_cancel_admin_tagset); +static const char *nvme_ctrl_state_str (enum nvme_ctrl_state st) +{ +static const char *const str[] = { + [NVME_CTRL_NEW ] ="new" , + [NVME_CTRL_LIVE ] ="live" , + [NVME_CTRL_RESETTING ] ="resetting" , + [NVME_CTRL_CONNECTING ] ="connecting" , + [NVME_CTRL_DELETING ] ="deleting" , + [NVME_CTRL_DELETING_NOIO ] ="deleting (no IO)" , + [NVME_CTRL_DEAD ] ="dead" , + }; + +return st <ARRAY_SIZE (str) && str[st] ? str[st] :"unknown state" ; +} +bool nvme_change_ctrl_state (struct nvme_ctrl *ctrl,enum nvme_ctrl_state new_state) {
@@ -621,6 +636,9 @@ bool nvme_change_ctrl_state(struct nvme_ctrl *ctrl, }if (changed) { +dev_dbg (ctrl->device,"%s %s -> %s\n" , __func__, +nvme_ctrl_state_str (old_state), +nvme_ctrl_state_str (new_state));WRITE_ONCE (ctrl->state, new_state);wake_up_all (&ctrl->state_wq); }
Signature · 1 line
2.40.0
PATCH 2/2 nvme: export and use previously added helper
Export nvme_ctrl_state_str() and use it in the host/sysfs.c:nvme_sysfs_show_state().
--- drivers/nvme/host/core.c | 2 +- drivers/nvme/host/nvme.h | 1 + drivers/nvme/host/sysfs.c | 16 ++-------------- 3 files changed, 4 insertions(+), 15 deletions(-)
drivers/nvme/host/core.c +1 -1
@@ -544,7 +544,7 @@ void nvme_cancel_admin_tagset(struct nvme_ctrl *ctrl) }EXPORT_SYMBOL_GPL (nvme_cancel_admin_tagset); -static const char *nvme_ctrl_state_str (enum nvme_ctrl_state st) +const char *nvme_ctrl_state_str (enum nvme_ctrl_state st) {static const char *const str[] = { [NVME_CTRL_NEW ] ="new" ,
drivers/nvme/host/nvme.h +1 -0
@@ -1094,6 +1094,7 @@ void nvme_passthru_end(struct nvme_ctrl *ctrl, struct nvme_ns *ns, u32 effects,struct nvme_ctrl *nvme_ctrl_from_file (struct file *file);struct nvme_ns *nvme_find_get_ns (struct nvme_ctrl *ctrl,unsigned nsid);void nvme_put_ns (struct nvme_ns *ns); +const char *nvme_ctrl_state_str (enum nvme_ctrl_state st);static inline bool nvme_multi_css (struct nvme_ctrl *ctrl) {
drivers/nvme/host/sysfs.c +2 -14
@@ -368,21 +368,9 @@ static ssize_t nvme_sysfs_show_state(struct device *dev, char *buf) {struct nvme_ctrl *ctrl =dev_get_drvdata (dev); -unsigned state = (unsigned )nvme_ctrl_state (ctrl); -static const char *const state_name[] = { - [NVME_CTRL_NEW ] ="new" , - [NVME_CTRL_LIVE ] ="live" , - [NVME_CTRL_RESETTING ] ="resetting" , - [NVME_CTRL_CONNECTING ] ="connecting" , - [NVME_CTRL_DELETING ] ="deleting" , - [NVME_CTRL_DELETING_NOIO ]="deleting (no IO)" , - [NVME_CTRL_DEAD ] ="dead" , - }; - -if (state <ARRAY_SIZE (state_name) && state_name[state]) -return sysfs_emit (buf,"%s\n" , state_name[state]); -return sysfs_emit (buf,"unknown state\n" ); +return sysfs_emit (buf,"%s\n" , +nvme_ctrl_state_str (nvme_ctrl_state (ctrl))); } static DEVICE_ATTR(state, S_IRUGO, nvme_sysfs_show_state, NULL);
Signature · 1 line
2.40.0
Re: [PATCH 2/2] nvme: export and use previously added helper
Quoted 67 lines · Chaitanya Kulkarni
Export
nvme_ctrl_state_str()and use it in thehost/sysfs.c:nvme_sysfs_show_state().Signed-off-by Chaitanya Kulkarni <[email protected]>---
drivers/nvme/host/core.c | 2 +- drivers/nvme/host/nvme.h | 1 + drivers/nvme/host/sysfs.c | 16 ++-------------- 3 files changed, 4 insertions(+), 15 deletions(-)drivers/nvme/host/core.c +1
-1@@ -544,7 +544,7 @@ void nvme_cancel_admin_tagset(struct nvme_ctrl *ctrl) }EXPORT_SYMBOL_GPL (nvme_cancel_admin_tagset); -static const char *nvme_ctrl_state_str (enum nvme_ctrl_state st) +const char *nvme_ctrl_state_str (enum nvme_ctrl_state st) {static const char *const str[] = { [NVME_CTRL_NEW ] ="new" ,drivers/nvme/host/nvme.h +1
-0@@ -1094,6 +1094,7 @@ void nvme_passthru_end(struct nvme_ctrl *ctrl, struct nvme_ns *ns, u32 effects,struct nvme_ctrl *nvme_ctrl_from_file (struct file *file);struct nvme_ns *nvme_find_get_ns (struct nvme_ctrl *ctrl,unsigned nsid);void nvme_put_ns (struct nvme_ns *ns); +const char *nvme_ctrl_state_str (enum nvme_ctrl_state st);static inline bool nvme_multi_css (struct nvme_ctrl *ctrl) {drivers/nvme/host/sysfs.c +2
-14@@ -368,21 +368,9 @@ static ssize_t nvme_sysfs_show_state(struct device *dev, char *buf) {struct nvme_ctrl *ctrl =dev_get_drvdata (dev); -unsigned state = (unsigned )nvme_ctrl_state (ctrl); -static const char *const state_name[] = { - [NVME_CTRL_NEW ] ="new" , - [NVME_CTRL_LIVE ] ="live" , - [NVME_CTRL_RESETTING ] ="resetting" , - [NVME_CTRL_CONNECTING ] ="connecting" , - [NVME_CTRL_DELETING ] ="deleting" , - [NVME_CTRL_DELETING_NOIO ]="deleting (no IO)" , - [NVME_CTRL_DEAD ] ="dead" , - }; - -if (state <ARRAY_SIZE (state_name) && state_name[state]) -return sysfs_emit (buf,"%s\n" , state_name[state]); -return sysfs_emit (buf,"unknown state\n" ); +return sysfs_emit (buf,"%s\n" , +nvme_ctrl_state_str (nvme_ctrl_state (ctrl))); } static DEVICE_ATTR(state, S_IRUGO, nvme_sysfs_show_state, NULL);Signature · 1 line
2.40.0
Re: [PATCH 2/2] nvme: export and use previously added helper
In reply to Chaitanya KulkarniQuoted 77 lines · Chaitanya Kulkarni
NVMe controller state machine has total 7 states and 13 state transition arcs. Debugging NVMeOF problems in the field is not straight-froward, since it involves complex combination of connect/reconnect/kato/timeout handlers etc scenarios.
We already have a helper insysfs.cthat reads the controller state, but one has to constantly read the state in order understand the complete state transition which is very inconvenient.It is often helpful to know full state transition of controller when dealing with NVMeOF issues at the time of analyzing the trace.
Add a helper that allows us to decode and print each controller state transition :-
blktests (master) # dmesg -c | grep
nvme_change_ctrl_stateblktests (master) # nvme_trtype=tcp ./check nvme/048
nvme/048 (Test queue count changes on reconnect) [passed]
runtime 6.264s ... 5.240s
blktests (master) # dmesg -c | grepnvme_change_ctrl_state[17080.988689] nvme nvme0: nvme_change_ctrl_state new -> connecting [17081.006623] nvme nvme0: nvme_change_ctrl_state connecting -> live [17081.038313] nvme nvme0: nvme_change_ctrl_state live -> resetting [17081.040730] nvme nvme0: nvme_change_ctrl_state resetting -> connecting [17083.056750] nvme nvme0: nvme_change_ctrl_state connecting -> live [17083.075906] nvme nvme0: nvme_change_ctrl_state live -> resetting [17083.076112] nvme nvme0: nvme_change_ctrl_state resetting -> connecting [17085.105270] nvme nvme0: nvme_change_ctrl_state connecting -> live [17086.126484] nvme nvme0: nvme_change_ctrl_state live -> deleting [17086.126506] nvme nvme0: nvme_change_ctrl_state deleting -> deleting (no IO)blktests (master) #
Signed-off-by Chaitanya Kulkarni <[email protected]>---
drivers/nvme/host/core.c| 18 ++++++++++++++++++1 file changed, 18 insertions(+)drivers/nvme/host/core.c +18
-0@@ -544,6 +544,21 @@ void nvme_cancel_admin_tagset(struct nvme_ctrl *ctrl) }EXPORT_SYMBOL_GPL (nvme_cancel_admin_tagset); +static const char *nvme_ctrl_state_str (enum nvme_ctrl_state st) +{ +static const char *const str[] = { + [NVME_CTRL_NEW ] ="new" , + [NVME_CTRL_LIVE ] ="live" , + [NVME_CTRL_RESETTING ] ="resetting" , + [NVME_CTRL_CONNECTING ] ="connecting" , + [NVME_CTRL_DELETING ] ="deleting" , + [NVME_CTRL_DELETING_NOIO ] ="deleting (no IO)" , + [NVME_CTRL_DEAD ] ="dead" , + }; + +return st <ARRAY_SIZE (str) && str[st] ? str[st] :"unknown state" ; +} +bool nvme_change_ctrl_state (struct nvme_ctrl *ctrl,enum nvme_ctrl_state new_state) {@@ -621,6 +636,9 @@ bool nvme_change_ctrl_state(struct nvme_ctrl *ctrl, }if (changed) { +dev_dbg (ctrl->device,"%s %s -> %s\n" , __func__, +nvme_ctrl_state_str (old_state), +nvme_ctrl_state_str (new_state));WRITE_ONCE (ctrl->state, new_state);wake_up_all (&ctrl->state_wq); }Signature · 1 line
2.40.0
Quoted 29 lines · Chaitanya Kulkarni
NVMe controller state machine has total 7 states and 13 state transition arcs. Debugging NVMeOF problems in the field is not straight-froward, since it involves complex combination of connect/reconnect/kato/timeout handlers etc scenarios.
We already have a helper insysfs.cthat reads the controller state, but one has to constantly read the state in order understand the complete state transition which is very inconvenient.It is often helpful to know full state transition of controller when dealing with NVMeOF issues at the time of analyzing the trace.
Add a helper that allows us to decode and print each controller state transition :-
blktests (master) # dmesg -c | grep
nvme_change_ctrl_stateblktests (master) # nvme_trtype=tcp ./check nvme/048
nvme/048 (Test queue count changes on reconnect) [passed]runtime 6.264s ... 5.240s blktests (master) # dmesg -c | grep nvme_change_ctrl_state [17080.988689] nvme nvme0: nvme_change_ctrl_state new -> connecting [17081.006623] nvme nvme0: nvme_change_ctrl_state connecting -> live [17081.038313] nvme nvme0: nvme_change_ctrl_state live -> resetting [17081.040730] nvme nvme0: nvme_change_ctrl_state resetting -> connecting [17083.056750] nvme nvme0: nvme_change_ctrl_state connecting -> live [17083.075906] nvme nvme0: nvme_change_ctrl_state live -> resetting [17083.076112] nvme nvme0: nvme_change_ctrl_state resetting -> connecting [17085.105270] nvme nvme0: nvme_change_ctrl_state connecting -> live [17086.126484] nvme nvme0: nvme_change_ctrl_state live -> deleting [17086.126506] nvme nvme0: nvme_change_ctrl_state deleting -> deleting (no IO)
Quoted 5 lines
blktests (master) #
Signed-off-by Chaitanya Kulkarni <[email protected]>---
gentle ping ...
-ck
Quoted 32 lines · Chaitanya Kulkarni
Quoted 31 lines · Chaitanya Kulkarni
NVMe controller state machine has total 7 states and 13 state transition arcs. Debugging NVMeOF problems in the field is not straight-froward, since it involves complex combination of connect/reconnect/kato/timeout handlers etc scenarios.
We already have a helper insysfs.cthat reads the controller state, but one has to constantly read the state in order understand the complete state transition which is very inconvenient.It is often helpful to know full state transition of controller when dealing with NVMeOF issues at the time of analyzing the trace.
Add a helper that allows us to decode and print each controller state transition :-
blktests (master) # dmesg -c | grep
nvme_change_ctrl_stateblktests (master) # nvme_trtype=tcp ./check nvme/048
nvme/048 (Test queue count changes on reconnect) [passed]runtime 6.264s ... 5.240s blktests (master) # dmesg -c | grep nvme_change_ctrl_state [17080.988689] nvme nvme0: nvme_change_ctrl_state new -> connecting [17081.006623] nvme nvme0: nvme_change_ctrl_state connecting -> live [17081.038313] nvme nvme0: nvme_change_ctrl_state live -> resetting [17081.040730] nvme nvme0: nvme_change_ctrl_state resetting -> connecting [17083.056750] nvme nvme0: nvme_change_ctrl_state connecting -> live [17083.075906] nvme nvme0: nvme_change_ctrl_state live -> resetting [17083.076112] nvme nvme0: nvme_change_ctrl_state resetting -> connecting [17085.105270] nvme nvme0: nvme_change_ctrl_state connecting -> live [17086.126484] nvme nvme0: nvme_change_ctrl_state live -> deleting [17086.126506] nvme nvme0: nvme_change_ctrl_state deleting -> deleting (no IO)blktests (master) #
Quoted 6 lines
Quoted 3 lines
Signed-off-by Chaitanya Kulkarni <[email protected]>---
gentle ping ...
-ck
gentle ping ...
-ck
Quoted 2 lines
Keith Busch
maintainerdrivers/nvme/host/core.c bool nvme_change_ctrl_state(struct nvme_ctrl *ctrl, · line 644
Quoted 10 lines · Chaitanya Kulkarni
drivers/nvme/host/core.c +3
-0@@ -621,6 +636,9 @@ bool nvme_change_ctrl_state(struct nvme_ctrl *ctrl, }if (changed) { +dev_dbg (ctrl->device,"%s %s -> %s\n" , __func__, +nvme_ctrl_state_str (old_state), +nvme_ctrl_state_str (new_state));WRITE_ONCE (ctrl->state, new_state);wake_up_all (&ctrl->state_wq); }
Turing the system logs up to debug level might get you a whole lot of unrelated stuff. Could a new trace_event get you what you need instead? There's more fine grain control on those.
Quoted 4 lines · Keith Busch
Quoted 3 lines · Chaitanya Kulkarni
(quoted diff) +0
-0@@ -621,6 +636,9 @@ bool nvme_change_ctrl_state(struct nvme_ctrl *ctrl, }
Quoted 10 lines
drivers/nvme/host/core.c bool nvme_change_ctrl_state(struct nvme_ctrl *ctrl, · line 644
Quoted 7 lines
drivers/nvme/host/core.c +3
-0if (changed) { +dev_dbg (ctrl->device,"%s %s -> %s\n" , __func__, +nvme_ctrl_state_str (old_state), +nvme_ctrl_state_str (new_state));WRITE_ONCE (ctrl->state, new_state);wake_up_all (&ctrl->state_wq); }Turing the system logs up to debug level might get you a whole lot of unrelated stuff. Could a new
trace_eventget you what you need instead? There's more fine grain control on those.
totally agree with you, I have patches to add few more tracepoints I'll add v2 to that series and post soon, thanks for the suggestion ...
-ck
Sagi Grimberg
maintainerQuoted 4 lines · Keith Busch
Quoted 3 lines · Chaitanya Kulkarni
(quoted diff) +0
-0@@ -621,6 +636,9 @@ bool nvme_change_ctrl_state(struct nvme_ctrl *ctrl, }
Quoted 11 lines
drivers/nvme/host/core.c bool nvme_change_ctrl_state(struct nvme_ctrl *ctrl, · line 644
Quoted 7 lines
drivers/nvme/host/core.c +3
-0if (changed) { +dev_dbg (ctrl->device,"%s %s -> %s\n" , __func__, +nvme_ctrl_state_str (old_state), +nvme_ctrl_state_str (new_state));WRITE_ONCE (ctrl->state, new_state);wake_up_all (&ctrl->state_wq); }Turing the system logs up to debug level might get you a whole lot of unrelated stuff. Could a new
trace_eventget you what you need instead? There's more fine grain control on those.
I actually think its useful here as is, dynamic debug can also turn on debug per function.