Skip to contentSkip to conversation

nvme: Add verbose error logging

PATCH Superseded Alan Adamson 2 participants 3 messages Jan 26, 2022 NVMe

v5 of 6 Superseded Latest is v6 →

lore
Share & tools
lore
#0 lore raw

v5 cover nvme: Add verbose error logging

V5:
- Change nvme_ops[] and nvme_admin_ops[] to a sparse array where the opcode is used as an index into the array.
- Various other nits.

V4:
- Adding logging for admin commands.
- Display NVMe Command Opcode value rather than generic block opcode.
- Various other nits.

V3:
- Don't populate nvme_errors[] array with NULL strings.

V2:
- Change nvme_errors[] to a sparse array where the status is used as an index into the array.

- Change pr_err() to pr_err_ratelimited().

- By enabling CONFIG_NVME_VERBOSE_ERRORS, the verbose status error is displayed. If it is not enabled, then a message will still be displayed, but without the verbose status.

- Remove call to nvme_error_status() when determining whether to call nvme_error_log(). Speeds up the fast path just a bit.

This patch improves logging for NVMe errors. Currently, we only get a a vague idea as to why commands fail since only the block layer status is captured on error. This patch allows us to see why a command was failed by the controller. This is very useful when debugging problems in the field.

An example of an improved logged error:

[  183.333734] nvme0n1: Read(0x2) @ LBA 0, 1 blocks, Unrecovered Read Error (sct 0x2 / sc 0x81) DNR
[  227.767945] nvme0: Activate Firmware(0x10), Invalid Field in Command (sct 0x0 / sc 0x2) DNR

Alan Adamson (1):
nvme: Add verbose error logging

 drivers/nvme/host/Kconfig  |   8 ++
 drivers/nvme/host/Makefile |   2 +-
 drivers/nvme/host/core.c   |   3 +
 drivers/nvme/host/errors.c | 230 +++++++++++++++++++++++++++++++++++++
 drivers/nvme/host/nvme.h   |   2 +
 include/linux/nvme.h       |   1 +
 6 files changed, 245 insertions(+), 1 deletion(-)
 create mode 100644 drivers/nvme/host/errors.c
Signature · 1 line
2.27.0
#1 lore raw

PATCH v5 1/1 nvme: Add verbose error logging

Improves logging of NVMe errors. If NVME_VERBOSE_ERRORS is configured, a verbose description of the error is logged, otherwise only status codes/bits is logged.

Verbose logging examples:

[  183.333734] nvme0n1: Read(0x2) @ LBA 0, 1 blocks, Unrecovered Read Error (sct 0x2 / sc 0x81) DNR
[  227.767945] nvme0: Activate Firmware(0x10), Invalid Field in Command (sct 0x0 / sc 0x2) DNR

Non-verbose logging examples:

[   41.280509] nvme0n1: I/O Cmd(0x2) @ LBA 0, 1 blocks, I/O Error (sct 0x2 / sc 0x81) DNR
[   77.249153] nvme0: Admin Cmd(0x10), I/O Error (sct 0x0 / sc 0x2) DNR
---
 drivers/nvme/host/Kconfig  |   8 ++
 drivers/nvme/host/Makefile |   2 +-
 drivers/nvme/host/core.c   |   3 +
 drivers/nvme/host/errors.c | 229 +++++++++++++++++++++++++++++++++++++
 drivers/nvme/host/nvme.h   |   2 +
 include/linux/nvme.h       |   1 +
 6 files changed, 244 insertions(+), 1 deletion(-)
 create mode 100644 drivers/nvme/host/errors.c
drivers/nvme/host/Kconfig +8 -0
index dc0450ca23a3..d6d056963c06 100644
@@ -24,6 +24,14 @@ config NVME_MULTIPATH
 	   /dev/nvmeXnY device will show up for each NVMe namespace,
 	   even if it is accessible through multiple controllers.
 
+config NVME_VERBOSE_ERRORS
+	bool "NVMe verbose error reporting"
+	depends on NVME_CORE
+	help
+	   This option enables verbose reporting for NVMe errors. The
+	   error translation table will grow the kernel image size by
+	   about 4 KB.
+
 config NVME_HWMON
 	bool "NVMe hardware monitoring"
 	depends on (NVME_CORE=y && HWMON=y) || (NVME_CORE=m && HWMON)
drivers/nvme/host/Makefile +1 -1
index dfaacd472e5d..ea3d702feb51 100644
@@ -9,7 +9,7 @@ obj-$(CONFIG_NVME_RDMA)			+= nvme-rdma.o
 obj-$(CONFIG_NVME_FC)			+= nvme-fc.o
 obj-$(CONFIG_NVME_TCP)			+= nvme-tcp.o
 
-nvme-core-y				:= core.o ioctl.o
+nvme-core-y				:= core.o ioctl.o errors.o
 nvme-core-$(CONFIG_TRACING)		+= trace.o
 nvme-core-$(CONFIG_NVME_MULTIPATH)	+= multipath.o
 nvme-core-$(CONFIG_BLK_DEV_ZONED)	+= zns.o
drivers/nvme/host/core.c +3 -0
index 5e0bfda04bd7..ca5d36fcf4f8 100644
@@ -354,6 +354,9 @@ void nvme_complete_rq(struct request *req)
 
 	switch (nvme_decide_disposition(req)) {
 	case COMPLETE:
+		if (unlikely(nvme_req(req)->status != NVME_SC_SUCCESS))
+			nvme_error_log(req);
+
 		nvme_end_req(req);
 		return;
 	case RETRY:
drivers/nvme/host/errors.c new file +229 -0

229 lines not on this page · raw message

drivers/nvme/host/nvme.h +2 -0
index a162f6c6da6e..3216156b71b2 100644
@@ -930,4 +930,6 @@ static inline bool nvme_multi_css(struct nvme_ctrl *ctrl)
 	return (ctrl->ctrl_config & NVME_CC_CSS_MASK) == NVME_CC_CSS_CSI;
 }
 
+extern void nvme_error_log(struct request *req);
+
 #endif /* _NVME_H */
include/linux/nvme.h +1 -0
index 855dd9b3e84b..1f946e5bf7c1 100644
@@ -1636,6 +1636,7 @@ enum {
 	NVME_SC_HOST_ABORTED_CMD	= 0x371,
 
 	NVME_SC_CRD			= 0x1800,
+	NVME_SC_MORE			= 0x2000,
 	NVME_SC_DNR			= 0x4000,
 };
 
Signature · 1 line
2.27.0
#2 lore raw

drivers/nvme/host/core.c void nvme_complete_rq(struct request *req) · line 358

Quoted 8 lines · Alan Adamson
drivers/nvme/host/core.c +2 -0
@@ -354,6 +354,9 @@ void nvme_complete_rq(struct request *req)
 
 	switch (nvme_decide_disposition(req)) {
 	case COMPLETE:
+		if (unlikely(nvme_req(req)->status != NVME_SC_SUCCESS))
+			nvme_error_log(req);

nvme_log_error would seem a more natural name.

Quoted 4 lines
drivers/nvme/host/core.c +4 -0
+// SPDX-License-Identifier: GPL-2.0
+/*
+ * NVM Express device driver verbose errors
+ * Copyright (c) 2021, Oracle and/or its affiliates

drivers/nvme/host/core.c void nvme_complete_rq(struct request *req)

Quoted 10 lines
drivers/nvme/host/core.c +10 -0
+ *
+ * This program is free software; you can redistribute it and/or modify it
+ * under the terms and conditions of the GNU General Public License,
+ * version 2, as published by the Free Software Foundation.
+ *
+ * This program is distributed in the hope it will be useful, but WITHOUT
+ * ANY WARRANTY; without even the implied warranty of MERCHANTABILITY or
+ * FITNESS FOR A PARTICULAR PURPOSE.  See the GNU General Public License for
+ * more details.
+ */

No more new license boilerplates, please. That's what we have the SPDX tag for.

drivers/nvme/host/core.c

Quoted 1 line
drivers/nvme/host/core.c +1 -0
+	[nvme_cmd_dsm] = "Deallocate (DSM)",

This should be "Dataset Management" as in the spec.

Quoted 7 lines
drivers/nvme/host/core.c +7 -0
+	if (ns) {
+#ifdef CONFIG_NVME_VERBOSE_ERRORS
+		if (nr->cmd->common.opcode < ARRAY_SIZE(nvme_ops) &&
+		    nvme_ops[nr->cmd->common.opcode] != NULL)
+			op_str = nvme_ops[nr->cmd->common.opcode];
+		else
+			op_str = "Unknown";

drivers/nvme/host/core.c

Quoted 10 lines
drivers/nvme/host/core.c +9 -0
+#endif
 
+	} else {
+#ifdef CONFIG_NVME_VERBOSE_ERRORS
+		if (nr->cmd->common.opcode < ARRAY_SIZE(nvme_admin_ops) &&
+		    nvme_admin_ops[nr->cmd->common.opcode] != NULL)
+			admin_op_str = nvme_admin_ops[nr->cmd->common.opcode];
+		else
+			admin_op_str = "Unknown";
+#endif

Can't we factor the ifdefs bits of code into little helpers that are stubbed out if CONFIG_NVME_VERBOSE_ERRORS is not set? That way the main logging helper could stay in core.c, and we could have a pretty_print.c with just the constants and these helpers.

Quoted 1 line

+extern void nvme_error_log(struct request *req);

No need for the extern.