Skip to contentSkip to conversation

nvme: add passthrough error logging opt-in

RFC Superseded Alan Adamson 3 participants 6 messages Mar 16, 2023 – Mar 22, 2023 NVMe

v1 of 4 Superseded Latest is v4 →

lore
Share & tools
lore
#0 lore raw

cover nvme: add passthrough error logging opt-in

Commit d7ac8dca938c ("nvme: quiet user passthrough command errors") disabled error logging for user passthrough commands. This commit adds the ability to opt in to passthrough error logging via debugfs.

Currently nvme uses debugfs for nvme fault injection. A change is needed to extend the nvme debugfs implementation to go beyond error injection.

Move Fault Injection to nvme-debugfs.c. Add new config parameter (CONFIG_NVME_FAULT_INJECTION_DEBUG_FS) which enables the compilation of Fault Injection functionality. Other consumers
of nvme-debugfs can be added without requiring Fault Injection.

Add new consumer of nvme-debugfs (CONFIG_NVME_ERROR_LOGGING_DEBUG_FS) that provides the ability of passthrough error logging.

To enable passthrough error logging:
echo Y > /sys/kernel/debug/nvme0/error-logging

To disable passthrough error logging:
echo N > /sys/kernel/debug/nvme0/error-logging

By default, passthrough error logging will remain disabled.

#1 lore raw

RFC 1/2 nvme: move fault injector to nvme-debugfs.c

Move the nvme fault injector code associated with debugfs into nvme-debugfs.c (new file) so all nvme debugfs associated can reside in a common place.

CONFIG_NVME_FAULT_INJECTION_DEBUG_FS needs to be enabled to be able to use fault injection.

---
 drivers/nvme/host/Kconfig                     |  6 ++
 drivers/nvme/host/Makefile                    |  2 +-
 drivers/nvme/host/core.c                      | 11 ++--
 .../host/{fault_inject.c => nvme-debugfs.c}   | 56 +++++++++----------
 drivers/nvme/host/nvme.h                      | 31 +++++-----
 5 files changed, 59 insertions(+), 47 deletions(-)
 rename drivers/nvme/host/{fault_inject.c => nvme-debugfs.c} (60%)
drivers/nvme/host/Kconfig +6 -0
index 2f6a7f8c94e8..70f380c1ccae 100644
@@ -32,6 +32,12 @@ config NVME_VERBOSE_ERRORS
 	   error translation table will grow the kernel image size by
 	   about 4 KB.
 
+config NVME_FAULT_INJECTION_DEBUG_FS
+	bool "NVMe Fault Injection"
+	depends on FAULT_INJECTION_DEBUG_FS
+	help
+	   This enables NVMe Fault Injection through debugfs.
+
 config NVME_HWMON
 	bool "NVMe hardware monitoring"
 	depends on (NVME_CORE=y && HWMON=y) || (NVME_CORE=m && HWMON)
drivers/nvme/host/Makefile +1 -1
index e27202d22c7d..c232d7703b65 100644
@@ -15,7 +15,7 @@ nvme-core-$(CONFIG_NVME_VERBOSE_ERRORS)	+= constants.o
 nvme-core-$(CONFIG_TRACING)		+= trace.o
 nvme-core-$(CONFIG_NVME_MULTIPATH)	+= multipath.o
 nvme-core-$(CONFIG_BLK_DEV_ZONED)	+= zns.o
-nvme-core-$(CONFIG_FAULT_INJECTION_DEBUG_FS)	+= fault_inject.o
+nvme-core-$(CONFIG_FAULT_INJECTION_DEBUG_FS)	+= nvme-debugfs.o
 nvme-core-$(CONFIG_NVME_HWMON)		+= hwmon.o
 nvme-core-$(CONFIG_NVME_AUTH)		+= auth.o
 
drivers/nvme/host/core.c +7 -4
index c2730b116dc6..cf5db19d50a5 100644
@@ -4326,7 +4326,9 @@ static void nvme_alloc_ns(struct nvme_ctrl *ctrl, struct nvme_ns_info *info)
 		nvme_add_ns_cdev(ns);
 
 	nvme_mpath_add_disk(ns, info->anagrpid);
-	nvme_fault_inject_init(&ns->fault_inject, ns->disk->disk_name);
+
+	nvme_debugfs_init(&ns->debugfs, ns->disk->disk_name);
+	nvme_fault_inject_init(&ns->debugfs);
 
 	return;
 
@@ -4357,7 +4359,7 @@ static void nvme_ns_remove(struct nvme_ns *ns)
 
 	clear_bit(NVME_NS_READY, &ns->flags);
 	set_capacity(ns->disk, 0);
-	nvme_fault_inject_fini(&ns->fault_inject);
+	nvme_debugfs_remove(&ns->debugfs);
 
 	/*
 	 * Ensure that !NVME_NS_READY is seen by other threads to prevent
@@ -5054,7 +5056,7 @@ EXPORT_SYMBOL_GPL(nvme_start_ctrl);
 void nvme_uninit_ctrl(struct nvme_ctrl *ctrl)
 {
 	nvme_hwmon_exit(ctrl);
-	nvme_fault_inject_fini(&ctrl->fault_inject);
+	nvme_debugfs_remove(&ctrl->debugfs);
 	dev_pm_qos_hide_latency_tolerance(ctrl->device);
 	cdev_device_del(&ctrl->cdev, ctrl->device);
 	nvme_put_ctrl(ctrl);
@@ -5179,7 +5181,8 @@ int nvme_init_ctrl(struct nvme_ctrl *ctrl, struct device *dev,
 	dev_pm_qos_update_user_latency_tolerance(ctrl->device,
 		min(default_ps_max_latency_us, (unsigned long)S32_MAX));
 
-	nvme_fault_inject_init(&ctrl->fault_inject, dev_name(ctrl->device));
+	nvme_debugfs_init(&ctrl->debugfs, dev_name(ctrl->device));
+	nvme_fault_inject_init(&ctrl->debugfs);
 	nvme_mpath_init_ctrl(ctrl);
 	ret = nvme_auth_init_ctrl(ctrl);
 	if (ret)
drivers/nvme/host/nvme-debugfs.c renamed from drivers/nvme/host/fault_inject.c +27 -29

98 lines not on this page · raw message

drivers/nvme/host/nvme.h +18 -13
index bf46f122e9e1..53d61723ca59 100644
@@ -231,12 +231,16 @@ enum nvme_ctrl_state {
 struct nvme_fault_inject {
 #ifdef CONFIG_FAULT_INJECTION_DEBUG_FS
 	struct fault_attr attr;
-	struct dentry *parent;
 	bool dont_retry;	/* DNR, do not retry */
 	u16 status;		/* status code */
 #endif
 };
 
+struct nvme_debugfs {
+	struct dentry *parent;
+	struct nvme_fault_inject fault_inject;
+};
+
 enum nvme_ctrl_flags {
 	NVME_CTRL_FAILFAST_EXPIRED	= 0,
 	NVME_CTRL_ADMIN_Q_STOPPED	= 1,
@@ -370,7 +374,7 @@ struct nvme_ctrl {
 	struct page *discard_page;
 	unsigned long discard_page_busy;
 
-	struct nvme_fault_inject fault_inject;
+	struct nvme_debugfs debugfs;
 
 	enum nvme_ctrl_type cntrltype;
 	enum nvme_dctype dctype;
@@ -496,8 +500,7 @@ struct nvme_ns {
 	struct cdev		cdev;
 	struct device		cdev_device;
 
-	struct nvme_fault_inject fault_inject;
-
+	struct nvme_debugfs debugfs;
 };
 
 /* NVMe ns supports metadata actions by the controller (generate/strip) */
@@ -598,21 +601,23 @@ static inline void nvme_print_device_info(struct nvme_ctrl *ctrl)
 }
 
 #ifdef CONFIG_FAULT_INJECTION_DEBUG_FS
-void nvme_fault_inject_init(struct nvme_fault_inject *fault_inj,
-			    const char *dev_name);
-void nvme_fault_inject_fini(struct nvme_fault_inject *fault_inject);
+void nvme_debugfs_init(struct nvme_debugfs *debugfs,
+		       const char *dev_name);
+void nvme_debugfs_remove(struct nvme_debugfs *debugfs);
+#else
+static inline void nvme_debugfs_init(struct nvme_debugfs *debugfs,
+			const char *dev_name) {}
+static inline void nvme_debugfs_remove(struct nvme_debugfs *debugfs) {}
+#endif
+#ifdef CONFIG_NVME_FAULT_INJECTION_DEBUG_FS
+void nvme_fault_inject_init(struct nvme_debugfs *debugfs);
 void nvme_should_fail(struct request *req);
 #else
-static inline void nvme_fault_inject_init(struct nvme_fault_inject *fault_inj,
-					  const char *dev_name)
-{
-}
-static inline void nvme_fault_inject_fini(struct nvme_fault_inject *fault_inj)
+static inline void nvme_fault_inject_init(struct nvme_debugfs *debugfs)
 {
 }
 static inline void nvme_should_fail(struct request *req) {}
 #endif
-
 bool nvme_wait_reset(struct nvme_ctrl *ctrl);
 int nvme_try_sched_reset(struct nvme_ctrl *ctrl);
 
Signature · 1 line
2.31.1
#2 lore raw

RFC 2/2 nvme: add error logging opt-in

In reply to Alan Adamson

Commit d7ac8dca938c ("nvme: quiet user passthrough command errors") disabled error logging for user passthrough commands. This commit adds the ability to opt-in to passthrough error logging.

To enable passthrough error logging:
echo Y > /sys/kernel/debug/nvme0/error-logging

To disable passthrough error logging:
echo N > /sys/kernel/debug/nvme0/error-logging

By default, passthrough error logging will remain disabled.

CONFIG_NVME_ERROR_LOGGING_DEBUG_FS needs to be enabled to to enable passthrough error logging.

---
 drivers/nvme/host/Kconfig        |  6 ++++++
 drivers/nvme/host/core.c         | 13 +++++++++++--
 drivers/nvme/host/ioctl.c        |  1 +
 drivers/nvme/host/nvme-debugfs.c |  6 ++++++
 drivers/nvme/host/nvme.h         |  8 ++++++++
 5 files changed, 32 insertions(+), 2 deletions(-)
drivers/nvme/host/Kconfig +6 -0
index 70f380c1ccae..5677292727bf 100644
@@ -38,6 +38,12 @@ config NVME_FAULT_INJECTION_DEBUG_FS
 	help
 	   This enables NVMe Fault Injection through debugfs.
 
+config NVME_ERROR_LOGGING_DEBUG_FS
+	bool "NVMe Pass Thru Error Logging"
+	depends on FAULT_INJECTION_DEBUG_FS
+	help
+	   This enables NVMe Pass Thru command error logging.
+
 config NVME_HWMON
 	bool "NVMe hardware monitoring"
 	depends on (NVME_CORE=y && HWMON=y) || (NVME_CORE=m && HWMON)
drivers/nvme/host/core.c +11 -2
index cf5db19d50a5..eb8d7caf2887 100644
@@ -666,6 +666,8 @@ static inline void nvme_clear_nvme_request(struct request *req)
 /* initialize a passthrough request */
 void nvme_init_request(struct request *req, struct nvme_command *cmd)
 {
+	struct nvme_request *nr = nvme_req(req);
+
 	if (req->q->queuedata)
 		req->timeout = NVME_IO_TIMEOUT;
 	else /* no queuedata implies admin queue */
@@ -678,8 +680,11 @@ void nvme_init_request(struct request *req, struct nvme_command *cmd)
 	if (req->mq_hctx->type == HCTX_TYPE_POLL)
 		req->cmd_flags |= REQ_POLLED;
 	nvme_clear_nvme_request(req);
-	req->rq_flags |= RQF_QUIET;
-	memcpy(nvme_req(req)->cmd, cmd, sizeof(*cmd));
+
+	if (!nr->ctrl->debugfs.error_logging)
+		req->rq_flags |= RQF_QUIET;
+
+	memcpy(nr->cmd, cmd, sizeof(*cmd));
 }
 EXPORT_SYMBOL_GPL(nvme_init_request);
 
@@ -5183,6 +5188,10 @@ int nvme_init_ctrl(struct nvme_ctrl *ctrl, struct device *dev,
 
 	nvme_debugfs_init(&ctrl->debugfs, dev_name(ctrl->device));
 	nvme_fault_inject_init(&ctrl->debugfs);
+
+	ctrl->debugfs.error_logging = false;
+	nvme_error_logging_init(&ctrl->debugfs);
+
 	nvme_mpath_init_ctrl(ctrl);
 	ret = nvme_auth_init_ctrl(ctrl);
 	if (ret)
drivers/nvme/host/ioctl.c +1 -0
index 723e7d5b778f..55d3f86524bf 100644
@@ -243,6 +243,7 @@ static int nvme_submit_user_cmd(struct request_queue *q,
 	ctrl = nvme_req(req)->ctrl;
 
 	effects = nvme_passthru_start(ctrl, ns, cmd->common.opcode);
+
 	ret = nvme_execute_rq(req, false);
 	if (result)
 		*result = le64_to_cpu(nvme_req(req)->result.u64);
drivers/nvme/host/nvme-debugfs.c +6 -0
index 87f78b864225..09953626a1dd 100644
@@ -77,4 +77,10 @@ void nvme_should_fail(struct request *req)
 }
 EXPORT_SYMBOL_GPL(nvme_should_fail);
 #endif /* CONFIG_NVME_FAULT_INJECTION_DEBUG_FS */
+#ifdef CONFIG_NVME_ERROR_LOGGING_DEBUG_FS
+void nvme_error_logging_init(struct nvme_debugfs *debugfs)
+{
+	debugfs_create_bool("error-logging", 0600, debugfs->parent, &debugfs->error_logging);
+}
+#endif /* CONFIG_NVME_ERROR_LOGGING_DEBUG_FS */
 #endif /* CONFIG_FAULT_INJECTION_DEBUG_FS */
drivers/nvme/host/nvme.h +8 -0
index 53d61723ca59..74bdf5fd9abd 100644
@@ -239,6 +239,7 @@ struct nvme_fault_inject {
 struct nvme_debugfs {
 	struct dentry *parent;
 	struct nvme_fault_inject fault_inject;
+	bool error_logging;
 };
 
 enum nvme_ctrl_flags {
@@ -618,6 +619,13 @@ static inline void nvme_fault_inject_init(struct nvme_debugfs *debugfs)
 }
 static inline void nvme_should_fail(struct request *req) {}
 #endif
+#ifdef CONFIG_NVME_ERROR_LOGGING_DEBUG_FS
+void nvme_error_logging_init(struct nvme_debugfs *debugfs);
+#else
+static inline void nvme_error_logging_init(struct nvme_debugfs *debugfs)
+{
+}
+#endif /* CONFIG_NVME_ERROR_LOGGING_DEBUG_FS */
 bool nvme_wait_reset(struct nvme_ctrl *ctrl);
 int nvme_try_sched_reset(struct nvme_ctrl *ctrl);
 
Signature · 1 line
2.31.1
#3 lore raw

Re: [RFC 2/2] nvme: add error logging opt-in

Quoted 7 lines · Alan Adamson

Commit d7ac8dca938c ("nvme: quiet user passthrough command errors") disabled error logging for user passthrough commands. This commit adds the ability to opt-in to passthrough error logging.

To enable passthrough error logging:
echo Y > /sys/kernel/debug/nvme0/error-logging

can we use 0/1 based enabling ? Y and N seems a bit odd, unless we can't for some reason ...

Quoted 22 lines

To disable passthrough error logging:
echo N > /sys/kernel/debug/nvme0/error-logging

By default, passthrough error logging will remain disabled.

CONFIG_NVME_ERROR_LOGGING_DEBUG_FS needs to be enabled to to enable passthrough error logging.

---
  drivers/nvme/host/Kconfig        |  6 ++++++
  drivers/nvme/host/core.c         | 13 +++++++++++--
  drivers/nvme/host/ioctl.c        |  1 +
  drivers/nvme/host/nvme-debugfs.c |  6 ++++++
  drivers/nvme/host/nvme.h         |  8 ++++++++
  5 files changed, 32 insertions(+), 2 deletions(-)
drivers/nvme/host/Kconfig +0 -0
index 70f380c1ccae..5677292727bf 100644

drivers/nvme/host/Kconfig config NVME_FAULT_INJECTION_DEBUG_FS · line 42

Quoted 5 lines
drivers/nvme/host/Kconfig +2 -0
@@ -38,6 +38,12 @@ config NVME_FAULT_INJECTION_DEBUG_FS
  	help
  	   This enables NVMe Fault Injection through debugfs.
 
+config NVME_ERROR_LOGGING_DEBUG_FS
+	bool "NVMe Pass Thru Error Loggigng"

/Pass Thru/Passthrough/s ?

Quoted 50 lines
drivers/nvme/host/Kconfig +4 -0
+	depends on FAULT_INJECTION_DEBUG_FS
+	help
+	   This enables NVMe Pass Thru command error logging.
+
  config NVME_HWMON
  	bool "NVMe hardware monitoring"
  	depends on (NVME_CORE=y && HWMON=y) || (NVME_CORE=m && HWMON)
drivers/nvme/host/core.c +11 -2
index cf5db19d50a5..eb8d7caf2887 100644
@@ -666,6 +666,8 @@ static inline void nvme_clear_nvme_request(struct request *req)
  /* initialize a passthrough request */
  void nvme_init_request(struct request *req, struct nvme_command *cmd)
  {
+	struct nvme_request *nr = nvme_req(req);
+
  	if (req->q->queuedata)
  		req->timeout = NVME_IO_TIMEOUT;
  	else /* no queuedata implies admin queue */
@@ -678,8 +680,11 @@ void nvme_init_request(struct request *req, struct nvme_command *cmd)
  	if (req->mq_hctx->type == HCTX_TYPE_POLL)
  		req->cmd_flags |= REQ_POLLED;
  	nvme_clear_nvme_request(req);
-	req->rq_flags |= RQF_QUIET;
-	memcpy(nvme_req(req)->cmd, cmd, sizeof(*cmd));
+
+	if (!nr->ctrl->debugfs.error_logging)
+		req->rq_flags |= RQF_QUIET;
+
+	memcpy(nr->cmd, cmd, sizeof(*cmd));
  }
  EXPORT_SYMBOL_GPL(nvme_init_request);
 
@@ -5183,6 +5188,10 @@ int nvme_init_ctrl(struct nvme_ctrl *ctrl, struct device *dev,
 
  	nvme_debugfs_init(&ctrl->debugfs, dev_name(ctrl->device));
  	nvme_fault_inject_init(&ctrl->debugfs);
+
+	ctrl->debugfs.error_logging = false;
+	nvme_error_logging_init(&ctrl->debugfs);
+
  	nvme_mpath_init_ctrl(ctrl);
  	ret = nvme_auth_init_ctrl(ctrl);
  	if (ret)
drivers/nvme/host/ioctl.c +0 -0
index 723e7d5b778f..55d3f86524bf 100644

drivers/nvme/host/ioctl.c static int nvme_submit_user_cmd(struct request_queue *q, · line 246

Quoted 4 lines
drivers/nvme/host/ioctl.c +1 -0
@@ -243,6 +243,7 @@ static int nvme_submit_user_cmd(struct request_queue *q,
  	ctrl = nvme_req(req)->ctrl;
 
  	effects = nvme_passthru_start(ctrl, ns, cmd->common.opcode);
+

no white in patch please ..

Quoted 8 lines
drivers/nvme/host/ioctl.c +0 -0
  	ret = nvme_execute_rq(req, false);
  	if (result)
  		*result = le64_to_cpu(nvme_req(req)->result.u64);
drivers/nvme/host/nvme-debugfs.c +0 -0
index 87f78b864225..09953626a1dd 100644

drivers/nvme/host/nvme-debugfs.c void nvme_should_fail(struct request *req) · line 83

Quoted 7 lines
drivers/nvme/host/nvme-debugfs.c +4 -0
@@ -77,4 +77,10 @@ void nvme_should_fail(struct request *req)
  }
  EXPORT_SYMBOL_GPL(nvme_should_fail);
  #endif /* CONFIG_NVME_FAULT_INJECTION_DEBUG_FS */
+#ifdef CONFIG_NVME_ERROR_LOGGING_DEBUG_FS
+void nvme_error_logging_init(struct nvme_debugfs *debugfs)
+{
+	debugfs_create_bool("error-logging", 0600, debugfs->parent, &debugfs->error_logging);

iff possible keep it under 80 char..

Quoted 16 lines
drivers/nvme/host/nvme-debugfs.c +2 -0
+}
+#endif /* CONFIG_NVME_ERROR_LOGGING_DEBUG_FS */
  #endif /* CONFIG_FAULT_INJECTION_DEBUG_FS */
drivers/nvme/host/nvme.h +1 -0
index 53d61723ca59..74bdf5fd9abd 100644
@@ -239,6 +239,7 @@ struct nvme_fault_inject {
  struct nvme_debugfs {
  	struct dentry *parent;
  	struct nvme_fault_inject fault_inject;
+	bool error_logging;
  };
 
  enum nvme_ctrl_flags {

drivers/nvme/host/nvme.h static inline void nvme_fault_inject_init(struct nvme_debugfs *debugfs) · line 627

Quoted 9 lines
drivers/nvme/host/nvme.h +6 -0
@@ -618,6 +619,13 @@ static inline void nvme_fault_inject_init(struct nvme_debugfs *debugfs)
  }
  static inline void nvme_should_fail(struct request *req) {}
  #endif
+#ifdef CONFIG_NVME_ERROR_LOGGING_DEBUG_FS
+void nvme_error_logging_init(struct nvme_debugfs *debugfs);
+#else
+static inline void nvme_error_logging_init(struct nvme_debugfs *debugfs)
+{
+}

since there is only one caller for above function can we use if (IS_ENABLED(....)) ?

drivers/nvme/host/nvme.h static inline void nvme_fault_inject_init(struct nvme_debugfs *debugfs) · line 631

Quoted 4 lines
drivers/nvme/host/nvme.h +1 -0
+#endif /* CONFIG_NVME_ERROR_LOGGING_DEBUG_FS */
  bool nvme_wait_reset(struct nvme_ctrl *ctrl);
  int nvme_try_sched_reset(struct nvme_ctrl *ctrl);
 

-ck

#4 lore raw

Re: [RFC 2/2] nvme: add error logging opt-in

In reply to Alan Adamson

drivers/nvme/host/core.c int nvme_init_ctrl(struct nvme_ctrl *ctrl, struct device *dev, · line 5192

Quoted 7 lines · Alan Adamson
drivers/nvme/host/core.c +2 -0
@@ -5183,6 +5188,10 @@ int nvme_init_ctrl(struct nvme_ctrl *ctrl, struct device *dev,
 
 	nvme_debugfs_init(&ctrl->debugfs, dev_name(ctrl->device));
 	nvme_fault_inject_init(&ctrl->debugfs);
+
+	ctrl->debugfs.error_logging = false;

Can we wrap the above statement into nvme_error_logging_init function? static inline void nvme_error_logging_init(struct nvme_debugfs *debugfs)

{
	debugfs->error_logging = false;
}
Quoted 12 lines
(quoted diff) +9 -0
+	nvme_error_logging_init(&ctrl->debugfs);
+
 	nvme_mpath_init_ctrl(ctrl);
 	ret = nvme_auth_init_ctrl(ctrl);
 	if (ret)
+#ifdef CONFIG_NVME_ERROR_LOGGING_DEBUG_FS
+void nvme_error_logging_init(struct nvme_debugfs *debugfs);
+#else
+static inline void nvme_error_logging_init(struct nvme_debugfs *debugfs)
+{
+}
+#endif /* CONFIG_NVME_ERROR_LOGGING_DEBUG_FS */
#5 lore raw

Re: [RFC 2/2] nvme: add error logging opt-in

Quoted 10 lines · Pankaj Raghav

drivers/nvme/host/core.c int nvme_init_ctrl(struct nvme_ctrl *ctrl, struct device *dev, · line 5192

Quoted 7 lines · Alan Adamson
drivers/nvme/host/core.c +2 -0
@@ -5183,6 +5188,10 @@ int nvme_init_ctrl(struct nvme_ctrl *ctrl, struct device *dev,
 
  	nvme_debugfs_init(&ctrl->debugfs, dev_name(ctrl->device));
  	nvme_fault_inject_init(&ctrl->debugfs);
+
+	ctrl->debugfs.error_logging = false;

Can we wrap the above statement into nvme_error_logging_init function? static inline void nvme_error_logging_init(struct nvme_debugfs *debugfs)

Quoted 3 lines
{
	debugfs->error_logging = false;
}

Since the error_logging flag is being used outside the debugfs environment, it will need to be initialized to false even if Pass Through errors is not configured.  I will move the flag to the nvme_ctrl.

Thanks,

Alan