Skip to contentSkip to conversation

nvme: enable to inject errors into admin commands

PATCH New Akinobu Mita 2 participants 6 messages Jun 9, 2019 – Jun 20, 2019

Subsystems (3)
lore
Share & tools
lore

Akinobu Mita

author maintainer
#0 lore raw

v3 cover nvme: enable to inject errors into admin commands

Currenlty fault injection support for nvme only enables to inject errors into the commands submitted to I/O queues.

This enables to inject errors into the commands submitted to the admin queue.

It is useful to test error handling in the controller initialization.

	# echo 100 > /sys/kernel/debug/nvme0/fault_inject/probability
	# echo 1 > /sys/kernel/debug/nvme0/fault_inject/times
	# echo 10 > /sys/kernel/debug/nvme0/fault_inject/space
	# nvme reset /dev/nvme0
	# dmesg
	...
	nvme nvme0: Could not set queue count (16385)
	nvme nvme0: IO queues not created

* v3
- add Reviewed-by tags
- WARN_ONCE() if no namespace is tied to the request
- remove thermal zone series dependency

* v2
- rename the argument 'name' to 'dev_name'
- add Reviewed-by tags
- add documentation for nvme fault injection

Akinobu Mita (3):
nvme: prepare for fault injection into admin commands
nvme: enable to inject errors into admin commands
Documentation: nvme: add an example for nvme fault injection

 .../fault-injection/nvme-fault-injection.txt       | 56 ++++++++++++++++++++++
 drivers/nvme/host/core.c                           |  7 ++-
 drivers/nvme/host/fault_inject.c                   | 41 ++++++++--------
 drivers/nvme/host/nvme.h                           | 36 ++++++++------
 4 files changed, 104 insertions(+), 36 deletions(-)
Cc Thomas Tai <thomas.tai at oracle.com>
Cc Keith Busch <kbusch at kernel.org>
Cc Jens Axboe <axboe at fb.com>
Cc Christoph Hellwig <hch at lst.de>
Cc Sagi Grimberg <sagi at grimberg.me>
Cc Minwoo Im <minwoo.im.dev at gmail.com>
Cc Chaitanya Kulkarni <chaitanya.kulkarni at wdc.com>
Signature · 1 line
2.7.4
author maintainer
#1 lore raw

PATCH v3 1/3 nvme: prepare for fault injection into admin commands

Currenlty fault injection support for nvme only enables to inject errors into the commands submitted to I/O queues.

In preparation for fault injection into the admin commands, this makes the helper functions independent of struct nvme_ns.

Cc Thomas Tai <thomas.tai at oracle.com>
Cc Keith Busch <kbusch at kernel.org>
Cc Jens Axboe <axboe at fb.com>
Cc Christoph Hellwig <hch at lst.de>
Cc Sagi Grimberg <sagi at grimberg.me>
Cc Minwoo Im <minwoo.im.dev at gmail.com>
Cc Chaitanya Kulkarni <chaitanya.kulkarni at wdc.com>
Signed-off-by Akinobu Mita <akinobu.mita at gmail.com>
Reviewed-by Minwoo Im <minwoo.im.dev at gmail.com>
Reviewed-by Sagi Grimberg <sagi at grimberg.me>
Reviewed-by Chaitanya Kulkarni <chaitanya.kulkarni at wdc.com>
Reviewed-by Christoph Hellwig <hch at lst.de>

---
* v3
- add Reviewed-by tags
- WARN_ONCE() if no namespace is tied to the request

 drivers/nvme/host/core.c         |  4 ++--
 drivers/nvme/host/fault_inject.c | 36 ++++++++++++++++++++----------------
 drivers/nvme/host/nvme.h         | 34 +++++++++++++++++++---------------
 3 files changed, 41 insertions(+), 33 deletions(-)
drivers/nvme/host/core.c +2 -2
index 1b7c2af..a66e7d8 100644
@@ -3318,7 +3318,7 @@ static int nvme_alloc_ns(struct nvme_ctrl *ctrl, unsigned nsid)
 	device_add_disk(ctrl->device, ns->disk, nvme_ns_id_attr_groups);
 
 	nvme_mpath_add_disk(ns, id);
-	nvme_fault_inject_init(ns);
+	nvme_fault_inject_init(&ns->fault_inject, ns->disk->disk_name);
 	kfree(id);
 
 	return 0;
@@ -3343,7 +3343,7 @@ static void nvme_ns_remove(struct nvme_ns *ns)
 	if (test_and_set_bit(NVME_NS_REMOVING, &ns->flags))
 		return;
 
-	nvme_fault_inject_fini(ns);
+	nvme_fault_inject_fini(&ns->fault_inject);
 	if (ns->disk && ns->disk->flags & GENHD_FL_UP) {
 		del_gendisk(ns->disk);
 		blk_cleanup_queue(ns->queue);
drivers/nvme/host/fault_inject.c +20 -16
index 4cfd2c9..e37b8c2 100644
@@ -15,11 +15,10 @@ static DECLARE_FAULT_ATTR(fail_default_attr);
 static char *fail_request;
 module_param(fail_request, charp, 0000);
 
-void nvme_fault_inject_init(struct nvme_ns *ns)
+void nvme_fault_inject_init(struct nvme_fault_inject *fault_inj,
+			    const char *dev_name)
 {
 	struct dentry *dir, *parent;
-	char *name = ns->disk->disk_name;
-	struct nvme_fault_inject *fault_inj = &ns->fault_inject;
 	struct fault_attr *attr = &fault_inj->attr;
 
 	/* set default fault injection attribute */
@@ -27,20 +26,20 @@ void nvme_fault_inject_init(struct nvme_ns *ns)
 		setup_fault_attr(&fail_default_attr, fail_request);
 
 	/* create debugfs directory and attribute */
-	parent = debugfs_create_dir(name, NULL);
+	parent = debugfs_create_dir(dev_name, NULL);
 	if (!parent) {
-		pr_warn("%s: failed to create debugfs directory\n", name);
+		pr_warn("%s: failed to create debugfs directory\n", dev_name);
 		return;
 	}
 
 	*attr = fail_default_attr;
 	dir = fault_create_debugfs_attr("fault_inject", parent, attr);
 	if (IS_ERR(dir)) {
-		pr_warn("%s: failed to create debugfs attr\n", name);
+		pr_warn("%s: failed to create debugfs attr\n", dev_name);
 		debugfs_remove_recursive(parent);
 		return;
 	}
-	ns->fault_inject.parent = parent;
+	fault_inj->parent = parent;
 
 	/* create debugfs for status code and dont_retry */
 	fault_inj->status = NVME_SC_INVALID_OPCODE;
@@ -49,29 +48,34 @@ void nvme_fault_inject_init(struct nvme_ns *ns)
 	debugfs_create_bool("dont_retry", 0600, dir, &fault_inj->dont_retry);
 }
 
-void nvme_fault_inject_fini(struct nvme_ns *ns)
+void nvme_fault_inject_fini(struct nvme_fault_inject *fault_inject)
 {
 	/* remove debugfs directories */
-	debugfs_remove_recursive(ns->fault_inject.parent);
+	debugfs_remove_recursive(fault_inject->parent);
 }
 
 void nvme_should_fail(struct request *req)
 {
 	struct gendisk *disk = req->rq_disk;
-	struct nvme_ns *ns = NULL;
+	struct nvme_fault_inject *fault_inject = NULL;
 	u16 status;
 
 	/*
 	 * make sure this request is coming from a valid namespace
 	 */
-	if (!disk)
-		return;
+	if (disk) {
+		struct nvme_ns *ns = disk->private_data;
+
+		if (ns)
+			fault_inject = &ns->fault_inject;
+		else
+			WARN_ONCE(1, "No namespace found for request\n");
+	}
 
-	ns = disk->private_data;
-	if (ns && should_fail(&ns->fault_inject.attr, 1)) {
+	if (fault_inject && should_fail(&fault_inject->attr, 1)) {
 		/* inject status code and DNR bit */
-		status = ns->fault_inject.status;
-		if (ns->fault_inject.dont_retry)
+		status = fault_inject->status;
+		if (fault_inject->dont_retry)
 			status |= NVME_SC_DNR;
 		nvme_req(req)->status =	status;
 	}
drivers/nvme/host/nvme.h +19 -15
index 55553d2..f00c596 100644
@@ -146,6 +146,15 @@ enum nvme_ctrl_state {
 	NVME_CTRL_DEAD,
 };
 
+struct nvme_fault_inject {
+#ifdef CONFIG_FAULT_INJECTION_DEBUG_FS
+	struct fault_attr attr;
+	struct dentry *parent;
+	bool dont_retry;	/* DNR, do not retry */
+	u16 status;		/* status code */
+#endif
+};
+
 struct nvme_ctrl {
 	bool comp_seen;
 	enum nvme_ctrl_state state;
@@ -313,15 +322,6 @@ struct nvme_ns_head {
 #endif
 };
 
-#ifdef CONFIG_FAULT_INJECTION_DEBUG_FS
-struct nvme_fault_inject {
-	struct fault_attr attr;
-	struct dentry *parent;
-	bool dont_retry;	/* DNR, do not retry */
-	u16 status;		/* status code */
-};
-#endif
-
 struct nvme_ns {
 	struct list_head list;
 
@@ -349,9 +349,7 @@ struct nvme_ns {
 #define NVME_NS_ANA_PENDING	2
 	u16 noiob;
 
-#ifdef CONFIG_FAULT_INJECTION_DEBUG_FS
 	struct nvme_fault_inject fault_inject;
-#endif
 
 };
 
@@ -372,12 +370,18 @@ struct nvme_ctrl_ops {
 };
 
 #ifdef CONFIG_FAULT_INJECTION_DEBUG_FS
-void nvme_fault_inject_init(struct nvme_ns *ns);
-void nvme_fault_inject_fini(struct nvme_ns *ns);
+void nvme_fault_inject_init(struct nvme_fault_inject *fault_inj,
+			    const char *dev_name);
+void nvme_fault_inject_fini(struct nvme_fault_inject *fault_inject);
 void nvme_should_fail(struct request *req);
 #else
-static inline void nvme_fault_inject_init(struct nvme_ns *ns) {}
-static inline void nvme_fault_inject_fini(struct nvme_ns *ns) {}
+static inline void nvme_fault_inject_init(struct nvme_fault_inject *fault_inj,
+					  const char *dev_name)
+{
+}
+static inline void nvme_fault_inject_fini(struct nvme_fault_inject *fault_inj)
+{
+}
 static inline void nvme_should_fail(struct request *req) {}
 #endif
 
Signature · 1 line
2.7.4
author maintainer
#2 lore raw

PATCH v3 2/3 nvme: enable to inject errors into admin commands

In reply to Akinobu Mita

This enables to inject errors into the commands submitted to the admin queue.

It is useful to test error handling in the controller initialization.

	# echo 100 > /sys/kernel/debug/nvme0/fault_inject/probability
	# echo 1 > /sys/kernel/debug/nvme0/fault_inject/times
	# echo 10 > /sys/kernel/debug/nvme0/fault_inject/space
	# nvme reset /dev/nvme0
	# dmesg
	...
	nvme nvme0: Could not set queue count (16385)
	nvme nvme0: IO queues not created
Cc Thomas Tai <thomas.tai at oracle.com>
Cc Keith Busch <kbusch at kernel.org>
Cc Jens Axboe <axboe at fb.com>
Cc Christoph Hellwig <hch at lst.de>
Cc Sagi Grimberg <sagi at grimberg.me>
Cc Minwoo Im <minwoo.im.dev at gmail.com>
Cc Chaitanya Kulkarni <chaitanya.kulkarni at wdc.com>
Signed-off-by Akinobu Mita <akinobu.mita at gmail.com>
Reviewed-by Minwoo Im <minwoo.im.dev at gmail.com>
Reviewed-by Sagi Grimberg <sagi at grimberg.me>
Reviewed-by Chaitanya Kulkarni <chaitanya.kulkarni at wdc.com>
Reviewed-by Christoph Hellwig <hch at lst.de>

---
* v3
- add Reviewed-by tags
- remove thermal zone series dependency

 drivers/nvme/host/core.c         | 3 +++
 drivers/nvme/host/fault_inject.c | 5 ++---
 drivers/nvme/host/nvme.h         | 2 ++
 3 files changed, 7 insertions(+), 3 deletions(-)
drivers/nvme/host/core.c +3 -0
index a66e7d8..80960e4 100644
@@ -3701,6 +3701,7 @@ EXPORT_SYMBOL_GPL(nvme_start_ctrl);
 
 void nvme_uninit_ctrl(struct nvme_ctrl *ctrl)
 {
+	nvme_fault_inject_fini(&ctrl->fault_inject);
 	dev_pm_qos_hide_latency_tolerance(ctrl->device);
 	cdev_device_del(&ctrl->cdev, ctrl->device);
 }
@@ -3796,6 +3797,8 @@ int nvme_init_ctrl(struct nvme_ctrl *ctrl, struct device *dev,
 	dev_pm_qos_update_user_latency_tolerance(ctrl->device,
 		min(default_ps_max_latency_us, (unsigned long)S32_MAX));
 
+	nvme_fault_inject_init(&ctrl->fault_inject, dev_name(ctrl->device));
+
 	return 0;
 out_free_name:
 	kfree_const(ctrl->device->kobj.name);
drivers/nvme/host/fault_inject.c +2 -3
index e37b8c2..1352159 100644
@@ -60,9 +60,6 @@ void nvme_should_fail(struct request *req)
 	struct nvme_fault_inject *fault_inject = NULL;
 	u16 status;
 
-	/*
-	 * make sure this request is coming from a valid namespace
-	 */
 	if (disk) {
 		struct nvme_ns *ns = disk->private_data;
 
@@ -70,6 +67,8 @@ void nvme_should_fail(struct request *req)
 			fault_inject = &ns->fault_inject;
 		else
 			WARN_ONCE(1, "No namespace found for request\n");
+	} else {
+		fault_inject = &nvme_req(req)->ctrl->fault_inject;
 	}
 
 	if (fault_inject && should_fail(&fault_inject->attr, 1)) {
drivers/nvme/host/nvme.h +2 -0
index f00c596..1b2624c 100644
@@ -256,6 +256,8 @@ struct nvme_ctrl {
 
 	struct page *discard_page;
 	unsigned long discard_page_busy;
+
+	struct nvme_fault_inject fault_inject;
 };
 
 enum nvme_iopolicy {
Signature · 1 line
2.7.4
author maintainer
#3 lore raw

PATCH v3 3/3 Documentation: nvme: add an example for nvme fault injection

In reply to Akinobu Mita

This adds an example of how to inject errors into admin commands.

Cc Thomas Tai <thomas.tai at oracle.com>
Cc Keith Busch <kbusch at kernel.org>
Cc Jens Axboe <axboe at fb.com>
Cc Christoph Hellwig <hch at lst.de>
Cc Sagi Grimberg <sagi at grimberg.me>
Cc Minwoo Im <minwoo.im.dev at gmail.com>
Cc Chaitanya Kulkarni <chaitanya.kulkarni at wdc.com>
Suggested-by Thomas Tai <thomas.tai at oracle.com>
Signed-off-by Akinobu Mita <akinobu.mita at gmail.com>
Reviewed-by Chaitanya Kulkarni <chaitanya.kulkarni at wdc.com>
Reviewed-by Minwoo Im <minwoo.im at samsung.com>
Reviewed-by Christoph Hellwig <hch at lst.de>

---
* v3
- add Reviewed-by tags

 .../fault-injection/nvme-fault-injection.txt       | 56 ++++++++++++++++++++++
 1 file changed, 56 insertions(+)
Documentation/fault-injection/nvme-fault-injection.txt +56 -0
index 8fbf3bf..efcb339 100644
@@ -114,3 +114,59 @@ R13: ffff88011a3c9680 R14: 0000000000000000 R15: 0000000000000000
   cpu_startup_entry+0x6f/0x80
   start_secondary+0x187/0x1e0
   secondary_startup_64+0xa5/0xb0
+
+Example 3: Inject an error into the 10th admin command
+------------------------------------------------------
+
+echo 100 > /sys/kernel/debug/nvme0/fault_inject/probability
+echo 10 > /sys/kernel/debug/nvme0/fault_inject/space
+echo 1 > /sys/kernel/debug/nvme0/fault_inject/times
+nvme reset /dev/nvme0
+
+Expected Result:
+
+After NVMe controller reset, the reinitialization may or may not succeed.
+It depends on which admin command is actually forced to fail.
+
+Message from dmesg:
+
+nvme nvme0: resetting controller
+FAULT_INJECTION: forcing a failure.
+name fault_inject, interval 1, probability 100, space 1, times 1
+CPU: 0 PID: 0 Comm: swapper/0 Not tainted 5.2.0-rc2+ #2
+Hardware name: MSI MS-7A45/B150M MORTAR ARCTIC (MS-7A45), BIOS 1.50 04/25/2017
+Call Trace:
+ <IRQ>
+ dump_stack+0x63/0x85
+ should_fail+0x14a/0x170
+ nvme_should_fail+0x38/0x80 [nvme_core]
+ nvme_irq+0x129/0x280 [nvme]
+ ? blk_mq_end_request+0xb3/0x120
+ __handle_irq_event_percpu+0x84/0x1a0
+ handle_irq_event_percpu+0x32/0x80
+ handle_irq_event+0x3b/0x60
+ handle_edge_irq+0x7f/0x1a0
+ handle_irq+0x20/0x30
+ do_IRQ+0x4e/0xe0
+ common_interrupt+0xf/0xf
+ </IRQ>
+RIP: 0010:cpuidle_enter_state+0xc5/0x460
+Code: ff e8 8f 5f 86 ff 80 7d c7 00 74 17 9c 58 0f 1f 44 00 00 f6 c4 02 0f 85 69 03 00 00 31 ff e8 62 aa 8c ff fb 66 0f 1f 44 00 00 <45> 85 ed 0f 88 37 03 00 00 4c 8b 45 d0 4c 2b 45 b8 48 ba cf f7 53
+RSP: 0018:ffffffff88c03dd0 EFLAGS: 00000246 ORIG_RAX: ffffffffffffffdc
+RAX: ffff9dac25a2ac80 RBX: ffffffff88d53760 RCX: 000000000000001f
+RDX: 0000000000000000 RSI: 000000002d958403 RDI: 0000000000000000
+RBP: ffffffff88c03e18 R08: fffffff75e35ffb7 R09: 00000a49a56c0b48
+R10: ffffffff88c03da0 R11: 0000000000001b0c R12: ffff9dac25a34d00
+R13: 0000000000000006 R14: 0000000000000006 R15: ffffffff88d53760
+ cpuidle_enter+0x2e/0x40
+ call_cpuidle+0x23/0x40
+ do_idle+0x201/0x280
+ cpu_startup_entry+0x1d/0x20
+ rest_init+0xaa/0xb0
+ arch_call_rest_init+0xe/0x1b
+ start_kernel+0x51c/0x53b
+ x86_64_start_reservations+0x24/0x26
+ x86_64_start_kernel+0x74/0x77
+ secondary_startup_64+0xa4/0xb0
+nvme nvme0: Could not set queue count (16385)
+nvme nvme0: IO queues not created
Signature · 1 line
2.7.4