nvme-pci: support device coredump
Subsystems (3)
v2 cover nvme-pci: support device coredump
This enables to capture snapshot of controller information via device coredump machanism, and it helps diagnose and debug issues.
The nvme device coredump is triggered when command timeout occurs, and creates the following coredump files.
- regs: NVMe controller registers (00h to 4Fh)
- sq<qid>: Submission queue
- cq<qid>: Completion queue
- telemetry-ctrl-log: Telemetry controller-initiated log (if available)
- data: Empty
(I don't have the NVMe device that supports telemetry log page for now, so capturing telemetry log is untested.)
The device coredump mechanism currently allows drivers to create only a single coredump file, so this also provides a new function that allows drivers to create several device coredump files in one crashed device.
* v2
- Add Reviewed-by tag.
- Add patch to fix typo in comment
- Remove unneeded braces.
- Allocate device_entry followed by an array of devcd_file elements.
- Add telemetry log page definisions
- Add facility to check log page attributes
- Exclude the doorbell registers from register dump.
- Save controller registers in a binary format instead of a text format.
- Create an empty 'data' file in the device coredump.
- Save telemetry controller-initiated log if available
- Make coredump procedure into two phases (before resetting controller and after resetting as soon as admin queue is available).
Akinobu Mita (7):
devcoredump: use memory_read_from_buffer
devcoredump: fix typo in comment
devcoredump: allow to create several coredump files in one device nvme.h: add telemetry log page definisions
nvme: add facility to check log page attributes
nvme-pci: add device coredump support
nvme-pci: trigger device coredump on command timeout
drivers/base/devcoredump.c | 168 ++++++++++------ drivers/nvme/host/Kconfig | 1 + drivers/nvme/host/core.c | 2 + drivers/nvme/host/nvme.h | 1 + drivers/nvme/host/pci.c | 460 ++++++++++++++++++++++++++++++++++++++++++-- include/linux/devcoredump.h | 33 ++++ include/linux/nvme.h | 25 +++ 7 files changed, 617 insertions(+), 73 deletions(-)
Signature · 1 line
2.7.4
PATCH v2 1/7 devcoredump: use memory_read_from_buffer
Use memory_read_from_buffer() to simplify devcd_readv().
---
* v2
- Add Reviewed-by tag.
drivers/base/devcoredump.c | 11 +---------- 1 file changed, 1 insertion(+), 10 deletions(-)
drivers/base/devcoredump.c +1 -10
@@ -164,16 +164,7 @@ static struct class devcd_class = {static ssize_t devcd_readv (char *buffer,loff_t offset,size_t count,void *data,size_t datalen) { -if (offset > datalen) -return -EINVAL ; - -if (offset + count > datalen) - count = datalen - offset; - -if (count) -memcpy (buffer, ((u8 *)data) + offset, count); - -return count; +return memory_read_from_buffer (buffer, count, &offset, data, datalen); }static void devcd_freev (void *data)
Signature · 1 line
2.7.4
PATCH v2 2/7 devcoredump: fix typo in comment
In reply to Akinobu Mitas/dev_coredumpmsg/dev_coredumpsg/
---
* v2
- New patch in this version.
drivers/base/devcoredump.c | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-)
drivers/base/devcoredump.c +1 -1
@@ -314,7 +314,7 @@ void dev_coredumpm(struct device *dev, struct module *owner,EXPORT_SYMBOL_GPL (dev_coredumpm);/** -* dev_coredumpmsg - create device coredump that uses scatterlist as data +* dev_coredumpsg - create device coredump that uses scatterlist as data * parameter * @dev: the struct device for the crashed device * @table: the dump data
Signature · 1 line
2.7.4
PATCH v2 3/7 devcoredump: allow to create several coredump files in one device
In reply to Akinobu MitaThe device coredump mechanism currently allows drivers to create only a single coredump file. If there are several binary blobs to dump, we need to define a binary format or conver to text format in order to put them into a single coredump file.
This provides a new function that allows drivers to create several device coredump files in one crashed device.
---
* v2
- Remove unneeded braces.
- Allocate device_entry followed by an array of devcd_file elements.
drivers/base/devcoredump.c | 155 ++++++++++++++++++++++++++++++-------------- include/linux/devcoredump.h | 33 ++++++++++ 2 files changed, 139 insertions(+), 49 deletions(-)
drivers/base/devcoredump.c +106 -49
@@ -25,16 +25,20 @@ static bool devcd_disabled;/* if data isn't read by userspace after 5 minutes then delete it */ #define DEVCD_TIMEOUT (HZ * 60 * 5) -struct devcd_entry { -struct device devcd_dev; -void *data; -size_t datalen; -struct module *owner; +struct devcd_file { +struct bin_attribute bin_attr;ssize_t (*read)(char *buffer,loff_t offset,size_t count,void *data,size_t datalen);void (*free)(void *data); +}; + +struct devcd_entry { +struct device devcd_dev; +struct module *owner;struct delayed_work del_wk;struct device *failing_dev; +int num_files; +struct devcd_file files[]; };static struct devcd_entry *dev_to_devcd (struct device *dev)
@@ -45,8 +49,14 @@ static struct devcd_entry *dev_to_devcd(struct device *dev)static void devcd_dev_release (struct device *dev) {struct devcd_entry *devcd =dev_to_devcd (dev); +int i; + +for (i =0 ; i < devcd->num_files; i++) { +struct devcd_file *file = &devcd->files[i]; + + file->free (file->bin_attr.private); + } - devcd->free (devcd->data);module_put (devcd->owner);/*
@@ -64,9 +74,14 @@ static void devcd_dev_release(struct device *dev)static void devcd_del (struct work_struct *wk) {struct devcd_entry *devcd; +int i; devcd = container_of(wk, struct devcd_entry, del_wk.work); +for (i =0 ; i < devcd->num_files; i++) +device_remove_bin_file (&devcd->devcd_dev, + &devcd->files[i].bin_attr); +device_del (&devcd->devcd_dev);put_device (&devcd->devcd_dev); }
@@ -75,10 +90,11 @@ static ssize_t devcd_data_read(struct file *filp, struct kobject *kobj,struct bin_attribute *bin_attr, char *buffer, loff_t offset, size_t count) { - struct device *dev = kobj_to_dev(kobj); -struct devcd_entry *devcd =dev_to_devcd (dev); + struct devcd_file *file = + container_of(bin_attr, struct devcd_file, bin_attr); -return devcd->read (buffer, offset, count, devcd->data, devcd->datalen); +return file->read (buffer, offset, count, bin_attr->private, + bin_attr->size); } static ssize_t devcd_data_write(struct file *filp, struct kobject *kobj,
@@ -93,25 +109,6 @@ static ssize_t devcd_data_write(struct file *filp, struct kobject *kobj,return count; } -static struct bin_attribute devcd_attr_data = { - .attr = { .name ="data" , .mode =S_IRUSR |S_IWUSR , }, - .size =0 , - .read = devcd_data_read, - .write = devcd_data_write, -}; - -static struct bin_attribute *devcd_dev_bin_attrs[] = { - &devcd_attr_data,NULL , -}; - -static const struct attribute_group devcd_dev_group = { - .bin_attrs = devcd_dev_bin_attrs, -}; - -static const struct attribute_group *devcd_dev_groups[] = { - &devcd_dev_group,NULL , -}; -static int devcd_free (struct device *dev,void *data) {struct devcd_entry *devcd =dev_to_devcd (dev);
@@ -157,7 +154,6 @@ static struct class devcd_class = { .name = "devcoredump", .owner = THIS_MODULE, .dev_release = devcd_dev_release, - .dev_groups = devcd_dev_groups, .class_groups = devcd_class_groups, };
@@ -234,30 +230,55 @@ static ssize_t devcd_read_from_sgtable(char *buffer, loff_t offset, offset); } +static struct devcd_entry *devcd_alloc (struct dev_coredumpm_bulk_data *files, +int num_files,gfp_t gfp) +{ +struct devcd_entry *devcd; +int i; + + devcd =kzalloc (struct_size (devcd, files, num_files), gfp); +if (!devcd) +return NULL ; + + devcd->num_files = num_files; + +for (i =0 ; i < devcd->num_files; i++) { +struct devcd_file *file = &devcd->files[i]; + +sysfs_bin_attr_init (&file->bin_attr); + file->bin_attr.attr.name = files[i].name; + + file->bin_attr.attr.mode =0600 ; + file->bin_attr.size = files[i].datalen; + file->bin_attr.private = files[i].data; + file->bin_attr.read = devcd_data_read; + file->bin_attr.write = devcd_data_write; + + file->read = files[i].read; + file->free = files[i].free; + } + +return devcd; +} +/** -* dev_coredumpm - create device coredump with read/free methods +* dev_coredumpm_bulk - create a number of device coredump files * @dev: the struct device for the crashed device * @owner: the module that contains the read/free functions, use %THIS_MODULE -* @data: data cookie for the @read/@free functions -* @datalen: length of the data * @gfp: allocation flags -* @read: function to read from the given buffer -* @free: function to free the given buffer +* @files: the configuration of device coredump files +* @num_files: the number of device coredump files to create * -* Creates a new device coredump for the given device. If a previous one hasn't -* been read yet, the new coredump is discarded. The data lifetime is determined -* by the device coredump framework and when it is no longer needed the @free -* function will be called to free the data. +* This function allows drivers to create several device coredump files in +* one crashed device. */ -void dev_coredumpm (struct device *dev,struct module *owner, -void *data,size_t datalen,gfp_t gfp, -ssize_t (*read)(char *buffer,loff_t offset,size_t count, -void *data,size_t datalen), -void (*free)(void *data)) +void dev_coredumpm_bulk (struct device *dev,struct module *owner,gfp_t gfp, +struct dev_coredumpm_bulk_data *files,int num_files) {static atomic_t devcd_count =ATOMIC_INIT (0 );struct devcd_entry *devcd;struct device *existing; +int i;if (devcd_disabled) goto free;
@@ -272,15 +293,11 @@ void dev_coredumpm(struct device *dev, struct module *owner,if (!try_module_get (owner)) goto free; - devcd =kzalloc (sizeof (*devcd), gfp); + devcd =devcd_alloc (files, num_files, gfp);if (!devcd) goto put_module; devcd->owner = owner; - devcd->data = data; - devcd->datalen = datalen; - devcd->read = read; - devcd->free = free; devcd->failing_dev =get_device (dev);device_initialize (&devcd->devcd_dev);
@@ -292,6 +309,12 @@ void dev_coredumpm(struct device *dev, struct module *owner,if (device_add (&devcd->devcd_dev)) goto put_device; +for (i =0 ; i < devcd->num_files; i++) { +if (device_create_bin_file (&devcd->devcd_dev, + &devcd->files[i].bin_attr)) +/* nothing - some files will be missing */ ; + } +if (sysfs_create_link (&devcd->devcd_dev.kobj, &dev->kobj,"failing_device" ))/* nothing - symlink will be missing */ ;
@@ -309,7 +332,41 @@ void dev_coredumpm(struct device *dev, struct module *owner, put_module:module_put (owner); free: -free (data); +for (i =0 ; i < num_files; i++) + files[i].free (files[i].data); +} +EXPORT_SYMBOL_GPL (dev_coredumpm_bulk); + +/** +* dev_coredumpm - create device coredump with read/free methods +* @dev: the struct device for the crashed device +* @owner: the module that contains the read/free functions, use %THIS_MODULE +* @data: data cookie for the @read/@free functions +* @datalen: length of the data +* @gfp: allocation flags +* @read: function to read from the given buffer +* @free: function to free the given buffer +* +* Creates a new device coredump for the given device. If a previous one hasn't +* been read yet, the new coredump is discarded. The data lifetime is determined +* by the device coredump framework and when it is no longer needed the @free +* function will be called to free the data. +*/ +void dev_coredumpm (struct device *dev,struct module *owner, +void *data,size_t datalen,gfp_t gfp, +ssize_t (*read)(char *buffer,loff_t offset,size_t count, +void *data,size_t datalen), +void (*free)(void *data)) +{ +struct dev_coredumpm_bulk_data bulk_data = { + .name ="data" , + .data = data, + .datalen = datalen, + .read = read, + .free = free, + }; + +dev_coredumpm_bulk (dev, owner, gfp, &bulk_data,1 ); }EXPORT_SYMBOL_GPL (dev_coredumpm);
include/linux/devcoredump.h +33 -0
@@ -65,6 +65,26 @@ static inline void _devcd_free_sgtable(struct scatterlist *table)kfree (delete_iter); } +/** +* struct dev_coredumpm_bulk_data - Data used for dev_coredumpm_bulk +* +* @name: coredump file name +* @data: data cookie for the @read/@free functions +* @datalen: length of the data +* @read: function to read from the given buffer +* @free: function to free the given buffer +* +* An array of this structure is passed as argument to dev_coredumpm_bulk, and +* used to describe each device coredump. +*/ +struct dev_coredumpm_bulk_data { +char *name; +void *data; +size_t datalen; +ssize_t (*read)(char *buffer,loff_t offset,size_t count, +void *data,size_t datalen); +void (*free)(void *data); +};#ifdef CONFIG_DEV_COREDUMP void dev_coredumpv(struct device *dev, void *data, size_t datalen,
@@ -76,6 +96,9 @@ void dev_coredumpm(struct device *dev, struct module *owner, void *data, size_t datalen), void (*free)(void *data)); +void dev_coredumpm_bulk (struct device *dev,struct module *owner,gfp_t gfp, +struct dev_coredumpm_bulk_data *files,int num_files); +void dev_coredumpsg (struct device *dev,struct scatterlist *table,size_t datalen,gfp_t gfp);#else
@@ -95,6 +118,16 @@ dev_coredumpm(struct device *dev, struct module *owner,free (data); } +static inline +void dev_coredumpm_bulk (struct device *dev,struct module *owner,gfp_t gfp, +struct dev_coredumpm_bulk_data *files,int num_files) +{ +int i; + +for (i =0 ; i < num_files; i++) + files[i].free (files[i].data); +} +static inline void dev_coredumpsg (struct device *dev,struct scatterlist *table,size_t datalen,gfp_t gfp) {
Signature · 1 line
2.7.4
Re: [PATCH v2 3/7] devcoredump: allow to create several coredump files in one device
drivers/base/devcoredump.c void dev_coredumpm(struct device *dev, struct module *owner, · line 315
Quoted 8 lines · Akinobu Mita
drivers/base/devcoredump.c +4
-0@@ -292,6 +309,12 @@ void dev_coredumpm(struct device *dev, struct module *owner,if (device_add (&devcd->devcd_dev)) goto put_device; +for (i =0 ; i < devcd->num_files; i++) { +if (device_create_bin_file (&devcd->devcd_dev, + &devcd->files[i].bin_attr)) +/* nothing - some files will be missing */ ;
Is the conditional necessary if you aren't going to do anything?
Quoted 5 lines
drivers/base/devcoredump.c +2
-0+ } +if (sysfs_create_link (&devcd->devcd_dev.kobj, &dev->kobj,"failing_device" ))/* nothing - symlink will be missing */ ;
Re: [PATCH v2 3/7] devcoredump: allow to create several coredump files in one device
2019?5?8?(?) 2:35 Heitke, Kenneth <kenneth.heitke at intel.com>:
Quoted 8 lines · Akinobu Mita
Quoted 4 lines
drivers/base/devcoredump.c +0
-0@@ -292,6 +309,12 @@ void dev_coredumpm(struct device *dev, struct module *owner,if (device_add (&devcd->devcd_dev)) goto put_device;
Quoted 6 lines
drivers/base/devcoredump.c void dev_coredumpm(struct device *dev, struct module *owner, · line 315
Quoted 4 lines
drivers/base/devcoredump.c +4
-0+for (i =0 ; i < devcd->num_files; i++) { +if (device_create_bin_file (&devcd->devcd_dev, + &devcd->files[i].bin_attr)) +/* nothing - some files will be missing */ ;Is the conditional necessary if you aren't going to do anything?
The device_create_bin_file() is declared with __must_check, so ignoring the return value emits warning.
PATCH v2 4/7 nvme.h: add telemetry log page definisions
In reply to Akinobu MitaCopy telemetry log page definisions from nvme-cli.
---
* v2
- New patch in this version.
include/linux/nvme.h | 23 +++++++++++++++++++++++ 1 file changed, 23 insertions(+)
include/linux/nvme.h +23 -0
@@ -396,6 +396,28 @@ enum { NVME_NIDT_UUID = 0x03, }; +/* Derived from 1.3a Figure 101: Get Log Page ? Telemetry Host +* -Initiated Log (Log Identifier 07h) +*/ +struct nvme_telemetry_log_page_hdr { +__u8 lpi;/* Log page identifier */ +__u8 rsvd[4 ]; +__u8 iee_oui[3 ]; +__le16 dalb1;/* Data area 1 last block */ +__le16 dalb2;/* Data area 2 last block */ +__le16 dalb3;/* Data area 3 last block */ +__u8 rsvd1[368 ];/* TODO verify */ +__u8 ctrlavail;/* Controller initiated data avail?*/ +__u8 ctrldgn;/* Controller initiated telemetry Data Gen # */ +__u8 rsnident[128 ]; +/* We'll have to double fetch so we can get the header, +* parse dalb1->3 determine how much size we need for the +* log then alloc below. Or just do a secondary non-struct +* allocation. +*/ +__u8 telemetry_dataarea[0 ]; +}; +struct nvme_smart_log {__u8 critical_warning;__u8 temperature[2 ];
@@ -832,6 +854,7 @@ enum {NVME_LOG_FW_SLOT =0x03 ,NVME_LOG_CHANGED_NS =0x04 ,NVME_LOG_CMD_EFFECTS =0x05 , +NVME_LOG_TELEMETRY_CTRL =0x08 ,NVME_LOG_ANA =0x0c ,NVME_LOG_DISC =0x70 , NVME_LOG_RESERVATION = 0x80,
Signature · 1 line
2.7.4
Re: [PATCH v2 4/7] nvme.h: add telemetry log page definisions
Quoted 25 lines · Akinobu Mita
Copy telemetry log page definisions from nvme-cli.
Cc Johannes Berg <johannes at sipsolutions.net>Cc Keith Busch <keith.busch at intel.com>Cc Jens Axboe <axboe at fb.com>Cc Christoph Hellwig <hch at lst.de>Cc Sagi Grimberg <sagi at grimberg.me>Cc Minwoo Im <minwoo.im.dev at gmail.com>Signed-off-by Akinobu Mita <akinobu.mita at gmail.com>---
* v2
- New patch in this version.include/linux/nvme.h | 23 +++++++++++++++++++++++ 1 file changed, 23 insertions(+)include/linux/nvme.h +1
-0@@ -396,6 +396,28 @@ enum { NVME_NIDT_UUID = 0x03, }; +/* Derived from 1.3a Figure 101: Get Log Page ? Telemetry Host
include/linux/nvme.h enum { · line 409
Quoted 10 lines
include/linux/nvme.h +10
-0+* -Initiated Log (Log Identifier 07h) +*/ +struct nvme_telemetry_log_page_hdr { +__u8 lpi;/* Log page identifier */ +__u8 rsvd[4 ]; +__u8 iee_oui[3 ]; +__le16 dalb1;/* Data area 1 last block */ +__le16 dalb2;/* Data area 2 last block */ +__le16 dalb3;/* Data area 3 last block */ +__u8 rsvd1[368 ];/* TODO verify */
Remove the TODO
include/linux/nvme.h enum { · line 417
Quoted 8 lines
include/linux/nvme.h +8
-0+__u8 ctrlavail;/* Controller initiated data avail?*/ +__u8 ctrldgn;/* Controller initiated telemetry Data Gen # */ +__u8 rsnident[128 ]; +/* We'll have to double fetch so we can get the header, +* parse dalb1->3 determine how much size we need for the +* log then alloc below. Or just do a secondary non-struct +* allocation. +*/
This comment isn't necessary. You usually can't read the entire telemetry log at once and the header is a fixed size. You would likely just read the header followed by reads of the different data areas.
Quoted 15 lines
include/linux/nvme.h +4
-0+__u8 telemetry_dataarea[0 ]; +}; +struct nvme_smart_log {__u8 critical_warning;__u8 temperature[2 ];@@ -832,6 +854,7 @@ enum {NVME_LOG_FW_SLOT =0x03 ,NVME_LOG_CHANGED_NS =0x04 ,NVME_LOG_CMD_EFFECTS =0x05 , +NVME_LOG_TELEMETRY_CTRL =0x08 ,NVME_LOG_ANA =0x0c ,NVME_LOG_DISC =0x70 , NVME_LOG_RESERVATION = 0x80,
Re: [PATCH v2 4/7] nvme.h: add telemetry log page definisions
2019?5?8?(?) 2:28 Heitke, Kenneth <kenneth.heitke at intel.com>:
Quoted 28 lines · Akinobu Mita
Quoted 24 lines
Copy telemetry log page definisions from nvme-cli.
Cc Johannes Berg <johannes at sipsolutions.net>Cc Keith Busch <keith.busch at intel.com>Cc Jens Axboe <axboe at fb.com>Cc Christoph Hellwig <hch at lst.de>Cc Sagi Grimberg <sagi at grimberg.me>Cc Minwoo Im <minwoo.im.dev at gmail.com>Signed-off-by Akinobu Mita <akinobu.mita at gmail.com>---
* v2
- New patch in this version.include/linux/nvme.h | 23 +++++++++++++++++++++++ 1 file changed, 23 insertions(+)include/linux/nvme.h +0
-0@@ -396,6 +396,28 @@ enum { NVME_NIDT_UUID = 0x03, };
Quoted 13 lines
include/linux/nvme.h enum { · line 409
Quoted 11 lines
include/linux/nvme.h +11
-0+/* Derived from 1.3a Figure 101: Get Log Page ? Telemetry Host +* -Initiated Log (Log Identifier 07h) +*/ +struct nvme_telemetry_log_page_hdr { +__u8 lpi;/* Log page identifier */ +__u8 rsvd[4 ]; +__u8 iee_oui[3 ]; +__le16 dalb1;/* Data area 1 last block */ +__le16 dalb2;/* Data area 2 last block */ +__le16 dalb3;/* Data area 3 last block */ +__u8 rsvd1[368 ];/* TODO verify */ Remove the TODO
OK.
Quoted 9 lines
include/linux/nvme.h enum { · line 417
Quoted 8 lines
include/linux/nvme.h +8
-0+__u8 ctrlavail;/* Controller initiated data avail?*/ +__u8 ctrldgn;/* Controller initiated telemetry Data Gen # */ +__u8 rsnident[128 ]; +/* We'll have to double fetch so we can get the header, +* parse dalb1->3 determine how much size we need for the +* log then alloc below. Or just do a secondary non-struct +* allocation. +*/
Quoted 3 lines
This comment isn't necessary. You usually can't read the entire telemetry log at once and the header is a fixed size. You would likely just read the header followed by reads of the different data areas.
This comment is derived from nvme-cli. So firstly, I'll send a patch for nvme-cli. If the changes are accepted, I'll update this comment, too.
Quoted 15 lines
Quoted 15 lines
include/linux/nvme.h +4
-0+__u8 telemetry_dataarea[0 ]; +}; +struct nvme_smart_log {__u8 critical_warning;__u8 temperature[2 ];@@ -832,6 +854,7 @@ enum {NVME_LOG_FW_SLOT =0x03 ,NVME_LOG_CHANGED_NS =0x04 ,NVME_LOG_CMD_EFFECTS =0x05 , +NVME_LOG_TELEMETRY_CTRL =0x08 ,NVME_LOG_ANA =0x0c ,NVME_LOG_DISC =0x70 , NVME_LOG_RESERVATION = 0x80,
Re: [PATCH v2 4/7] nvme.h: add telemetry log page definisions
In reply to Akinobu MitaQuoted 21 lines · Akinobu Mita
Copy telemetry log page definisions from nvme-cli.
Cc Johannes Berg <johannes at sipsolutions.net>Cc Keith Busch <keith.busch at intel.com>Cc Jens Axboe <axboe at fb.com>Cc Christoph Hellwig <hch at lst.de>Cc Sagi Grimberg <sagi at grimberg.me>Cc Minwoo Im <minwoo.im.dev at gmail.com>Signed-off-by Akinobu Mita <akinobu.mita at gmail.com>---
* v2
- New patch in this version.include/linux/nvme.h | 23 +++++++++++++++++++++++ 1 file changed, 23 insertions(+)include/linux/nvme.h +0
-0
include/linux/nvme.h enum { · line 401
Quoted 6 lines
include/linux/nvme.h +3
-0@@ -396,6 +396,28 @@ enum { NVME_NIDT_UUID = 0x03, }; +/* Derived from 1.3a Figure 101: Get Log Page ? Telemetry Host +* -Initiated Log (Log Identifier 07h) +*/
Is this Host Initiated or Controller Initiated? The comment says host initiated but everything else seems to indicated controller initiated. Is controller initiated even the correct choice because the controller would have sent an AER to indicate that the host should pull the telemetry data.
Quoted 31 lines
include/linux/nvme.h +20
-0+struct nvme_telemetry_log_page_hdr { +__u8 lpi;/* Log page identifier */ +__u8 rsvd[4 ]; +__u8 iee_oui[3 ]; +__le16 dalb1;/* Data area 1 last block */ +__le16 dalb2;/* Data area 2 last block */ +__le16 dalb3;/* Data area 3 last block */ +__u8 rsvd1[368 ];/* TODO verify */ +__u8 ctrlavail;/* Controller initiated data avail?*/ +__u8 ctrldgn;/* Controller initiated telemetry Data Gen # */ +__u8 rsnident[128 ]; +/* We'll have to double fetch so we can get the header, +* parse dalb1->3 determine how much size we need for the +* log then alloc below. Or just do a secondary non-struct +* allocation. +*/ +__u8 telemetry_dataarea[0 ]; +}; +struct nvme_smart_log {__u8 critical_warning;__u8 temperature[2 ];@@ -832,6 +854,7 @@ enum {NVME_LOG_FW_SLOT =0x03 ,NVME_LOG_CHANGED_NS =0x04 ,NVME_LOG_CMD_EFFECTS =0x05 , +NVME_LOG_TELEMETRY_CTRL =0x08 ,NVME_LOG_ANA =0x0c ,NVME_LOG_DISC =0x70 , NVME_LOG_RESERVATION = 0x80,
Re: [PATCH v2 4/7] nvme.h: add telemetry log page definisions
2019?5?8?(?) 2:53 Heitke, Kenneth <kenneth.heitke at intel.com>:
Quoted 28 lines · Akinobu Mita
Quoted 24 lines
Copy telemetry log page definisions from nvme-cli.
Cc Johannes Berg <johannes at sipsolutions.net>Cc Keith Busch <keith.busch at intel.com>Cc Jens Axboe <axboe at fb.com>Cc Christoph Hellwig <hch at lst.de>Cc Sagi Grimberg <sagi at grimberg.me>Cc Minwoo Im <minwoo.im.dev at gmail.com>Signed-off-by Akinobu Mita <akinobu.mita at gmail.com>---
* v2
- New patch in this version.include/linux/nvme.h | 23 +++++++++++++++++++++++ 1 file changed, 23 insertions(+)include/linux/nvme.h +0
-0@@ -396,6 +396,28 @@ enum { NVME_NIDT_UUID = 0x03, };
Quoted 6 lines
include/linux/nvme.h enum { · line 401
Quoted 3 lines
include/linux/nvme.h +3
-0+/* Derived from 1.3a Figure 101: Get Log Page ? Telemetry Host +* -Initiated Log (Log Identifier 07h) +*/ Is this Host Initiated or Controller Initiated? The comment says host initiated but everything else seems to indicated controller initiated.
Both telemetry host initiated and controller initiated log headers have the same structure. If this comment is confusing, it is also considered to be removed.
Quoted 3 lines
Is controller initiated even the correct choice because the controller would have sent an AER to indicate that the host should pull the telemetry data.
It seems useful to retrieve telemetry log continually with the aid of user space tool reacting an Asynchronous Event.
Similarly, it could be useful to retrieve telemetry log as soon as the device is successfully recovered from the crash. (Although I still do not find the device that has Telemetry Controller-Initiated Data Available field is set to 1h.)
PATCH v2 5/7 nvme: add facility to check log page attributes
In reply to Akinobu MitaThis provides a facility to check whether the controller supports the telemetry log pages and log page offset field for the Get Log Page command.
---
* v2
- New patch in this version.
drivers/nvme/host/core.c | 1 + drivers/nvme/host/nvme.h | 1 + include/linux/nvme.h | 2 ++ 3 files changed, 4 insertions(+)
drivers/nvme/host/core.c +1 -0
@@ -2580,6 +2580,7 @@ int nvme_init_identify(struct nvme_ctrl *ctrl) }else ctrl->shutdown_timeout = shutdown_timeout; + ctrl->lpa = id->lpa; ctrl->npss = id->npss; ctrl->apsta = id->apsta; prev_apst_enabled = ctrl->apst_enabled;
drivers/nvme/host/nvme.h +1 -0
@@ -195,6 +195,7 @@ struct nvme_ctrl {u32 vs;u32 sgls;u16 kas; +u8 lpa;u8 npss;u8 apsta;u32 oaes;
include/linux/nvme.h +2 -0
@@ -294,6 +294,8 @@ enum {NVME_CTRL_OACS_DIRECTIVES =1 <<5 ,NVME_CTRL_OACS_DBBUF_SUPP =1 <<8 ,NVME_CTRL_LPA_CMD_EFFECTS_LOG =1 <<1 , +NVME_CTRL_LPA_EXTENDED_DATA =1 <<2 , + NVME_CTRL_LPA_TELEMETRY_LOG = 1 << 3, }; struct nvme_lbaf {
Signature · 1 line
2.7.4
PATCH v2 6/7 nvme-pci: add device coredump support
In reply to Akinobu MitaThis enables to capture snapshot of controller information via device coredump machanism.
The nvme device coredump creates the following coredump files.
- regs: NVMe controller registers (00h to 4Fh)
- sq<qid>: Submission queue
- cq<qid>: Completion queue
- telemetry-ctrl-log: Telemetry controller-initiated log (if available)
- data: Empty
The reason for an empty 'data' file is to provide a uniform way to notify the device coredump is no longer needed by writing the 'data' file.
Since all existing drivers using the device coredump provide a 'data' file if the nvme device coredump doesn't provide it, the userspace programs need to know which driver provides what coredump file.
---
* v2
- Exclude the doorbell registers from register dump.
- Save controller registers in a binary format instead of a text format.
- Create an empty 'data' file in the device coredump.
- Save telemetry controller-initiated log if available
drivers/nvme/host/Kconfig | 1 + drivers/nvme/host/core.c | 1 + drivers/nvme/host/pci.c | 425 ++++++++++++++++++++++++++++++++++++++++++++++ 3 files changed, 427 insertions(+)
drivers/nvme/host/Kconfig +1 -0
@@ -5,6 +5,7 @@ config BLK_DEV_NVME tristate "NVM Express block device" depends on PCI && BLOCK select NVME_CORE + select WANT_DEV_COREDUMP ---help--- The NVM Express driver is for solid state drives directly connected to the PCI or PCI Express bus. If you know you
drivers/nvme/host/core.c +1 -0
@@ -2457,6 +2457,7 @@ int nvme_get_log(struct nvme_ctrl *ctrl, u32 nsid, u8 log_page, u8 lsp,return nvme_submit_sync_cmd (ctrl->admin_q, &c, log, size); } +EXPORT_SYMBOL_GPL (nvme_get_log);static int nvme_get_effects_log (struct nvme_ctrl *ctrl) {
drivers/nvme/host/pci.c +425 -0
449 lines not on this page · raw message
Signature · 1 line
2.7.4
Keith Busch
maintainerRe: [PATCH v2 6/7] nvme-pci: add device coredump support
drivers/nvme/host/pci.c static int nvme_resume(struct device *dev) · line 3275
Quoted 11 lines · Akinobu Mita
drivers/nvme/host/pci.c +11
-0+static void nvme_coredump (struct device *dev) +{ +struct nvme_dev *ndev =dev_get_drvdata (dev); + +mutex_lock (&ndev->shutdown_lock); + +nvme_coredump_prologue (ndev); +nvme_coredump_epilogue (ndev); + +mutex_unlock (&ndev->shutdown_lock); +}
This is a bit of a mine field. The shutdown_lock is held when reclaiming requests that didn't see a response. If you're holding it here and your telemetry log page times out, we're going to deadlock. And since the controller is probably in a buggered state when you try to retrieve one, I would guess an unrecoverable timeout is the most likely outcome.
Re: [PATCH v2 6/7] nvme-pci: add device coredump support
Quoted 5 lines
This is a bit of a mine field. The
shutdown_lockis held when reclaiming requests that didn't see a response. If you're holding it here and your telemetry log page times out, we're going to deadlock. And since the controller is probably in a buggered state when you try to retrieve one, I would guess an unrecoverable timeout is the most likely outcome.
Akinobu,
I actually agree with Keith's one. In my experience, there was always internal error inside device when timeout occurs in nvme driver which means the following command might not be completed due to lack of response from device.
Re: [PATCH v2 6/7] nvme-pci: add device coredump support
2019?5?8?(?) 9:25 Minwoo Im <minwoo.im at samsung.com>:
Quoted 7 lines
Quoted 6 lines
This is a bit of a mine field. The
shutdown_lockis held when reclaiming requests that didn't see a response. If you're holding it here and your telemetry log page times out, we're going to deadlock. And since the controller is probably in a buggered state when you try to retrieve one, I would guess an unrecoverable timeout is the most likely outcome.
Quoted 6 lines
Akinobu,
I actually agree with Keith's one. In my experience, there was always internal error inside device when timeout occurs in nvme driver which means the following command might not be completed due to lack of response from device.
The nvme_coredump() is .coredump() callback of device_driver which is called when anything is written to the /sys/devices/.../coredump. Providing this callback is optional, but simply removing this manual device coredump method is a bit inconvenient.
So instead of directly retrieving the snapshot with the shutdown_lock held in this callback, I'll change this to just scheduling the reset work, and the actual device coredump will be triggered by the same procedure that is implemented in the patch 7/7. Therefore telemetry log is retrieved only when the controller is successfully recovered from the crash.
Re: [PATCH v2 6/7] nvme-pci: add device coredump support
In reply to Akinobu MitaQuoted 2 lines · Akinobu Mita
This enables to capture snapshot of controller information via device coredump machanism.
s/machanism/mechanism/
Quoted 23 lines
The nvme device coredump creates the following coredump files.
- regs: NVMe controller registers (00h to 4Fh)
- sq<qid>: Submission queue
- cq<qid>: Completion queue
- telemetry-ctrl-log: Telemetry controller-initiated log (if available)
- data: EmptyThe reason for an empty 'data' file is to provide a uniform way to notify the device coredump is no longer needed by writing the 'data' file.
Since all existing drivers using the device coredump provide a 'data' file if the nvme device coredump doesn't provide it, the userspace programs need to know which driver provides what coredump file.
Cc Johannes Berg <johannes at sipsolutions.net>Cc Keith Busch <keith.busch at intel.com>Cc Jens Axboe <axboe at fb.com>Cc Christoph Hellwig <hch at lst.de>Cc Sagi Grimberg <sagi at grimberg.me>Cc Minwoo Im <minwoo.im.dev at gmail.com>Signed-off-by Akinobu Mita <akinobu.mita at gmail.com>
Re: [PATCH v2 6/7] nvme-pci: add device coredump support
In reply to Akinobu Mitadrivers/nvme/host/pci.c static int nvme_resume(struct device *dev) · line 3129
Quoted 5 lines · Akinobu Mita
drivers/nvme/host/pci.c +5
-0+ +static int nvme_get_telemetry_log_blocks (struct nvme_ctrl *ctrl,void *buf, +size_t bytes,loff_t offset) +{ +const size_t chunk_size = ctrl->max_hw_sectors * ctrl->page_size;
Just curious if chunk_size is correct since page size and block size can be different.
Quoted 16 lines
drivers/nvme/host/pci.c +16
-0+loff_t pos =0 ; + +while (pos < bytes) { +size_t size =min_t (size_t, bytes - pos, chunk_size); +int ret; + + ret =nvme_get_log (ctrl,NVME_NSID_ALL ,NVME_LOG_TELEMETRY_CTRL , +0 , buf + pos, size, offset + pos); +if (ret) +return ret; + + pos += size; + } + +return 0 ; +}
Keith Busch
maintainerRe: [PATCH v2 6/7] nvme-pci: add device coredump support
Quoted 9 lines · Kenneth
drivers/nvme/host/pci.c static int nvme_resume(struct device *dev) · line 3129
Quoted 5 lines · Akinobu Mita
drivers/nvme/host/pci.c +5
-0+ +static int nvme_get_telemetry_log_blocks (struct nvme_ctrl *ctrl,void *buf, +size_t bytes,loff_t offset) +{ +const size_t chunk_size = ctrl->max_hw_sectors * ctrl->page_size;Just curious if
chunk_sizeis correct since page size and block size can be different.
They're always different. ctrl->page_size is hard-coded to 4k, while sectors are always 512b.
Re: [PATCH v2 6/7] nvme-pci: add device coredump support
2019?5?8?(?) 6:28 Keith Busch <kbusch at kernel.org>:
Quoted 12 lines · Kenneth
Quoted 9 lines · Akinobu Mita
drivers/nvme/host/pci.c static int nvme_resume(struct device *dev) · line 3129
Quoted 5 lines
drivers/nvme/host/pci.c +5
-0+ +static int nvme_get_telemetry_log_blocks (struct nvme_ctrl *ctrl,void *buf, +size_t bytes,loff_t offset) +{ +const size_t chunk_size = ctrl->max_hw_sectors * ctrl->page_size;Just curious if
chunk_sizeis correct since page size and block size can be different.
Quoted 2 lines
They're always different.
ctrl->page_sizeis hard-coded to 4k, while sectors are always 512b.
Oops. I misunderstood how ctrl->max_hw_sectors is initialized from MDTS. Also overflow check was required here for the architectures that use "unsigned int" size_t.
PATCH v2 7/7 nvme-pci: trigger device coredump on command timeout
In reply to Akinobu MitaThis enables the nvme driver to trigger a device coredump when command timeout occurs, and it helps diagnose and debug issues.
This can be tested with fail_io_timeout fault injection.
# echo 1 > /sys/kernel/debug/fail_io_timeout/probability # echo 1 > /sys/kernel/debug/fail_io_timeout/times # echo 1 > /sys/block/nvme0n1/io-timeout-fail # dd if=/dev/nvme0n1 of=/dev/null
---
- Make coredump procedure into two phases (before resetting controller and after resetting as soon as admin queue is available).
drivers/nvme/host/pci.c | 35 ++++++++++++++++++++++------------- 1 file changed, 22 insertions(+), 13 deletions(-)
drivers/nvme/host/pci.c +22 -13
@@ -87,9 +87,12 @@ MODULE_PARM_DESC(poll_queues, "Number of queues to use for polled IO.");struct nvme_dev ;struct nvme_queue ; -static void nvme_dev_disable (struct nvme_dev *dev,bool shutdown); +static void nvme_dev_disable (struct nvme_dev *dev,bool shutdown,bool dump);static bool __nvme_disable_io_queues (struct nvme_dev *dev,u8 opcode); +static int nvme_coredump_prologue (struct nvme_dev *dev); +static void nvme_coredump_epilogue (struct nvme_dev *dev); +/* * Represents an NVM Express device. Each nvme_dev is a PCI function. */
@@ -1289,7 +1292,7 @@ static enum blk_eh_timer_return nvme_timeout(struct request *req, bool reserved)*/ if (nvme_should_reset (dev, csts)) {nvme_warn_reset (dev, csts); -nvme_dev_disable (dev, false); +nvme_dev_disable (dev, false, true);nvme_reset_ctrl (&dev->ctrl);return BLK_EH_DONE ; }
@@ -1316,7 +1319,7 @@ static enum blk_eh_timer_return nvme_timeout(struct request *req, bool reserved)dev_warn_ratelimited (dev->ctrl.device,"I/O %d QID %d timeout, disable controller\n" , req->tag, nvmeq->qid); -nvme_dev_disable (dev, false); +nvme_dev_disable (dev, false, true);nvme_req (req)->flags |=NVME_REQ_CANCELLED ;return BLK_EH_DONE ;default :
@@ -1332,7 +1335,7 @@ static enum blk_eh_timer_return nvme_timeout(struct request *req, bool reserved)dev_warn (dev->ctrl.device,"I/O %d QID %d timeout, reset controller\n" , req->tag, nvmeq->qid); -nvme_dev_disable (dev, false); +nvme_dev_disable (dev, false, true);nvme_reset_ctrl (&dev->ctrl);nvme_req (req)->flags |=NVME_REQ_CANCELLED ;
@@ -2399,7 +2402,7 @@ static void nvme_pci_disable(struct nvme_dev *dev) } } -static void nvme_dev_disable (struct nvme_dev *dev,bool shutdown) +static void nvme_dev_disable (struct nvme_dev *dev,bool shutdown,bool dump) {bool dead = true;struct pci_dev *pdev =to_pci_dev (dev->dev);
@@ -2424,6 +2427,9 @@ static void nvme_dev_disable(struct nvme_dev *dev, bool shutdown)nvme_wait_freeze_timeout (&dev->ctrl,NVME_IO_TIMEOUT ); } +if (dump) +nvme_coredump_prologue (dev); +nvme_stop_queues (&dev->ctrl);if (!dead && dev->ctrl.queue_count >0 ) {
@@ -2491,7 +2497,7 @@ static void nvme_remove_dead_ctrl(struct nvme_dev *dev, int status)dev_warn (dev->ctrl.device,"Removing after probe failure status: %d\n" , status);nvme_get_ctrl (&dev->ctrl); -nvme_dev_disable (dev, false); +nvme_dev_disable (dev, false, false);nvme_kill_queues (&dev->ctrl);if (!queue_work (nvme_wq, &dev->remove_work))nvme_put_ctrl (&dev->ctrl);
@@ -2513,7 +2519,7 @@ static void nvme_reset_work(struct work_struct *work)* moving on. */ if (dev->ctrl.ctrl_config &NVME_CC_ENABLE ) -nvme_dev_disable (dev, false); +nvme_dev_disable (dev, false, false);mutex_lock (&dev->shutdown_lock); result =nvme_pci_enable (dev);
@@ -2550,6 +2556,8 @@ static void nvme_reset_work(struct work_struct *work)if (result) goto out; +nvme_coredump_epilogue (dev); +if (dev->ctrl.oacs &NVME_CTRL_OACS_SEC_SUPP ) {if (!dev->ctrl.opal_dev) dev->ctrl.opal_dev =
@@ -2612,6 +2620,7 @@ static void nvme_reset_work(struct work_struct *work) out_unlock:mutex_unlock (&dev->shutdown_lock); out: +nvme_coredump_epilogue (dev);nvme_remove_dead_ctrl (dev, result); }
@@ -2802,7 +2811,7 @@ static int nvme_probe(struct pci_dev *pdev, const struct pci_device_id *id)static void nvme_reset_prepare (struct pci_dev *pdev) {struct nvme_dev *dev =pci_get_drvdata (pdev); -nvme_dev_disable (dev, false); +nvme_dev_disable (dev, false, false); }static void nvme_reset_done (struct pci_dev *pdev)
@@ -2814,7 +2823,7 @@ static void nvme_reset_done(struct pci_dev *pdev)static void nvme_shutdown (struct pci_dev *pdev) {struct nvme_dev *dev =pci_get_drvdata (pdev); -nvme_dev_disable (dev, true); +nvme_dev_disable (dev, true, false); }/*
@@ -2831,14 +2840,14 @@ static void nvme_remove(struct pci_dev *pdev)if (!pci_device_is_present (pdev)) {nvme_change_ctrl_state (&dev->ctrl,NVME_CTRL_DEAD ); -nvme_dev_disable (dev, true); +nvme_dev_disable (dev, true, false);nvme_dev_remove_admin (dev); }flush_work (&dev->ctrl.reset_work);nvme_stop_ctrl (&dev->ctrl);nvme_remove_namespaces (&dev->ctrl); -nvme_dev_disable (dev, true); +nvme_dev_disable (dev, true, false);nvme_release_cmb (dev);nvme_free_host_mem (dev);nvme_dev_remove_admin (dev);
@@ -2855,7 +2864,7 @@ static int nvme_suspend(struct device *dev)struct pci_dev *pdev =to_pci_dev (dev);struct nvme_dev *ndev =pci_get_drvdata (pdev); -nvme_dev_disable (ndev, true); +nvme_dev_disable (ndev, true, false);return 0 ; }
@@ -3307,7 +3316,7 @@ static pci_ers_result_t nvme_error_detected(struct pci_dev *pdev,case pci_channel_io_frozen:dev_warn (dev->ctrl.device,"frozen state error detected, reset controller\n" ); -nvme_dev_disable (dev, false); +nvme_dev_disable (dev, false, false);return PCI_ERS_RESULT_NEED_RESET ;case pci_channel_io_perm_failure: dev_warn(dev->ctrl.device,
Signature · 1 line
2.7.4
Quoted 2 lines · Akinobu Mita
This enables to capture snapshot of controller information via device coredump machanism, and it helps diagnose and debug issues.
s/machanism/mechanism/
Quoted 9 lines
The nvme device coredump is triggered when command timeout occurs, and creates the following coredump files.
- regs: NVMe controller registers (00h to 4Fh)
- sq<qid>: Submission queue
- cq<qid>: Completion queue
- telemetry-ctrl-log: Telemetry controller-initiated log (if available)
- data: Empty