Make NVME shutdown async
Subsystems (4)
This is a rebased update and resend of a patchset originally written by Tanjore Suresh <[email protected]> to make shutdown of nvme devices asynchronous. Minor changes made to use an enum shutdown_type instead of an integer flag.
Currently the Linux nvme driver shutdown code steps through each connected drive, sets the NVME_CC_SHN_NORMAL (normal shutdown) flag and then polls the given drive waiting for the response NVME_CSTS_SHST_CMPLT flag (shutdown complete).
Each drive is taking around 13 seconds to respond to this.
The customer has 20+ drives on the box so this time adds up on shutdown when the nvme driver is being shut down.
This patchset changes shutdown to proceed in parallel, so the NVME_CC_SHN_NORMAL (normal shutdown) flag is sent to all drives first, and then it polls waiting for the NVME_CSTS_SHST_CMPLT flag (shutdown complete) for all drives.
In the specific customer case it reduces the NVME shutdown time from over 300 seconds to around 15 seconds.
Thanks for your consideration,
Jeremy Allison.
CIQ / Samba Team.
PATCH 1/3 driver core: Support asynchronous driver shutdown
From: Tanjore Suresh <[email protected]>
This changes the bus driver interface with additional entry points to enable devices to implement asynchronous shutdown. The existing synchronous interface to shutdown is unmodified and retained for backward compatibility.
This changes the common device shutdown code to enable devices to participate in asynchronous shutdown implementation.
--- drivers/base/core.c | 39 +++++++++++++++++++++++++++++++++++++- include/linux/device/bus.h | 10 ++++++++++ 2 files changed, 48 insertions(+), 1 deletion(-)
drivers/base/core.c +38 -1
@@ -4725,6 +4725,7 @@ EXPORT_SYMBOL_GPL(device_change_owner);void device_shutdown (void ) {struct device *dev, *parent; +LIST_HEAD (async_shutdown_list);wait_for_device_probe ();device_block_probing ();
@@ -4769,7 +4770,14 @@ void device_shutdown(void)dev_info (dev,"shutdown_pre\n" ); dev->class->shutdown_pre (dev); } -if (dev->bus && dev->bus->shutdown) { + +if (dev->bus && dev->bus->shutdown_pre) { +if (initcall_debug) +dev_info (dev,"shutdown_pre\n" ); + dev->bus->shutdown_pre (dev); +list_add (&dev->kobj.entry, + &async_shutdown_list); + }else if (dev->bus && dev->bus->shutdown) {if (initcall_debug)dev_info (dev,"shutdown\n" ); dev->bus->shutdown (dev);
@@ -4789,6 +4797,35 @@ void device_shutdown(void)spin_lock (&devices_kset->list_lock); }spin_unlock (&devices_kset->list_lock); + +/* +* Second pass spin for only devices, that have configured +* Asynchronous shutdown. +*/ +while (!list_empty (&async_shutdown_list)) { + dev = list_entry(async_shutdown_list.next, struct device, + kobj.entry); + parent =get_device (dev->parent); +get_device (dev); +/* +* Make sure the device is off the list +*/ +list_del_init (&dev->kobj.entry); +if (parent) +device_lock (parent); +device_lock (dev); +if (dev->bus && dev->bus->shutdown_post) { +if (initcall_debug) +dev_info (dev, +"shutdown_post called\n" ); + dev->bus->shutdown_post (dev); + } +device_unlock (dev); +if (parent) +device_unlock (parent); +put_device (dev); +put_device (parent); + } }/*
include/linux/device/bus.h +10 -0
@@ -48,6 +48,14 @@ struct fwnode_handle;* will never get called until they do. * @remove: Called when a device removed from this bus. * @shutdown: Called at shut-down time to quiesce the device. +* @shutdown_pre: Called at the shutdown-time to start the shutdown +* process on the device. This entry point will be called +* only when the bus driver has indicated it would like +* to participate in asynchronous shutdown completion. +* @shutdown_post: Called at shutdown-time to complete the shutdown +* process of the device. This entry point will be called +* only when the bus drive has indicated it would like to +* participate in the asynchronous shutdown completion. * * @online: Called to put the device back online (after offlining it). * @offline: Called to put the device offline for hot-removal. May fail.
@@ -90,6 +98,8 @@ struct bus_type {void (*sync_state)(struct device *dev);void (*remove)(struct device *dev);void (*shutdown)(struct device *dev); +void (*shutdown_pre)(struct device *dev); +void (*shutdown_post)(struct device *dev);int (*online)(struct device *dev);int (*offline)(struct device *dev);
Signature · 1 line
2.39.3
Sagi Grimberg
maintainerRe: [PATCH 1/3] driver core: Support asynchronous driver shutdown
Quoted 29 lines
From: Tanjore Suresh <[email protected]>
This changes the bus driver interface with additional entry points to enable devices to implement asynchronous shutdown. The existing synchronous interface to shutdown is unmodified and retained for backward compatibility.
This changes the common device shutdown code to enable devices to participate in asynchronous shutdown implementation.
Signed-off-by Tanjore Suresh <[email protected]>--- drivers/base/core.c | 39 +++++++++++++++++++++++++++++++++++++- include/linux/device/bus.h | 10 ++++++++++ 2 files changed, 48 insertions(+), 1 deletion(-)drivers/base/core.c +1
-0@@ -4725,6 +4725,7 @@ EXPORT_SYMBOL_GPL(device_change_owner);void device_shutdown (void ) {struct device *dev, *parent; +LIST_HEAD (async_shutdown_list);wait_for_device_probe ();device_block_probing ();
drivers/base/core.c void device_shutdown(void) · line 4774
Quoted 6 lines
drivers/base/core.c +2
-1@@ -4769,7 +4770,14 @@ void device_shutdown(void)dev_info (dev,"shutdown_pre\n" ); dev->class->shutdown_pre (dev); } -if (dev->bus && dev->bus->shutdown) { + +if (dev->bus && dev->bus->shutdown_pre) {
I'm assuming that there is no shutdown_pre without a shutdown_post paired with it, so I think the code should verify that.
Quoted 73 lines
drivers/base/core.c +35
-0+if (initcall_debug) +dev_info (dev,"shutdown_pre\n" ); + dev->bus->shutdown_pre (dev); +list_add (&dev->kobj.entry, + &async_shutdown_list); + }else if (dev->bus && dev->bus->shutdown) {if (initcall_debug)dev_info (dev,"shutdown\n" ); dev->bus->shutdown (dev);@@ -4789,6 +4797,35 @@ void device_shutdown(void)spin_lock (&devices_kset->list_lock); }spin_unlock (&devices_kset->list_lock); + +/* +* Second pass spin for only devices, that have configured +* Asynchronous shutdown. +*/ +while (!list_empty (&async_shutdown_list)) { + dev = list_entry(async_shutdown_list.next, struct device, + kobj.entry); + parent =get_device (dev->parent); +get_device (dev); +/* +* Make sure the device is off the list +*/ +list_del_init (&dev->kobj.entry); +if (parent) +device_lock (parent); +device_lock (dev); +if (dev->bus && dev->bus->shutdown_post) { +if (initcall_debug) +dev_info (dev, +"shutdown_post called\n" ); + dev->bus->shutdown_post (dev); + } +device_unlock (dev); +if (parent) +device_unlock (parent); +put_device (dev); +put_device (parent); + } }/* include/linux/device/bus.h +10
-0@@ -48,6 +48,14 @@ struct fwnode_handle;* will never get called until they do. * @remove: Called when a device removed from this bus. * @shutdown: Called at shut-down time to quiesce the device. +* @shutdown_pre: Called at the shutdown-time to start the shutdown +* process on the device. This entry point will be called +* only when the bus driver has indicated it would like +* to participate in asynchronous shutdown completion. +* @shutdown_post: Called at shutdown-time to complete the shutdown +* process of the device. This entry point will be called +* only when the bus drive has indicated it would like to +* participate in the asynchronous shutdown completion. * * @online: Called to put the device back online (after offlining it). * @offline: Called to put the device offline for hot-removal. May fail. @@ -90,6 +98,8 @@ struct bus_type {void (*sync_state)(struct device *dev);void (*remove)(struct device *dev);void (*shutdown)(struct device *dev); +void (*shutdown_pre)(struct device *dev); +void (*shutdown_post)(struct device *dev);int (*online)(struct device *dev);int (*offline)(struct device *dev);
Re: [PATCH 1/3] driver core: Support asynchronous driver shutdown
Good catch. Thanks. I'll fix that and re-submit.
Quoted 113 lines · Sagi Grimberg
Quoted 30 lines
From: Tanjore Suresh <[email protected]>
This changes the bus driver interface with additional entry points to enable devices to implement asynchronous shutdown. The existing synchronous interface to shutdown is unmodified and retained for backward compatibility.
This changes the common device shutdown code to enable devices to participate in asynchronous shutdown implementation.
Signed-off-by Tanjore Suresh <[email protected]>--- drivers/base/core.c | 39 +++++++++++++++++++++++++++++++++++++- include/linux/device/bus.h | 10 ++++++++++ 2 files changed, 48 insertions(+), 1 deletion(-)drivers/base/core.c +1
-0@@ -4725,6 +4725,7 @@ EXPORT_SYMBOL_GPL(device_change_owner);void device_shutdown (void ) {struct device *dev, *parent; +LIST_HEAD (async_shutdown_list);wait_for_device_probe ();device_block_probing ();drivers/base/core.c void device_shutdown(void) · line 4774
Quoted 6 lines
drivers/base/core.c +2
-1@@ -4769,7 +4770,14 @@ void device_shutdown(void)dev_info (dev,"shutdown_pre\n" ); dev->class->shutdown_pre (dev); } -if (dev->bus && dev->bus->shutdown) { + +if (dev->bus && dev->bus->shutdown_pre) {I'm assuming that there is no
shutdown_prewithout ashutdown_postpaired with it, so I think the code should verify that.Quoted 73 lines
drivers/base/core.c +35
-0+if (initcall_debug) +dev_info (dev,"shutdown_pre\n" ); + dev->bus->shutdown_pre (dev); +list_add (&dev->kobj.entry, + &async_shutdown_list); + }else if (dev->bus && dev->bus->shutdown) {if (initcall_debug)dev_info (dev,"shutdown\n" ); dev->bus->shutdown (dev);@@ -4789,6 +4797,35 @@ void device_shutdown(void)spin_lock (&devices_kset->list_lock); }spin_unlock (&devices_kset->list_lock); + +/* +* Second pass spin for only devices, that have configured +* Asynchronous shutdown. +*/ +while (!list_empty (&async_shutdown_list)) { + dev = list_entry(async_shutdown_list.next, struct device, + kobj.entry); + parent =get_device (dev->parent); +get_device (dev); +/* +* Make sure the device is off the list +*/ +list_del_init (&dev->kobj.entry); +if (parent) +device_lock (parent); +device_lock (dev); +if (dev->bus && dev->bus->shutdown_post) { +if (initcall_debug) +dev_info (dev, +"shutdown_post called\n" ); + dev->bus->shutdown_post (dev); + } +device_unlock (dev); +if (parent) +device_unlock (parent); +put_device (dev); +put_device (parent); + } }/* include/linux/device/bus.h +10
-0@@ -48,6 +48,14 @@ struct fwnode_handle;* will never get called until they do. * @remove: Called when a device removed from this bus. * @shutdown: Called at shut-down time to quiesce the device. +* @shutdown_pre: Called at the shutdown-time to start the shutdown +* process on the device. This entry point will be called +* only when the bus driver has indicated it would like +* to participate in asynchronous shutdown completion. +* @shutdown_post: Called at shutdown-time to complete the shutdown +* process of the device. This entry point will be called +* only when the bus drive has indicated it would like to +* participate in the asynchronous shutdown completion. * * @online: Called to put the device back online (after offlining it). * @offline: Called to put the device offline for hot-removal. May fail. @@ -90,6 +98,8 @@ struct bus_type {void (*sync_state)(struct device *dev);void (*remove)(struct device *dev);void (*shutdown)(struct device *dev); +void (*shutdown_pre)(struct device *dev); +void (*shutdown_post)(struct device *dev);int (*online)(struct device *dev);int (*offline)(struct device *dev);
Re: [PATCH 1/3] driver core: Support asynchronous driver shutdown
In reply to Jeremy AllisonQuoted 7 lines · Jeremy Allison
From: Tanjore Suresh <[email protected]>
This changes the bus driver interface with additional entry points to enable devices to implement asynchronous shutdown. The existing synchronous interface to shutdown is unmodified and retained for backward compatibility.
Quoted 8 lines
This changes the common device shutdown code to enable devices to participate in asynchronous shutdown implementation.
Signed-off-by Tanjore Suresh <[email protected]>--- drivers/base/core.c | 39 +++++++++++++++++++++++++++++++++++++- include/linux/device/bus.h | 10 ++++++++++ 2 files changed, 48 insertions(+), 1 deletion(-)
From the MAINTAINERS file:
DRIVER CORE, KOBJECTS, DEBUGFS AND SYSFS
M: Greg Kroah-Hartman <[email protected]> R: "Rafael J. Wysocki" <[email protected]> S: Supported T: git git://git.kernel.org/pub/scm/linux/kernel/git/gregkh/driver-core.git F: Documentation/core-api/kobject.rst F: drivers/base/ F: fs/debugfs/ F: fs/sysfs/ F: include/linux/debugfs.h F: include/linux/fwnode.h F: include/linux/kobj* F: include/linux/property.h F: lib/kobj*
Please Cc the maintainers of the modified files when posting kernel patches.
Thanks,
Bart.
PATCH 2/3 PCI: Support asynchronous shutdown
In reply to Jeremy AllisonFrom: Tanjore Suresh <[email protected]>
Enhances the base PCI driver to add support for asynchronous shutdown.
Assume a device takes n secs to shutdown. If a machine has been populated with M such devices, the total time spent in shutting down all the devices will be M * n secs, if the shutdown is done synchronously. For example, if NVMe PCI Controllers take 5 secs to shutdown and if there are 16 such NVMe controllers in a system, system will spend a total of 80 secs to shutdown all
NVMe devices in that system.
In order to speed up the shutdown time, asynchronous interface to shutdown has been implemented. This will significantly reduce the machine reboot time.
--- drivers/pci/pci-driver.c | 17 ++++++++++++++--- include/linux/pci.h | 2 ++ 2 files changed, 16 insertions(+), 3 deletions(-)
drivers/pci/pci-driver.c +14 -3
@@ -502,14 +502,16 @@ static void pci_device_remove(struct device *dev)pci_dev_put (pci_dev); } -static void pci_device_shutdown (struct device *dev) +static void pci_device_shutdown_pre (struct device *dev) {struct pci_dev *pci_dev =to_pci_dev (dev);struct pci_driver *drv = pci_dev->driver;pm_runtime_resume (dev); -if (drv && drv->shutdown) +if (drv && drv->shutdown_pre) + drv->shutdown_pre (pci_dev); +else if (drv && drv->shutdown) drv->shutdown (pci_dev);/*
@@ -547,6 +549,14 @@ static int pci_restore_standard_config(struct pci_dev *pci_dev) }#endif /* CONFIG_PM_SLEEP */ +static void pci_device_shutdown_post (struct device *dev) +{ +struct pci_dev *pci_dev =to_pci_dev (dev); +struct pci_driver *drv = pci_dev->driver; + +if (drv && drv->shutdown_post) + drv->shutdown_post (pci_dev); +}#ifdef CONFIG_PM /* Auxiliary functions used for system resume and run-time resume */
@@ -1681,7 +1691,8 @@ struct bus_type pci_bus_type = { .uevent = pci_uevent, .probe = pci_device_probe, .remove = pci_device_remove, - .shutdown = pci_device_shutdown, + .shutdown_pre = pci_device_shutdown_pre, + .shutdown_post = pci_device_shutdown_post, .dev_groups = pci_dev_groups, .bus_groups = pci_bus_groups, .drv_groups = pci_drv_groups,
include/linux/pci.h +2 -0
@@ -948,6 +948,8 @@ struct pci_driver {int (*suspend)(struct pci_dev *dev,pm_message_t state);/* Device suspended */ int (*resume)(struct pci_dev *dev);/* Device woken up */ void (*shutdown)(struct pci_dev *dev); +void (*shutdown_pre)(struct pci_dev *dev); +void (*shutdown_post)(struct pci_dev *dev);int (*sriov_configure)(struct pci_dev *dev,int num_vfs);/* On PF */ int (*sriov_set_msix_vec_count)(struct pci_dev *vf,int msix_vec_count);/* On PF */ u32 (*sriov_get_vf_total_msix)(struct pci_dev *pf);
Signature · 1 line
2.39.3
Re: [PATCH 2/3] PCI: Support asynchronous shutdown
Quoted 17 lines · Jeremy Allison
From: Tanjore Suresh <[email protected]>
Enhances the base PCI driver to add support for asynchronous shutdown.
Assume a device takes n secs to shutdown. If a machine has been populated with M such devices, the total time spent in shutting down all the devices will be M * n secs, if the shutdown is done synchronously. For example, if NVMe PCI Controllers take 5 secs to shutdown and if there are 16 such NVMe controllers in a system, system will spend a total of 80 secs to shutdown all
NVMe devices in that system.In order to speed up the shutdown time, asynchronous interface to shutdown has been implemented. This will significantly reduce the machine reboot time.
Quoted 5 lines
Signed-off-by Tanjore Suresh <[email protected]>--- drivers/pci/pci-driver.c | 17 ++++++++++++++--- include/linux/pci.h | 2 ++ 2 files changed, 16 insertions(+), 3 deletions(-)
Please Cc the PCI maintainer (Bjorn Helgaas) when reposting this patch series.
Thanks,
Bart.
PATCH 3/3 nvme: Add async shutdown support
In reply to Jeremy AllisonFrom: Tanjore Suresh <[email protected]>
This works with the asynchronous shutdown mechanism setup for the PCI drivers and participates to provide both pre and post shutdown routines at pci_driver structure level.
The shutdown_pre routine starts the shutdown and does not wait for the shutdown to complete. The shutdown_post routine waits for the shutdown to complete on individual controllers that this driver instance controls. This mechanism optimizes to speed up the shutdown in a system which host many controllers.
--- drivers/nvme/host/core.c | 31 +++++++++++++++-- drivers/nvme/host/nvme.h | 9 ++++- drivers/nvme/host/pci.c | 68 ++++++++++++++++++++++++-------------- drivers/nvme/host/rdma.c | 3 +- drivers/nvme/host/tcp.c | 2 +- drivers/nvme/target/loop.c | 2 +- 6 files changed, 84 insertions(+), 31 deletions(-)
drivers/nvme/host/core.c +28 -3
@@ -2201,12 +2201,12 @@ static int nvme_wait_ready(struct nvme_ctrl *ctrl, u32 mask, u32 val,return ret; } -int nvme_disable_ctrl (struct nvme_ctrl *ctrl,bool shutdown) +int nvme_disable_ctrl (struct nvme_ctrl *ctrl,enum shutdown_type shutdown_type) {int ret; ctrl->ctrl_config &= ~NVME_CC_SHN_MASK ; -if (shutdown) +if (shutdown_type !=DO_NOT_SHUTDOWN ) ctrl->ctrl_config |=NVME_CC_SHN_NORMAL ;else ctrl->ctrl_config &= ~NVME_CC_ENABLE ;
@@ -2215,10 +2215,24 @@ int nvme_disable_ctrl(struct nvme_ctrl *ctrl, bool shutdown)if (ret)return ret; -if (shutdown) { +switch (shutdown_type) { +case SHUTDOWN_TYPE_ASYNC : +/* +* nvme_wait_for_shutdown_cmpl() will read the reply for this. +*/ +return ret; +case SHUTDOWN_TYPE_SYNC : +/* +* Spin on the read of the control register. +*/ return nvme_wait_ready (ctrl,NVME_CSTS_SHST_MASK ,NVME_CSTS_SHST_CMPLT , ctrl->shutdown_timeout,"shutdown" ); +case DO_NOT_SHUTDOWN : +/* +* Doing a reset here. Handle below. +*/ +break ; }if (ctrl->quirks &NVME_QUIRK_DELAY_BEFORE_CHK_RDY )msleep (NVME_QUIRK_DELAY_AMOUNT );
@@ -2227,6 +2241,17 @@ int nvme_disable_ctrl(struct nvme_ctrl *ctrl, bool shutdown) }EXPORT_SYMBOL_GPL (nvme_disable_ctrl); +int nvme_wait_for_shutdown_cmpl (struct nvme_ctrl *ctrl) +{ + ctrl->ctrl_config &= ~NVME_CC_SHN_MASK ; + ctrl->ctrl_config |=NVME_CC_SHN_NORMAL ; + +return nvme_wait_ready (ctrl,NVME_CSTS_SHST_MASK , +NVME_CSTS_SHST_CMPLT , + ctrl->shutdown_timeout,"shutdown" ); +} +EXPORT_SYMBOL_GPL (nvme_wait_for_shutdown_cmpl); +int nvme_enable_ctrl (struct nvme_ctrl *ctrl) {unsigned dev_page_min;
drivers/nvme/host/nvme.h +8 -1
@@ -186,6 +186,12 @@ enum { NVME_MPATH_IO_STATS = (1 << 2), }; +enum shutdown_type { +DO_NOT_SHUTDOWN =0 , +SHUTDOWN_TYPE_SYNC =1 , +SHUTDOWN_TYPE_ASYNC =2 , +}; +static inline struct nvme_request *nvme_req (struct request *req) {return blk_mq_rq_to_pdu (req);
@@ -746,7 +752,8 @@ void nvme_cancel_tagset(struct nvme_ctrl *ctrl);void nvme_cancel_admin_tagset (struct nvme_ctrl *ctrl);bool nvme_change_ctrl_state (struct nvme_ctrl *ctrl,enum nvme_ctrl_state new_state); -int nvme_disable_ctrl (struct nvme_ctrl *ctrl,bool shutdown); +int nvme_disable_ctrl (struct nvme_ctrl *ctrl,enum shutdown_type shutdown_type); +int nvme_wait_for_shutdown_cmpl (struct nvme_ctrl *ctrl);int nvme_enable_ctrl (struct nvme_ctrl *ctrl);int nvme_init_ctrl (struct nvme_ctrl *ctrl,struct device *dev,const struct nvme_ctrl_ops *ops,unsigned long quirks);
drivers/nvme/host/pci.c +44 -24
201 lines not on this page · raw message
drivers/nvme/host/rdma.c +2 -1
@@ -2136,7 +2136,8 @@ static void nvme_rdma_shutdown_ctrl(struct nvme_rdma_ctrl *ctrl, bool shutdown) {nvme_rdma_teardown_io_queues (ctrl, shutdown);nvme_quiesce_admin_queue (&ctrl->ctrl); -nvme_disable_ctrl (&ctrl->ctrl, shutdown); +nvme_disable_ctrl (&ctrl->ctrl, + shutdown ?SHUTDOWN_TYPE_SYNC :DO_NOT_SHUTDOWN );nvme_rdma_teardown_admin_queue (ctrl, shutdown); }
drivers/nvme/host/tcp.c +1 -1
@@ -2292,7 +2292,7 @@ static void nvme_tcp_teardown_ctrl(struct nvme_ctrl *ctrl, bool shutdown) {nvme_tcp_teardown_io_queues (ctrl, shutdown);nvme_quiesce_admin_queue (ctrl); -nvme_disable_ctrl (ctrl, shutdown); +nvme_disable_ctrl (ctrl, shutdown ?SHUTDOWN_TYPE_SYNC :DO_NOT_SHUTDOWN );nvme_tcp_teardown_admin_queue (ctrl, shutdown); }
drivers/nvme/target/loop.c +1 -1
@@ -401,7 +401,7 @@ static void nvme_loop_shutdown_ctrl(struct nvme_loop_ctrl *ctrl)nvme_quiesce_admin_queue (&ctrl->ctrl);if (ctrl->ctrl.state ==NVME_CTRL_LIVE ) -nvme_disable_ctrl (&ctrl->ctrl, true); +nvme_disable_ctrl (&ctrl->ctrl,SHUTDOWN_TYPE_SYNC );nvme_cancel_admin_tagset (&ctrl->ctrl);nvme_loop_destroy_admin_queue (ctrl);
Signature · 1 line
2.39.3