linux-nvme 3.20 plans
Subsystems (2)
PATCH 1/1 linux-nvme 3.20 plans
Hi Willy,
This cover letter is not necessarily for the following patch. I just want to know what your plans are for 3.20. We'd be remiss to miss another merge window.
Keith Busch (1):
NVMe: Async probe
drivers/block/nvme-core.c | 55 ++++++++++++++++++++++++--------------------- include/linux/nvme.h | 2 ++ 2 files changed, 32 insertions(+), 25 deletions(-)
Signature · 1 line
1.7.10.4
PATCH 1/1 NVMe: Async probe
This performs the longest parts of nvme device probe asynchronously. This speeds up probe significantly when multiple devices are in use.
Just to drive how important this is for many distros, 'systemd' sends a fatal signal to the modprobe routine during boot if you have a lot of NVMe drives; it might take a while to initailize them and exceed some arbitrary timeout that no one knows how to change. The result is only a subset of your drives are discovered after boot since nvme probe bails on devices when fatal_signal_pending is set.
--- drivers/block/nvme-core.c | 55 ++++++++++++++++++++++++--------------------- include/linux/nvme.h | 2 ++ 2 files changed, 32 insertions(+), 25 deletions(-)
drivers/block/nvme-core.c +30 -25
@@ -2799,6 +2799,8 @@ static void nvme_reset_workfn(struct work_struct *work) dev->reset_workfn (work); } +static void nvme_async_probe (void *data,async_cookie_t cookie); +static int nvme_probe (struct pci_dev *pdev,const struct pci_device_id *id) {int node, result = -ENOMEM ;
@@ -2833,48 +2835,50 @@ static int nvme_probe(struct pci_dev *pdev, const struct pci_device_id *id)if (result) goto release; + dev->probe_cookie =async_schedule (nvme_async_probe, dev); +return result; + + release: +nvme_release_instance (dev); + put_pci: +pci_dev_put (dev->pci_dev); + free: +kfree (dev->queues); +kfree (dev->entry); +kfree (dev); +return result; +} + +static void nvme_async_probe (void *data,async_cookie_t cookie) +{ +struct nvme_dev *dev = data; +int result; +kref_init (&dev->kref); result =nvme_dev_start (dev);if (result) - goto release_pools; + goto reset;if (dev->online_queues >1 ) result =nvme_dev_add (dev);if (result) - goto shutdown; + goto reset;scnprintf (dev->name,sizeof (dev->name),"nvme%d" , dev->instance); dev->miscdev.minor =MISC_DYNAMIC_MINOR ; - dev->miscdev.parent = &pdev->dev; + dev->miscdev.parent = &dev->pci_dev->dev; dev->miscdev.name = dev->name; dev->miscdev.fops = &nvme_dev_fops; result =misc_register (&dev->miscdev);if (result) - goto remove; + goto reset;nvme_set_irq_hints (dev); - dev->initialized =1 ; -return 0 ; - - remove: -nvme_dev_remove (dev); -nvme_dev_remove_admin (dev); -nvme_free_namespaces (dev); - shutdown: -nvme_dev_shutdown (dev); - release_pools: -nvme_free_queues (dev,0 ); -nvme_release_prp_pools (dev); - release: -nvme_release_instance (dev); - put_pci: -pci_dev_put (dev->pci_dev); - free: -kfree (dev->queues); -kfree (dev->entry); -kfree (dev); -return result; +return ; + reset: + dev->reset_workfn = nvme_reset_failed_dev; +queue_work (nvme_workq, &dev->reset_work); }static void nvme_reset_notify (struct pci_dev *pdev,bool prepare)
@@ -2902,6 +2906,7 @@ static void nvme_remove(struct pci_dev *pdev)spin_unlock (&dev_list_lock);pci_set_drvdata (pdev,NULL ); +async_synchronize_cookie (dev->probe_cookie);flush_work (&dev->reset_work);misc_deregister (&dev->miscdev);nvme_dev_shutdown (dev);
include/linux/nvme.h +2 -0
@@ -16,6 +16,7 @@#define _LINUX_NVME_H#include <uapi/linux/nvme.h> +#include <linux/async.h> #include <linux/pci.h> #include <linux/miscdevice.h> #include <linux/kref.h>
@@ -94,6 +95,7 @@ struct nvme_dev {struct miscdevice miscdev;work_func_t reset_workfn;struct work_struct reset_work; +async_cookie_t probe_cookie;char name[12 ];char serial[20 ];char model[40 ];
Signature · 1 line
1.7.10.4
PATCH 1/1 NVMe: Async probe
Quoted 10 lines
-----Original Message-----
From: Linux-nvme [mailto:linux-nvme-bounces at lists.infradead.org] On Behalf Of Keith Busch
Sent: Thursday, February 12, 2015 6:41 AM
To: linux-nvme at lists.infradead.org; willy at linux.intel.com
Cc: Keith Busch
Subject: [PATCH] NVMe: Async probe
Quoted 4 lines
This performs the longest parts of nvme device probe asynchronously. This speeds up probe significantly when multiple devices are in use.
Just to drive how important this is for many distros, 'systemd' sends a
fatal signal
Quoted 1 line
to the modprobe routine during boot if you have a lot of NVMe drives; it
might
Quoted 1 line
take a while to initailize them and exceed some arbitrary timeout that no
one
Quoted 1 line
knows how to change. The result is only a subset of your drives are
discovered
Quoted 1 line
after boot since nvme probe bails on devices when
fatal_signal_pendingis
set.
Quoted 4 lines
Signed-off-by Keith Busch <keith.busch at intel.com>--- drivers/block/nvme-core.c | 55
++++++++++++++++++++++++---------------------
Quoted 8 lines
include/linux/nvme.h | 2 ++ 2 files changed, 32 insertions(+), 25 deletions(-)drivers/block/nvme-core.c index renamed from drivers/block/nvme-core.c +0
-0b3cb67d..89cb4c5 100644
drivers/block/nvme-core.c static void nvme_reset_workfn(struct work_struct · line 2805
Quoted 7 lines
drivers/block/nvme-core.c +2
-0@@ -2799,6 +2799,8 @@ static void nvme_reset_workfn(struct work_struct *work) dev->reset_workfn (work); } +static void nvme_async_probe (void *data,async_cookie_t cookie); + static int nvme_probe(struct pci_dev *pdev, const struct pci_device_id
*id) {
Quoted 67 lines
drivers/block/nvme-core.c +22
-22int node, result = -ENOMEM ;@@ -2833,48 +2835,50 @@ static int nvme_probe(struct pci_dev *pdev, const struct pci_device_id *id)if (result) goto release; + dev->probe_cookie =async_schedule (nvme_async_probe, dev); +return result; + + release: +nvme_release_instance (dev); + put_pci: +pci_dev_put (dev->pci_dev); + free: +kfree (dev->queues); +kfree (dev->entry); +kfree (dev); +return result; +} + +static void nvme_async_probe (void *data,async_cookie_t cookie) { +struct nvme_dev *dev = data; +int result; +kref_init (&dev->kref); result =nvme_dev_start (dev);if (result) - goto release_pools; + goto reset;if (dev->online_queues >1 ) result =nvme_dev_add (dev);if (result) - goto shutdown; + goto reset;scnprintf (dev->name,sizeof (dev->name),"nvme%d" , dev->instance); dev->miscdev.minor =MISC_DYNAMIC_MINOR ; - dev->miscdev.parent = &pdev->dev; + dev->miscdev.parent = &dev->pci_dev->dev; dev->miscdev.name = dev->name; dev->miscdev.fops = &nvme_dev_fops; result =misc_register (&dev->miscdev);if (result) - goto remove; + goto reset;nvme_set_irq_hints (dev); - dev->initialized =1 ; -return 0 ; - - remove: -nvme_dev_remove (dev); -nvme_dev_remove_admin (dev); -nvme_free_namespaces (dev); - shutdown: -nvme_dev_shutdown (dev); - release_pools: -nvme_free_queues (dev,0 ); -nvme_release_prp_pools (dev); - release: -nvme_release_instance (dev); - put_pci: -pci_dev_put (dev->pci_dev); - free: -kfree (dev->queues);
drivers/block/nvme-core.c static int nvme_probe(struct pci_dev *pdev, const
Quoted 10 lines
drivers/block/nvme-core.c +4
-3-kfree (dev->entry); -kfree (dev); -return result; +return ; + reset: + dev->reset_workfn = nvme_reset_failed_dev; +queue_work (nvme_workq, &dev->reset_work); } static void nvme_reset_notify(struct pci_dev *pdev, bool prepare) @@
-2902,6
drivers/block/nvme-core.c static void nvme_remove(struct pci_dev *pdev) · line 2909
Quoted 5 lines
+2906,7 @@ static void
nvme_remove(struct pci_dev *pdev)drivers/block/nvme-core.c +1
-0spin_unlock (&dev_list_lock);pci_set_drvdata (pdev,NULL ); +async_synchronize_cookie (dev->probe_cookie);
Keith, I think this may not work. All asynchronous functions called prior to the one identified by cookie
are guaranteed to have completed. The current one is not.
And I just remembered Matthew's feedback on one of the previous attempts at asynchronous probe.
We should be able to rmmod the driver during probe if suppose there are a million namespaces.
Is there any plan to address such scenarios?
include/linux/nvme.h index
Quoted 4 lines
drivers/block/nvme-core.c +0
-0flush_work (&dev->reset_work);misc_deregister (&dev->miscdev);nvme_dev_shutdown (dev);include/linux/nvme.h index renamed from include/linux/nvme.h +0
-0
19a5d4b..ebfbfa8
Quoted 27 lines
100644
include/linux/nvme.h +2
-1@@ -16,6 +16,7 @@#define _LINUX_NVME_H#include <uapi/linux/nvme.h> +#include <linux/async.h> #include <linux/pci.h> #include <linux/miscdevice.h> #include <linux/kref.h> @@ -94,6 +95,7 @@ struct nvme_dev {struct miscdevice miscdev;work_func_t reset_workfn;struct work_struct reset_work; +async_cookie_t probe_cookie;char name[12 ];char serial[20 ];char model[40 ]; --1.7.10.4
_______________________________________________
Linux-nvme mailing list
Linux-nvme at lists.infradead.org
http://lists.infradead.org/mailman/listinfo/linux-nvme
PATCH 1/1 NVMe: Async probe
Quoted 11 lines · Indraneel Mukherjee
Quoted 1 line
static void
nvme_reset_notify(struct pci_dev *pdev, bool prepare)@@-2902,6
drivers/block/nvme-core.c static void nvme_remove(struct pci_dev *pdev) · line 2909
Quoted 5 lines
+2906,7 @@ static void
nvme_remove(struct pci_dev *pdev)drivers/block/nvme-core.c +1
-0spin_unlock (&dev_list_lock);pci_set_drvdata (pdev,NULL ); +async_synchronize_cookie (dev->probe_cookie);Keith, I think this may not work. All asynchronous functions called prior to the one identified by cookie
are guaranteed to have completed. The current one is not.
Huh, so it is. That rules out asynchronous_schedule as being useful here. Workqueues FTW, then.
Quoted 5 lines
And I just remembered Matthew's feedback on one of the previous attempts at asynchronous probe.
We should be able to rmmod the driver during probe if suppose there are a million namespaces.
Is there any plan to address such scenarios?
This idea would have allowed that if sync cookie worked like I'd thought. The rmmod would just synchronize with the namespace discovery first. I think we can use the global work queue and flush individual work items in the remove path.