NVMe: Queue bio requests on device
Subsystems (2)
PATCH 1/8 NVMe: Queue bio requests on device
A bio request is not tied to an NVMe IO submission queue, so bio requests can be resubmitted on any available queue. Instead of adding bios on the nvme_queue, this queues bios on the nvme_dev. This should help balance the load across the submission queues.
--- drivers/block/nvme.c | 57 +++++++++++++++++++++++-------------------------- 1 files changed, 27 insertions(+), 30 deletions(-)
drivers/block/nvme.c +27 -30
@@ -80,6 +80,8 @@ struct nvme_dev {char model[40 ];char firmware_rev[8 ];u32 max_hw_sectors; +struct bio_list bio_list; +spinlock_t dev_lock; };/*
@@ -109,8 +111,6 @@ struct nvme_queue {dma_addr_t sq_dma_addr;dma_addr_t cq_dma_addr;wait_queue_head_t sq_full; -wait_queue_t sq_cong_wait; -struct bio_list sq_cong;u32 __iomem *q_db;u16 q_depth;u16 cq_vector;
@@ -245,6 +245,8 @@ static void *free_cmdid(struct nvme_queue *nvmeq, int cmdid, info[cmdid].ctx =CMD_CTX_COMPLETED ;clear_bit (cmdid, nvmeq->cmdid_data);wake_up (&nvmeq->sq_full); +if (!bio_list_empty (&nvmeq->dev->bio_list)) +wake_up_process (nvme_thread);return ctx; }
@@ -363,11 +365,10 @@ static void nvme_free_iod(struct nvme_dev *dev, struct nvme_iod *iod)static void requeue_bio (struct nvme_dev *dev,struct bio *bio) { -struct nvme_queue *nvmeq =get_nvmeq (dev); -if (bio_list_empty (&nvmeq->sq_cong)) -add_wait_queue (&nvmeq->sq_full, &nvmeq->sq_cong_wait); -bio_list_add (&nvmeq->sq_cong, bio); -put_nvmeq (nvmeq); +unsigned long flags; +spin_lock_irqsave (&dev->dev_lock, flags); +bio_list_add (&dev->bio_list, bio); +spin_unlock_irqrestore (&dev->dev_lock, flags);wake_up_process (nvme_thread); }
@@ -624,19 +625,17 @@ static int nvme_submit_bio_queue(struct nvme_queue *nvmeq, struct nvme_ns *ns,static void nvme_make_request (struct request_queue *q,struct bio *bio) {struct nvme_ns *ns = q->queuedata; -struct nvme_queue *nvmeq =get_nvmeq (ns->dev); -int result = -EBUSY ; +struct nvme_dev *dev = ns->dev; +struct nvme_queue *nvmeq =get_nvmeq (dev); +int result;spin_lock_irq (&nvmeq->q_lock); -if (bio_list_empty (&nvmeq->sq_cong)) - result =nvme_submit_bio_queue (nvmeq, ns, bio); -if (unlikely (result)) { -if (bio_list_empty (&nvmeq->sq_cong)) -add_wait_queue (&nvmeq->sq_full, &nvmeq->sq_cong_wait); -bio_list_add (&nvmeq->sq_cong, bio); - } - + result =nvme_submit_bio_queue (nvmeq, ns, bio);spin_unlock_irq (&nvmeq->q_lock); + +if (unlikely (result)) +requeue_bio (dev, bio); +put_nvmeq (nvmeq); }
@@ -912,10 +911,6 @@ static void nvme_free_queue(struct nvme_dev *dev, int qid)spin_lock_irq (&nvmeq->q_lock);nvme_cancel_ios (nvmeq, false); -while (bio_list_peek (&nvmeq->sq_cong)) { -struct bio *bio =bio_list_pop (&nvmeq->sq_cong); -bio_endio (bio, -EIO ); - }spin_unlock_irq (&nvmeq->q_lock);irq_set_affinity_hint (vector,NULL );
@@ -957,8 +952,6 @@ static struct nvme_queue *nvme_alloc_queue(struct nvme_dev *dev, int qid, nvmeq->cq_head =0 ; nvmeq->cq_phase =1 ;init_waitqueue_head (&nvmeq->sq_full); -init_waitqueue_entry (&nvmeq->sq_cong_wait, nvme_thread); -bio_list_init (&nvmeq->sq_cong); nvmeq->q_db = &dev->dbs[qid << (dev->db_stride +1 )]; nvmeq->q_depth = depth; nvmeq->cq_vector = vector;
@@ -1279,17 +1272,19 @@ static const struct block_device_operations nvme_fops = {static void nvme_resubmit_bios (struct nvme_queue *nvmeq) { -while (bio_list_peek (&nvmeq->sq_cong)) { -struct bio *bio =bio_list_pop (&nvmeq->sq_cong); +unsigned long flags; +struct bio_list *list = &nvmeq->dev->bio_list; + +spin_lock_irqsave (&nvmeq->dev->dev_lock, flags); +while (bio_list_peek (list)) { +struct bio *bio =bio_list_pop (list);struct nvme_ns *ns = bio->bi_bdev->bd_disk->private_data;if (nvme_submit_bio_queue (nvmeq, ns, bio)) { -bio_list_add_head (&nvmeq->sq_cong, bio); +bio_list_add_head (list, bio);break ; } -if (bio_list_empty (&nvmeq->sq_cong)) -remove_wait_queue (&nvmeq->sq_full, - &nvmeq->sq_cong_wait); } +spin_unlock_irqrestore (&nvmeq->dev->dev_lock, flags); }static int nvme_kthread (void *data)
@@ -1309,7 +1304,8 @@ static int nvme_kthread(void *data)if (nvme_process_cq (nvmeq))printk ("process_cq did something\n" );nvme_cancel_ios (nvmeq, true); -nvme_resubmit_bios (nvmeq); +if (i) +nvme_resubmit_bios (nvmeq);spin_unlock_irq (&nvmeq->q_lock); } }
@@ -1660,6 +1656,7 @@ static int __devinit nvme_probe(struct pci_dev *pdev, goto disable;INIT_LIST_HEAD (&dev->namespaces); +spin_lock_init (&dev->dev_lock); dev->pci_dev = pdev;pci_set_drvdata (pdev, dev);dma_set_mask (&pdev->dev,DMA_BIT_MASK (64 ));
Signature · 1 line
1.7.0.4
PATCH 2/8 NVMe: Controller reset from user
Allow a user to issue a controller reset. A reset does not delete the gendisks so that IO may continue, or the namespaces may be mounted. This may be done by a user if they need to reset the controller for any reason, like if it is required as part of an activate firmware operation.
--- drivers/block/nvme.c | 155 +++++++++++++++++++++++++++++++++++++++++++++++--- 1 files changed, 147 insertions(+), 8 deletions(-)
drivers/block/nvme.c +147 -8
@@ -111,6 +111,7 @@ struct nvme_queue {dma_addr_t sq_dma_addr;dma_addr_t cq_dma_addr;wait_queue_head_t sq_full; +atomic_t busy;u32 __iomem *q_db;u16 q_depth;u16 cq_vector;
@@ -265,11 +266,18 @@ static void *cancel_cmdid(struct nvme_queue *nvmeq, int cmdid,static struct nvme_queue *get_nvmeq (struct nvme_dev *dev) { -return dev->queues[get_cpu () +1 ]; +struct nvme_queue *nvmeq; +spin_lock (&dev->dev_lock); + nvmeq = dev->queues[get_cpu () +1 ]; +if (nvmeq) +atomic_inc (&nvmeq->busy); +spin_unlock (&dev->dev_lock); +return nvmeq; }static void put_nvmeq (struct nvme_queue *nvmeq) { +atomic_dec (&nvmeq->busy);put_cpu (); }
@@ -629,6 +637,11 @@ static void nvme_make_request(struct request_queue *q, struct bio *bio)struct nvme_queue *nvmeq =get_nvmeq (dev);int result; +if (!nvmeq) { +requeue_bio (dev, bio); +return ; + } +spin_lock_irq (&nvmeq->q_lock); result =nvme_submit_bio_queue (nvmeq, ns, bio);spin_unlock_irq (&nvmeq->q_lock);
@@ -909,10 +922,15 @@ static void nvme_free_queue(struct nvme_dev *dev, int qid)struct nvme_queue *nvmeq = dev->queues[qid];int vector = dev->entry[nvmeq->cq_vector].vector; -spin_lock_irq (&nvmeq->q_lock); -nvme_cancel_ios (nvmeq, false); -spin_unlock_irq (&nvmeq->q_lock); +spin_lock (&dev->dev_lock); + dev->queues[qid] =NULL ; +spin_unlock (&dev->dev_lock); + +while (atomic_read (&nvmeq->busy)) +msleep (10 ); + +synchronize_irq (vector);irq_set_affinity_hint (vector,NULL );free_irq (vector, nvmeq);
@@ -922,6 +940,11 @@ static void nvme_free_queue(struct nvme_dev *dev, int qid)adapter_delete_cq (dev, qid); } +spin_lock_irq (&nvmeq->q_lock); +nvme_process_cq (nvmeq); +nvme_cancel_ios (nvmeq, false); +spin_unlock_irq (&nvmeq->q_lock); +nvme_free_queue_mem (nvmeq); }
@@ -1014,7 +1037,7 @@ static __devinit struct nvme_queue *nvme_create_queue(struct nvme_dev *dev,return ERR_PTR (result); } -static int __devinit nvme_configure_admin_queue (struct nvme_dev *dev) +static int nvme_configure_admin_queue (struct nvme_dev *dev) {int result =0 ;u32 aqa;
@@ -1177,6 +1200,11 @@ static int nvme_submit_io(struct nvme_ns *ns, struct nvme_user_io __user *uio) length =nvme_setup_prps (dev, &c.common, iod, length,GFP_KERNEL ); nvmeq =get_nvmeq (dev); +if (!nvmeq) { + status = -EFAULT ; + goto unmap_pages; + } +/* * Since nvme_submit_sync_cmd sleeps, we can't keep preemption * disabled. We may be preempted at any point, and be rescheduled
@@ -1189,6 +1217,7 @@ static int nvme_submit_io(struct nvme_ns *ns, struct nvme_user_io __user *uio)else status =nvme_submit_sync_cmd (nvmeq, &c,NULL ,NVME_IO_TIMEOUT ); + unmap_pages:nvme_unmap_user_pages (dev, io.opcode &1 , iod);nvme_free_iod (dev, iod);return status;
@@ -1419,7 +1448,7 @@ static int set_queue_count(struct nvme_dev *dev, int count)return min (result &0xffff , result >>16 ) +1 ; } -static int __devinit nvme_setup_io_queues (struct nvme_dev *dev) +static int nvme_setup_io_queues (struct nvme_dev *dev) {int result, cpu, i, nr_io_queues, db_bar_size, q_depth;
@@ -1490,6 +1519,7 @@ static void nvme_free_queues(struct nvme_dev *dev)for (i = dev->queue_count -1 ; i >=0 ; i--)nvme_free_queue (dev, i); + dev->queue_count =0 ; }static int __devinit nvme_dev_add (struct nvme_dev *dev)
@@ -1630,6 +1660,108 @@ static void nvme_release_instance(struct nvme_dev *dev)spin_unlock (&dev_list_lock); } +static int nvme_shutdown_controller (struct nvme_dev *dev) +{ +int i; +unsigned long timeout; + +spin_lock (&dev_list_lock); +list_del (&dev->node); +spin_unlock (&dev_list_lock); + +spin_lock (&dev->dev_lock); +for (i = dev->queue_count; i <num_possible_cpus (); i++) + dev->queues[i] =NULL ; +spin_unlock (&dev->dev_lock); +nvme_free_queues (dev); + + dev->ctrl_config |=NVME_CC_SHN_NORMAL ; +writel (dev->ctrl_config, &dev->bar->cc); + timeout =HZ + jiffies; + +while (!(readl (&dev->bar->csts) &NVME_CSTS_SHST_CMPLT )) { +msleep (5 ); +if (fatal_signal_pending (current)) +break ; +if (time_after (jiffies, timeout)) { +dev_err (&dev->pci_dev->dev, +"Device still ready; aborting shutdown\n" ); +break ; + } + } + +pci_disable_msix (dev->pci_dev); +iounmap (dev->bar); +pci_disable_device (dev->pci_dev); +pci_release_regions (dev->pci_dev); + +return 0 ; +} + +static int nvme_restart_controller (struct nvme_dev *dev) +{ +int bars, result = -ENOMEM ; + +if (pci_enable_device_mem (dev->pci_dev)) +return -ENOMEM ; + +pci_set_master (dev->pci_dev); + bars =pci_select_bars (dev->pci_dev,IORESOURCE_MEM ); +if (pci_request_selected_regions (dev->pci_dev, bars,"nvme" )) + goto disable_pci; + +dma_set_mask (&dev->pci_dev->dev,DMA_BIT_MASK (64 )); +dma_set_coherent_mask (&dev->pci_dev->dev,DMA_BIT_MASK (64 )); + dev->entry[0 ].vector = dev->pci_dev->irq; + dev->bar =ioremap (pci_resource_start (dev->pci_dev,0 ),8192 ); +if (!dev->bar) + goto disable; + + result =nvme_configure_admin_queue (dev); +if (result) + goto unmap; + dev->queue_count++; + +spin_lock (&dev_list_lock); +list_add (&dev->node, &dev_list); +spin_unlock (&dev_list_lock); + + result =nvme_setup_io_queues (dev); +if (result) + goto remove; + +return 0 ; + + remove: +nvme_dev_remove (dev); + unmap: +iounmap (dev->bar); + disable: +pci_release_regions (dev->pci_dev); + disable_pci: +pci_disable_device (dev->pci_dev); +return result; +} + +static int nvme_reset_controller (struct nvme_dev *dev) +{ +int ret =nvme_shutdown_controller (dev); +if (ret) +return ret; + ret =nvme_restart_controller (dev); +return ret; +} + +static ssize_t reset_controller (struct device *dev, +struct device_attribute *attr,const char *buf,size_t count) +{ + struct pci_dev *pdev = container_of(dev, struct pci_dev, dev); +struct nvme_dev *ndev =pci_get_drvdata (pdev); +nvme_reset_controller (ndev); +return count; +} +static DEVICE_ATTR(reset_controller, S_IWUSR, NULL, reset_controller); +static int __devinit nvme_probe (struct pci_dev *pdev,const struct pci_device_id *id) {
@@ -1686,13 +1818,19 @@ static int __devinit nvme_probe(struct pci_dev *pdev,list_add (&dev->node, &dev_list);spin_unlock (&dev_list_lock); - result =nvme_dev_add (dev); + result =device_create_file (&pdev->dev, &dev_attr_reset_controller);if (result) goto delete; + result =nvme_dev_add (dev); +if (result) + goto del_sysfs; +return 0 ; - delete: + del_sysfs: +device_remove_file (&pdev->dev, &dev_attr_reset_controller); +delete:spin_lock (&dev_list_lock);list_del (&dev->node);spin_unlock (&dev_list_lock);
@@ -1718,6 +1856,7 @@ static void __devexit nvme_remove(struct pci_dev *pdev) {struct nvme_dev *dev =pci_get_drvdata (pdev);nvme_dev_remove (dev); +device_remove_file (&pdev->dev, &dev_attr_reset_controller);pci_disable_msix (pdev);iounmap (dev->bar);nvme_release_instance (dev);
Signature · 1 line
1.7.0.4
PATCH 2/8 NVMe: Controller reset from user
Quoted 8 lines · Keith Busch
Allow a user to issue a controller reset. A reset does not delete the gendisks so that IO may continue, or the namespaces may be mounted. This may be done by a user if they need to reset the controller for any reason, like if it is required as part of an activate firmware operation.
drivers/block/nvme.c +0
-0@@ -265,11 +266,18 @@ static void *cancel_cmdid(struct nvme_queue *nvmeq, int cmdid,static struct nvme_queue *get_nvmeq (struct nvme_dev *dev)
drivers/block/nvme.c static void *cancel_cmdid(struct nvme_queue *nvmeq, int cmdid, · line 276
Quoted 10 lines
drivers/block/nvme.c +7
-1{ -return dev->queues[get_cpu () +1 ]; +struct nvme_queue *nvmeq; +spin_lock (&dev->dev_lock); + nvmeq = dev->queues[get_cpu () +1 ]; +if (nvmeq) +atomic_inc (&nvmeq->busy); +spin_unlock (&dev->dev_lock); +return nvmeq; }
OK, we can't do this. Right now, submitting an I/O is entirely per-CPU ... this introduces cacheline contention for the dev_lock. Even if it's never contended, the cacheline has to be obtained from the other CPU, so it'll be a major slowdown for large systems, and destroys scaling.
We have options to fix this. Here are a few:
1. Use RCU. You can find it documented in Documentation/RCU/ but the basic idea is to use ordering to protect against future users of the queue, and then wait for all existing users to be done with it before freeing it.
2. Avoid freeing the queue. When resetting the controller, we don't actually have to free the memory and then reallocate it; we can reprogram the controller when it comes back up with the addresses that this memory already has. So what we could do is put a single bit in nvme_queue ... call it something like q_suspended, and check it thusly:
static struct nvme_queue *get_nvmeq(struct nvme_dev *dev) {
- return dev->queues[get_cpu() + 1];
+ struct nvme_queue *nvmeq = dev->queues[get_cpu() + 1];
+ if (nvmeq->q_suspended) {
+ put_cpu();
+ return NULL;
+ } else {
+ nvmeq->q_usage++;
+ return nvmeq;
+ }
}
I think q_usage can be a simple variable at this point ... unless you see a reason for it to be atomic?
Quoted 99 lines
drivers/block/nvme.c +95
-0@@ -1630,6 +1660,108 @@ static void nvme_release_instance(struct nvme_dev *dev)spin_unlock (&dev_list_lock); } +static int nvme_shutdown_controller (struct nvme_dev *dev) +{ +int i; +unsigned long timeout; + +spin_lock (&dev_list_lock); +list_del (&dev->node); +spin_unlock (&dev_list_lock); + +spin_lock (&dev->dev_lock); +for (i = dev->queue_count; i <num_possible_cpus (); i++) + dev->queues[i] =NULL ; +spin_unlock (&dev->dev_lock); +nvme_free_queues (dev); + + dev->ctrl_config |=NVME_CC_SHN_NORMAL ; +writel (dev->ctrl_config, &dev->bar->cc); + timeout =HZ + jiffies; + +while (!(readl (&dev->bar->csts) &NVME_CSTS_SHST_CMPLT )) { +msleep (5 ); +if (fatal_signal_pending (current)) +break ; +if (time_after (jiffies, timeout)) { +dev_err (&dev->pci_dev->dev, +"Device still ready; aborting shutdown\n" ); +break ; + } + } + +pci_disable_msix (dev->pci_dev); +iounmap (dev->bar); +pci_disable_device (dev->pci_dev); +pci_release_regions (dev->pci_dev); + +return 0 ; +} + +static int nvme_restart_controller (struct nvme_dev *dev) +{ +int bars, result = -ENOMEM ; + +if (pci_enable_device_mem (dev->pci_dev)) +return -ENOMEM ; + +pci_set_master (dev->pci_dev); + bars =pci_select_bars (dev->pci_dev,IORESOURCE_MEM ); +if (pci_request_selected_regions (dev->pci_dev, bars,"nvme" )) + goto disable_pci; + +dma_set_mask (&dev->pci_dev->dev,DMA_BIT_MASK (64 )); +dma_set_coherent_mask (&dev->pci_dev->dev,DMA_BIT_MASK (64 )); + dev->entry[0 ].vector = dev->pci_dev->irq; + dev->bar =ioremap (pci_resource_start (dev->pci_dev,0 ),8192 ); +if (!dev->bar) + goto disable; + + result =nvme_configure_admin_queue (dev); +if (result) + goto unmap; + dev->queue_count++; + +spin_lock (&dev_list_lock); +list_add (&dev->node, &dev_list); +spin_unlock (&dev_list_lock); + + result =nvme_setup_io_queues (dev); +if (result) + goto remove; + +return 0 ; + + remove: +nvme_dev_remove (dev); + unmap: +iounmap (dev->bar); + disable: +pci_release_regions (dev->pci_dev); + disable_pci: +pci_disable_device (dev->pci_dev); +return result; +} + +static int nvme_reset_controller (struct nvme_dev *dev) +{ +int ret =nvme_shutdown_controller (dev); +if (ret) +return ret; + ret =nvme_restart_controller (dev); +return ret; +} + +static ssize_t reset_controller (struct device *dev, +struct device_attribute *attr,const char *buf,size_t count) +{
drivers/block/nvme.c static void nvme_release_instance(struct nvme_dev *dev) · line 1767
Quoted 10 lines
drivers/block/nvme.c +7
-0+ struct pci_dev *pdev = container_of(dev, struct pci_dev, dev); +struct nvme_dev *ndev =pci_get_drvdata (pdev); +nvme_reset_controller (ndev); +return count; +} +static DEVICE_ATTR(reset_controller, S_IWUSR, NULL, reset_controller); +static int __devinit nvme_probe (struct pci_dev *pdev,const struct pci_device_id *id) {
The problem with this implementation of reset controller is that it requires the controller to be sufficiently alive to process admin commands. I think we need reset controller to be able to work if the admin queue is broken for some reason. So just whacking the config register, waiting for it to complete, then reprogramming the registers, and resending all the admin init commands seems like the right move to me.
PATCH 2/8 NVMe: Controller reset from user
Quoted 5 lines · Matthew Wilcox
2. Avoid freeing the queue. When resetting the controller, we don't actually have to free the memory and then reallocate it; we can reprogram the controller when it comes back up with the addresses that this memory already has. So what we could do is put a single bit in
nvme_queue... call it something likeq_suspended, and check it thusly:
This sounds like a pretty good idea. The only problem I see is the queues had to have been created in the first place, and if a controller was security locked, it would not have allowed this operation.
I was trying to reuse the exisiting queue initialization as much as possible in case something changes after the reset, like more cpu's were brought online to allow more queues, or if the controller configuration changed from new firmware that may alter the queue count or depth. I think this can all be accounted for without freeing the queue memory, but not quite as easily.
Quoted 6 lines
The problem with this implementation of reset controller is that it requires the controller to be sufficiently alive to process admin commands. I think we need reset controller to be able to work if the admin queue is broken for some reason. So just whacking the config register, waiting for it to complete, then reprogramming the registers, and resending all the admin init commands seems like the right move to me.
The patch was following the Shutdown sequence from the NVMe spec (7.6.2 in revision 1.1). Hitting CC.EN to reset the controller isn't recommended for normal shutdowns, but you're right, we can't hold up a shutdown if the controller isn't responding to admin commands either. I added schedule_timeout for sync commands in this patch set for commands that do not complete, but that would hold things up for an unexceptable time if the controller is not responding. Maybe we should have separate graceful and abrupt shutdown sequences?
Anyway, thanks for the info. I'll take another shot at this.
PATCH 3/8 NVMe: Suspend/resume power management
In reply to Keith Busch--- drivers/block/nvme.c | 24 ++++++++++++++++++++++-- 1 files changed, 22 insertions(+), 2 deletions(-)
drivers/block/nvme.c +22 -2
@@ -1874,8 +1874,27 @@ static void __devexit nvme_remove(struct pci_dev *pdev)#define nvme_link_reset NULL#define nvme_slot_reset NULL#define nvme_error_resume NULL + +#ifdef CONFIG_PM +static int nvme_suspend (struct device *dev) +{ +struct pci_dev *pdev =to_pci_dev (dev); +struct nvme_dev *ndev =pci_get_drvdata (pdev); +return nvme_shutdown_controller (ndev); +} + +static int nvme_resume (struct device *dev) +{ +struct pci_dev *pdev =to_pci_dev (dev); +struct nvme_dev *ndev =pci_get_drvdata (pdev); +return nvme_restart_controller (ndev); +} +#else #define nvme_suspend NULL#define nvme_resume NULL +#endif + +UNIVERSAL_DEV_PM_OPS (nvme_dev_pm_ops, nvme_suspend, nvme_resume,NULL );static const struct pci_error_handlers nvme_err_handler = { .error_detected = nvme_error_detected,
@@ -1899,8 +1918,9 @@ static struct pci_driver nvme_driver = { .id_table = nvme_id_table, .probe = nvme_probe, .remove = __devexit_p(nvme_remove), - .suspend = nvme_suspend, - .resume = nvme_resume, + .driver = { + .pm = &nvme_dev_pm_ops, + }, .err_handler = &nvme_err_handler, };
Signature · 1 line
1.7.0.4
PATCH 4/8 NVMe: Add shutdown callback
In reply to Keith BuschShuts down the controller in a controlled way in accordance with the NVMe spec when a system shutdown request occurs.
--- drivers/block/nvme.c | 7 +++++++ 1 files changed, 7 insertions(+), 0 deletions(-)
drivers/block/nvme.c +7 -0
@@ -1875,6 +1875,12 @@ static void __devexit nvme_remove(struct pci_dev *pdev)#define nvme_slot_reset NULL#define nvme_error_resume NULL +static void nvme_shutdown (struct pci_dev *pdev) +{ +struct nvme_dev *dev =pci_get_drvdata (pdev); +nvme_shutdown_controller (dev); +} +#ifdef CONFIG_PM static int nvme_suspend (struct device *dev) {
@@ -1918,6 +1924,7 @@ static struct pci_driver nvme_driver = { .id_table = nvme_id_table, .probe = nvme_probe, .remove = __devexit_p(nvme_remove), + .shutdown = nvme_shutdown, .driver = { .pm = &nvme_dev_pm_ops, },
Signature · 1 line
1.7.0.4
PATCH 5/8 NVMe: End queued bio requests for removed disks
In reply to Keith BuschBio requests queued in the driver should not submit the request to disks being deleted. This patch checks if the disk is up before submitting and lets the kthread run on a device being removed until after the namespaces have been cleared so that the bio's can be ended.
--- drivers/block/nvme.c | 12 ++++++++---- 1 files changed, 8 insertions(+), 4 deletions(-)
drivers/block/nvme.c +8 -4
@@ -1308,6 +1308,10 @@ static void nvme_resubmit_bios(struct nvme_queue *nvmeq)while (bio_list_peek (list)) {struct bio *bio =bio_list_pop (list);struct nvme_ns *ns = bio->bi_bdev->bd_disk->private_data; +if (!(bio->bi_bdev->bd_disk->flags &GENHD_FL_UP )) { +bio_endio (bio, -EIO ); +continue ; + }if (nvme_submit_bio_queue (nvmeq, ns, bio)) {bio_list_add_head (list, bio);break ;
@@ -1592,16 +1596,16 @@ static int nvme_dev_remove(struct nvme_dev *dev) {struct nvme_ns *ns, *next; -spin_lock (&dev_list_lock); -list_del (&dev->node); -spin_unlock (&dev_list_lock); -list_for_each_entry_safe (ns, next, &dev->namespaces, list) {list_del (&ns->list);del_gendisk (ns->disk);nvme_ns_free (ns); } +spin_lock (&dev_list_lock); +list_del (&dev->node); +spin_unlock (&dev_list_lock); +nvme_free_queues (dev);return 0 ;
Signature · 1 line
1.7.0.4
PATCH 6/8 NVMe: Wait for controller status ready to clear
In reply to Keith BuschThe nvme spec specifies potentially undefined results if we continue to use the driver after clearing enable but not waiting for CSTS.RDY to clear. This waits for ready status to clear before proceeding.
--- drivers/block/nvme.c | 16 ++++++++++++++-- 1 files changed, 14 insertions(+), 2 deletions(-)
drivers/block/nvme.c +14 -2
@@ -1045,6 +1045,20 @@ static int nvme_configure_admin_queue(struct nvme_dev *dev)unsigned long timeout;struct nvme_queue *nvmeq; + cap =readq (&dev->bar->cap); + timeout = ((NVME_CAP_TIMEOUT (cap) +1 ) *HZ /2 ) + jiffies; +writel (0 , &dev->bar->cc); +while (!result && (readl (&dev->bar->csts) &NVME_CSTS_RDY )) { +msleep (100 ); +if (fatal_signal_pending (current)) +return -EINTR ; +if (time_after (jiffies, timeout)) { +dev_err (&dev->pci_dev->dev, +"Device still ready; aborting initialisation\n" ); +return -ENODEV ; + } + } + dev->dbs = ((void __iomem *)dev->bar) +4096 ; nvmeq =nvme_alloc_queue (dev,0 ,64 ,0 );
@@ -1059,13 +1073,11 @@ static int nvme_configure_admin_queue(struct nvme_dev *dev) dev->ctrl_config |=NVME_CC_ARB_RR |NVME_CC_SHN_NONE ; dev->ctrl_config |=NVME_CC_IOSQES |NVME_CC_IOCQES ; -writel (0 , &dev->bar->cc);writel (aqa, &dev->bar->aqa);writeq (nvmeq->sq_dma_addr, &dev->bar->asq);writeq (nvmeq->cq_dma_addr, &dev->bar->acq);writel (dev->ctrl_config, &dev->bar->cc); - cap =readq (&dev->bar->cap); timeout = ((NVME_CAP_TIMEOUT (cap) +1 ) *HZ /2 ) + jiffies; dev->db_stride =NVME_CAP_STRIDE (cap);
Signature · 1 line
1.7.0.4
PATCH 7/8 NVMe: Automatically reset failed controller
In reply to Keith BuschAn nvme controller may indicate it is failed and requires a reset by setting its CSTS.CFS register to 1. This patch has the polling thread check this value and automatically issue the reset when the controller reports itself as failed.
Also, if the controller fails to return an IO, this will be considered a failed controller and require a reset. Previously the polling thread would timeout an IO and free up the resources. This is bad because if the controller happens to service that command in the future, the resources the driver allocated for that IO are no longer valid and could result in the controller accessing memory that is associated with something else, resulting in memory or data corruption.
--- drivers/block/nvme.c | 42 +++++++++++++++++++++++++++++++++++++++--- 1 files changed, 39 insertions(+), 3 deletions(-)
drivers/block/nvme.c +39 -3
@@ -58,6 +58,7 @@ module_param(use_threaded_interrupts, int, 0);static DEFINE_SPINLOCK (dev_list_lock);static LIST_HEAD (dev_list);static struct task_struct *nvme_thread; +static struct workqueue_struct *nvme_workq;/* * Represents an NVM Express device. Each nvme_dev is a PCI function.
@@ -82,6 +83,7 @@ struct nvme_dev {u32 max_hw_sectors;struct bio_list bio_list;spinlock_t dev_lock; +struct work_struct ws; };/*
@@ -886,7 +888,7 @@ static int nvme_set_features(struct nvme_dev *dev, unsigned fid,* @queue: The queue to cancel I/Os on * @timeout: True to only cancel I/Os which have timed out */ -static void nvme_cancel_ios (struct nvme_queue *nvmeq,bool timeout) +static int nvme_cancel_ios (struct nvme_queue *nvmeq,bool timeout) {int depth = nvmeq->q_depth -1 ;struct nvme_cmd_info *info =nvme_cmd_info (nvmeq);
@@ -902,10 +904,15 @@ static void nvme_cancel_ios(struct nvme_queue *nvmeq, bool timeout)if (timeout && !time_after (now, info[cmdid].timeout))continue ; +if (timeout) +return 1 ; +dev_warn (nvmeq->q_dmadev,"Cancelling I/O %d\n" , cmdid); ctx =cancel_cmdid (nvmeq, cmdid, &fn);fn (nvmeq->dev, ctx, &cqe); } + +return 0 ; }static void nvme_free_queue_mem (struct nvme_queue *nvmeq)
@@ -1341,6 +1348,12 @@ static int nvme_kthread(void *data)spin_lock (&dev_list_lock);list_for_each_entry (dev, &dev_list, node) {int i; +if (readl (&dev->bar->csts) &NVME_CSTS_CFS ) { +dev_warn (&dev->pci_dev->dev, +"failed status, reset controller\n" ); +queue_work (nvme_workq, &dev->ws); +continue ; + }for (i =0 ; i < dev->queue_count; i++) {struct nvme_queue *nvmeq = dev->queues[i];if (!nvmeq)
@@ -1348,7 +1361,12 @@ static int nvme_kthread(void *data)spin_lock_irq (&nvmeq->q_lock);if (nvme_process_cq (nvmeq))printk ("process_cq did something\n" ); -nvme_cancel_ios (nvmeq, true); +if (nvme_cancel_ios (nvmeq, true)) { +dev_warn (&dev->pci_dev->dev, +"timed out I/O, reset controller\n" ); +queue_work (nvme_workq, &dev->ws); +break ; + }if (i)nvme_resubmit_bios (nvmeq);spin_unlock_irq (&nvmeq->q_lock);
@@ -1778,6 +1796,14 @@ static ssize_t reset_controller(struct device *dev, } static DEVICE_ATTR(reset_controller, S_IWUSR, NULL, reset_controller); +static void nvme_reset_failed_dev (struct work_struct *ws) +{ + struct nvme_dev *dev = container_of(ws, struct nvme_dev, ws); +if (nvme_reset_controller (dev)) +dev_warn (&dev->pci_dev->dev, +"failed to reset failed controller\n" ); +} +static int __devinit nvme_probe (struct pci_dev *pdev,const struct pci_device_id *id) {
@@ -1830,6 +1856,8 @@ static int __devinit nvme_probe(struct pci_dev *pdev, goto unmap; dev->queue_count++; +INIT_WORK (&dev->ws, nvme_reset_failed_dev); +spin_lock (&dev_list_lock);list_add (&dev->node, &dev_list);spin_unlock (&dev_list_lock);
@@ -1955,9 +1983,14 @@ static int __init nvme_init(void)if (IS_ERR (nvme_thread))return PTR_ERR (nvme_thread); + result = -ENOMEM ; + nvme_workq =create_workqueue ("nvme" ); +if (!nvme_workq) + goto kill_kthread; + result =register_blkdev (nvme_major,"nvme" );if (result <0 ) - goto kill_kthread; + goto kill_workq;else if (result >0 ) nvme_major = result;
@@ -1968,6 +2001,8 @@ static int __init nvme_init(void) unregister_blkdev:unregister_blkdev (nvme_major,"nvme" ); + kill_workq: +destroy_workqueue (nvme_workq); kill_kthread:kthread_stop (nvme_thread);return result;
@@ -1977,6 +2012,7 @@ static void __exit nvme_exit(void) {pci_unregister_driver (&nvme_driver);unregister_blkdev (nvme_major,"nvme" ); +destroy_workqueue (nvme_workq);kthread_stop (nvme_thread); }
Signature · 1 line
1.7.0.4
PATCH 8/8 NVMe: Use schedule_timeout for sync commands
In reply to Keith BuschSchedule a timeout for sync commands should the controller fail to return status and the device is not being polled for timed out commands. This could happen when a device's queues are being deleted.
--- drivers/block/nvme.c | 2 +- 1 files changed, 1 insertions(+), 1 deletions(-)
drivers/block/nvme.c +1 -1
@@ -755,7 +755,7 @@ static int nvme_submit_sync_cmd(struct nvme_queue *nvmeq,set_current_state (TASK_KILLABLE );nvme_submit_cmd (nvmeq, cmd); -schedule (); +schedule_timeout (timeout);if (cmdinfo.status == -EINTR ) {nvme_abort_command (nvmeq, cmdid);
Signature · 1 line
1.7.0.4