Controller error handling set
Subsystems (2)
cover Controller error handling set
Additional handling for nvme device errors. I've submitted patches along these lines in the past, but this is better tested against hot-plug capable platforms and low power state transitions.
Obsoletes these:
http://merlin.infradead.org/pipermail/linux-nvme/2013-October/000563.html
http://merlin.infradead.org/pipermail/linux-nvme/2013-October/000566.html
Keith Busch (5):
NVMe: Device resume error handling
NVMe: Schedule reset for failed controllers
NVMe: Abort timed out commands
NVMe: Surprise removal handling
NVMe: Async IO queue deletion
drivers/block/nvme-core.c | 433 +++++++++++++++++++++++++++++++++++++++++---- include/linux/nvme.h | 3 + include/uapi/linux/nvme.h | 11 ++ 3 files changed, 416 insertions(+), 31 deletions(-)
Signature · 1 line
1.7.10.4
PATCH 1/5 NVMe: Device resume error handling
Adds controller error handling on resume power management. If the device fails to initialize, the device is queued for a reset. If the reset fails, a thread is spawned to remove the pci device.
If the device resumes as "busy", the device is responding to admin commands but will not create IO queues. In this case, we need to remove the gendisks and free the IO queues since they can't be used and may be holding bios in their lists.
Quoted 1 line
From testing, the dma pools require a pci device so this had to change
the pci driver 'remove' to release the dma resources in line with that call instead of after all references to the device are released.
--- drivers/block/nvme-core.c | 107 +++++++++++++++++++++++++++++++++++++++------ include/linux/nvme.h | 1 + 2 files changed, 94 insertions(+), 14 deletions(-)
drivers/block/nvme-core.c +93 -14
@@ -58,6 +58,7 @@ module_param(use_threaded_interrupts, int, 0);static DEFINE_SPINLOCK (dev_list_lock);static LIST_HEAD (dev_list);static struct task_struct *nvme_thread; +static struct workqueue_struct *nvme_workq;/* * An NVM Express queue. Each device has at least two (one for admin
@@ -1956,7 +1957,6 @@ static int nvme_dev_map(struct nvme_dev *dev)else goto disable_pci; -pci_set_drvdata (pdev, dev); dev->bar =ioremap (pci_resource_start (pdev,0 ),8192 );if (!dev->bar) goto disable;
@@ -1983,9 +1983,9 @@ static void nvme_dev_unmap(struct nvme_dev *dev)if (dev->bar) {iounmap (dev->bar); dev->bar =NULL ; +pci_release_regions (dev->pci_dev); } -pci_release_regions (dev->pci_dev);if (pci_is_enabled (dev->pci_dev))pci_disable_device (dev->pci_dev); }
@@ -2073,11 +2073,6 @@ static void nvme_release_instance(struct nvme_dev *dev)static void nvme_free_dev (struct kref *kref) { struct nvme_dev *dev = container_of(kref, struct nvme_dev, kref); -nvme_dev_remove (dev); -nvme_dev_shutdown (dev); -nvme_free_queues (dev); -nvme_release_instance (dev); -nvme_release_prp_pools (dev);kfree (dev->queues);kfree (dev->entry);kfree (dev);
@@ -2149,6 +2144,70 @@ static int nvme_dev_start(struct nvme_dev *dev)return result; } +static int nvme_remove_dead_ctrl (void *arg) +{ +struct nvme_dev *dev = (struct nvme_dev *)arg; +struct pci_dev *pdev = dev->pci_dev; + +if (pci_get_drvdata (pdev)) +pci_stop_and_remove_bus_device (pdev); +kref_put (&dev->kref, nvme_free_dev); +return 0 ; +} + +static void nvme_remove_disks (struct work_struct *ws) +{ +int i; + struct nvme_dev *dev = container_of(ws, struct nvme_dev, reset_work); + +nvme_dev_remove (dev); +spin_lock (&dev_list_lock); +for (i = dev->queue_count -1 ; i >0 ; i--) { +BUG_ON (!dev->queues[i] || !dev->queues[i]->q_suspended); +nvme_free_queue (dev->queues[i]); + dev->queue_count--; + dev->queues[i] =NULL ; + } +spin_unlock (&dev_list_lock); +} + +static int nvme_dev_resume (struct nvme_dev *dev) +{ +int ret; + + ret =nvme_dev_start (dev); +if (ret && ret != -EBUSY ) +return ret; +if (ret == -EBUSY ) { +spin_lock (&dev_list_lock); +INIT_WORK (&dev->reset_work, nvme_remove_disks); +queue_work (nvme_workq, &dev->reset_work); +spin_unlock (&dev_list_lock); + } +return 0 ; +} + +static void nvme_dev_reset (struct nvme_dev *dev) +{ +nvme_dev_shutdown (dev); +if (nvme_dev_resume (dev)) { +dev_err (&dev->pci_dev->dev,"Device failed to resume\n" ); +kref_get (&dev->kref); +if (IS_ERR (kthread_run (nvme_remove_dead_ctrl, dev,"nvme%d" , + dev->instance))) { +dev_err (&dev->pci_dev->dev, +"Failed to start controller remove task\n" ); +kref_put (&dev->kref, nvme_free_dev); + } + } +} + +static void nvme_reset_failed_dev (struct work_struct *ws) +{ + struct nvme_dev *dev = container_of(ws, struct nvme_dev, reset_work); +nvme_dev_reset (dev); +} +static int nvme_probe (struct pci_dev *pdev,const struct pci_device_id *id) {int result = -ENOMEM ;
@@ -2168,6 +2227,7 @@ static int nvme_probe(struct pci_dev *pdev, const struct pci_device_id *id)INIT_LIST_HEAD (&dev->namespaces); dev->pci_dev = pdev; +pci_set_drvdata (pdev, dev); result =nvme_set_instance (dev);if (result) goto free;
@@ -2219,7 +2279,19 @@ static int nvme_probe(struct pci_dev *pdev, const struct pci_device_id *id)static void nvme_remove (struct pci_dev *pdev) {struct nvme_dev *dev =pci_get_drvdata (pdev); + +spin_lock (&dev_list_lock); +list_del_init (&dev->node); +spin_unlock (&dev_list_lock); + +pci_set_drvdata (pdev,NULL ); +flush_work (&dev->reset_work);misc_deregister (&dev->miscdev); +nvme_dev_remove (dev); +nvme_dev_shutdown (dev); +nvme_free_queues (dev); +nvme_release_instance (dev); +nvme_release_prp_pools (dev);kref_put (&dev->kref, nvme_free_dev); }
@@ -2243,13 +2315,12 @@ static int nvme_resume(struct device *dev) {struct pci_dev *pdev =to_pci_dev (dev);struct nvme_dev *ndev =pci_get_drvdata (pdev); -int ret; - ret =nvme_dev_start (ndev); -/* XXX: should remove gendisks if resume fails */ -if (ret) -nvme_free_queues (ndev); -return ret; +if (nvme_dev_resume (ndev) && !work_busy (&ndev->reset_work)) { +INIT_WORK (&ndev->reset_work, nvme_reset_failed_dev); +queue_work (nvme_workq, &ndev->reset_work); + } +return 0 ; } static SIMPLE_DEV_PM_OPS(nvme_dev_pm_ops, nvme_suspend, nvme_resume);
@@ -2290,9 +2361,14 @@ static int __init nvme_init(void)if (IS_ERR (nvme_thread))return PTR_ERR (nvme_thread); + result = -ENOMEM ; + nvme_workq =create_singlethread_workqueue ("nvme" ); +if (!nvme_workq) + goto kill_kthread; + result =register_blkdev (nvme_major,"nvme" );if (result <0 ) - goto kill_kthread; + goto kill_workq;else if (result >0 ) nvme_major = result;
@@ -2303,6 +2379,8 @@ static int __init nvme_init(void) unregister_blkdev:unregister_blkdev (nvme_major,"nvme" ); + kill_workq: +destroy_workqueue (nvme_workq); kill_kthread:kthread_stop (nvme_thread);return result;
@@ -2312,6 +2390,7 @@ static void __exit nvme_exit(void) {pci_unregister_driver (&nvme_driver);unregister_blkdev (nvme_major,"nvme" ); +destroy_workqueue (nvme_workq);kthread_stop (nvme_thread); }
include/linux/nvme.h +1 -0
@@ -87,6 +87,7 @@ struct nvme_dev {struct list_head namespaces;struct kref kref;struct miscdevice miscdev; +struct work_struct reset_work;char name[12 ];char serial[20 ];char model[40 ];
Signature · 1 line
1.7.10.4
PATCH 2/5 NVMe: Schedule reset for failed controllers
In reply to Keith BuschSchedules a controller reset when it indicates it has a failed status. If the device does not become ready after a reset, the pci device will be scheduled for removal.
--- drivers/block/nvme-core.c | 20 ++++++++++++++++++-- include/linux/nvme.h | 1 + 2 files changed, 19 insertions(+), 2 deletions(-)
drivers/block/nvme-core.c +18 -2
@@ -60,6 +60,8 @@ static LIST_HEAD(dev_list);static struct task_struct *nvme_thread;static struct workqueue_struct *nvme_workq; +static void nvme_reset_failed_dev (struct work_struct *ws); +/* * An NVM Express queue. Each device has at least two (one for admin * commands and one for I/O commands).
@@ -1597,13 +1599,24 @@ static void nvme_resubmit_bios(struct nvme_queue *nvmeq)static int nvme_kthread (void *data) { -struct nvme_dev *dev; +struct nvme_dev *dev, *next;while (!kthread_should_stop ()) {set_current_state (TASK_INTERRUPTIBLE );spin_lock (&dev_list_lock); -list_for_each_entry (dev, &dev_list, node) { +list_for_each_entry_safe (dev, next, &dev_list, node) {int i; +if (readl (&dev->bar->csts) &NVME_CSTS_CFS && + dev->initialized) { +if (work_busy (&dev->reset_work)) +continue ; +list_del_init (&dev->node); +dev_warn (&dev->pci_dev->dev, +"Failed status, reset controller\n" ); +INIT_WORK (&dev->reset_work, nvme_reset_failed_dev); +queue_work (nvme_workq, &dev->reset_work); +continue ; + }for (i =0 ; i < dev->queue_count; i++) {struct nvme_queue *nvmeq = dev->queues[i];if (!nvmeq)
@@ -1994,6 +2007,7 @@ static void nvme_dev_shutdown(struct nvme_dev *dev) {int i; + dev->initialized =0 ;for (i = dev->queue_count -1 ; i >=0 ; i--)nvme_disable_queue (dev, i);
@@ -2184,6 +2198,7 @@ static int nvme_dev_resume(struct nvme_dev *dev)queue_work (nvme_workq, &dev->reset_work);spin_unlock (&dev_list_lock); } + dev->initialized =1 ;return 0 ; }
@@ -2257,6 +2272,7 @@ static int nvme_probe(struct pci_dev *pdev, const struct pci_device_id *id)if (result) goto remove; + dev->initialized =1 ;kref_init (&dev->kref);return 0 ;
include/linux/nvme.h +1 -0
@@ -95,6 +95,7 @@ struct nvme_dev {u32 max_hw_sectors;u32 stripe_size;u16 oncs; +u8 initialized; };/*
Signature · 1 line
1.7.10.4
PATCH 3/5 NVMe: Abort timed out commands
In reply to Keith BuschSend nvme abort command to io requests that have timed out on an initialized device. If the command is not returned after another timeout, schedule the controller for reset.
--- drivers/block/nvme-core.c | 68 ++++++++++++++++++++++++++++++++++++++++++++- include/linux/nvme.h | 1 + include/uapi/linux/nvme.h | 11 ++++++++ 3 files changed, 79 insertions(+), 1 deletion(-)
drivers/block/nvme-core.c +67 -1
@@ -83,6 +83,7 @@ struct nvme_queue {u16 sq_head;u16 sq_tail;u16 cq_head; +u16 qid;u8 cq_phase;u8 cqe_seen;u8 q_suspended;
@@ -100,6 +101,7 @@ static inline void _nvme_check_size(void)BUILD_BUG_ON (sizeof (struct nvme_delete_queue ) !=64 );BUILD_BUG_ON (sizeof (struct nvme_features ) !=64 );BUILD_BUG_ON (sizeof (struct nvme_format_cmd ) !=64 ); +BUILD_BUG_ON (sizeof (struct nvme_abort_cmd ) !=64 );BUILD_BUG_ON (sizeof (struct nvme_command ) !=64 );BUILD_BUG_ON (sizeof (struct nvme_id_ctrl ) !=4096 );BUILD_BUG_ON (sizeof (struct nvme_id_ns ) !=4096 );
@@ -114,6 +116,7 @@ struct nvme_cmd_info {nvme_completion_fn fn;void *ctx;unsigned long timeout; +int aborted; };static struct nvme_cmd_info *nvme_cmd_info (struct nvme_queue *nvmeq)
@@ -157,6 +160,7 @@ static int alloc_cmdid(struct nvme_queue *nvmeq, void *ctx, info[cmdid].fn = handler; info[cmdid].ctx = ctx; info[cmdid].timeout = jiffies + timeout; + info[cmdid].aborted =0 ;return cmdid; }
@@ -175,6 +179,7 @@ static int alloc_cmdid_killable(struct nvme_queue *nvmeq, void *ctx,#define CMD_CTX_COMPLETED (0x310 + CMD_CTX_BASE)#define CMD_CTX_INVALID (0x314 + CMD_CTX_BASE)#define CMD_CTX_FLUSH (0x318 + CMD_CTX_BASE) +#define CMD_CTX_ABORT (0x31C + CMD_CTX_BASE)static void special_completion (struct nvme_dev *dev,void *ctx,struct nvme_completion *cqe)
@@ -183,6 +188,10 @@ static void special_completion(struct nvme_dev *dev, void *ctx,return ;if (ctx ==CMD_CTX_FLUSH )return ; +if (ctx ==CMD_CTX_ABORT ) { + ++dev->abort_limit; +return ; + }if (ctx ==CMD_CTX_COMPLETED ) { dev_warn(&dev->pci_dev->dev, "completed id %d twice on queue %d\n",
@@ -1005,6 +1014,56 @@ int nvme_set_features(struct nvme_dev *dev, unsigned fid, unsigned dword11, }/** +* nvme_abort_cmd - Attempt aborting a command +* @cmdid: Command id of a timed out IO +* @queue: The queue with timed out IO +* +* Schedule controller reset if the command was already aborted once before and +* still hasn't been returned to the driver, or if this is the admin queue. +*/ +static void nvme_abort_cmd (int cmdid,struct nvme_queue *nvmeq) +{ +int a_cmdid; +struct nvme_command cmd; +struct nvme_dev *dev = nvmeq->dev; +struct nvme_cmd_info *info =nvme_cmd_info (nvmeq); + +if (!nvmeq->qid || info[cmdid].aborted) { +if (work_busy (&dev->reset_work)) +return ; +list_del_init (&dev->node); +dev_warn (&dev->pci_dev->dev, +"I/O %d QID %d timeout, reset controller\n" , cmdid, + nvmeq->qid); +INIT_WORK (&dev->reset_work, nvme_reset_failed_dev); +queue_work (nvme_workq, &dev->reset_work); +return ; + } + +if (!dev->abort_limit) +return ; + + a_cmdid =alloc_cmdid (dev->queues[0 ],CMD_CTX_ABORT , special_completion, +ADMIN_TIMEOUT ); +if (a_cmdid <0 ) +return ; + +memset (&cmd,0 ,sizeof (cmd)); + cmd.abort.opcode = nvme_admin_abort_cmd; + cmd.abort.cid = cmdid; + cmd.abort.sqid = nvmeq->qid; + cmd.abort.command_id = a_cmdid; + + --dev->abort_limit; + info[cmdid].aborted =1 ; + info[cmdid].timeout = jiffies +ADMIN_TIMEOUT ; + +dev_warn (nvmeq->q_dmadev,"Aborting I/O %d QID %d\n" , cmdid, + nvmeq->qid); +nvme_submit_cmd (dev->queues[0 ], &cmd); +} + +/** * nvme_cancel_ios - Cancel outstanding I/Os * @queue: The queue to cancel I/Os on * @timeout: True to only cancel I/Os which have timed out
@@ -1027,7 +1086,12 @@ static void nvme_cancel_ios(struct nvme_queue *nvmeq, bool timeout)continue ;if (info[cmdid].ctx ==CMD_CTX_CANCELLED )continue ; -dev_warn (nvmeq->q_dmadev,"Cancelling I/O %d\n" , cmdid); +if (timeout && nvmeq->dev->initialized) { +nvme_abort_cmd (cmdid, nvmeq); +continue ; + } +dev_warn (nvmeq->q_dmadev,"Cancelling I/O %d QID %d\n" , cmdid, + nvmeq->qid); ctx =cancel_cmdid (nvmeq, cmdid, &fn);fn (nvmeq->dev, ctx, &cqe); }
@@ -1119,6 +1183,7 @@ static struct nvme_queue *nvme_alloc_queue(struct nvme_dev *dev, int qid, nvmeq->q_db = &dev->dbs[qid << (dev->db_stride +1 )]; nvmeq->q_depth = depth; nvmeq->cq_vector = vector; + nvmeq->qid = qid; nvmeq->q_suspended =1 ; dev->queue_count++;
@@ -1913,6 +1978,7 @@ static int nvme_dev_add(struct nvme_dev *dev) ctrl = mem; nn =le32_to_cpup (&ctrl->nn); dev->oncs =le16_to_cpup (&ctrl->oncs); + dev->abort_limit = ctrl->acl +1 ;memcpy (dev->serial, ctrl->sn,sizeof (ctrl->sn));memcpy (dev->model, ctrl->mn,sizeof (ctrl->mn));memcpy (dev->firmware_rev, ctrl->fr,sizeof (ctrl->fr));
include/linux/nvme.h +1 -0
@@ -95,6 +95,7 @@ struct nvme_dev {u32 max_hw_sectors;u32 stripe_size;u16 oncs; +u16 abort_limit;u8 initialized; };
include/uapi/linux/nvme.h +11 -0
@@ -350,6 +350,16 @@ struct nvme_delete_queue {__u32 rsvd11[5 ]; }; +struct nvme_abort_cmd { +__u8 opcode; +__u8 flags; +__u16 command_id; +__u32 rsvd1[9 ]; +__le16 sqid; +__le16 cid; +__u32 rsvd11[5 ]; +}; +struct nvme_download_firmware {__u8 opcode;__u8 flags;
@@ -384,6 +394,7 @@ struct nvme_command {struct nvme_download_firmware dlfw;struct nvme_format_cmd format;struct nvme_dsm_cmd dsm; +struct nvme_abort_cmd abort; }; };
Signature · 1 line
1.7.10.4
PATCH 4/5 NVMe: Surprise removal handling
In reply to Keith BuschThis adds checks to see if the nvme pci device was removed. The check reads the status register for the value of -1, which it should never be unless the device is no longer present.
If a user performs a surprise removal on an nvme device, the driver will be notified either by the pci driver remove callback if the platform's slot is capable of this event, or via reading the device BAR status register, which will indicate controller failure and trigger a reset.
Either way, the device is not present so all outstanding commands would timeout. This will not send queue deletion commands to a drive that isn't present and fail after ioremap, significantly speeding up surprise removal; previously this took over 2 minutes per IO queue pair created, but this will complete removing the device within a few seconds.
--- drivers/block/nvme-core.c | 14 ++++++++++---- 1 file changed, 10 insertions(+), 4 deletions(-)
drivers/block/nvme-core.c +10 -4
@@ -1140,8 +1140,9 @@ static void nvme_disable_queue(struct nvme_dev *dev, int qid)irq_set_affinity_hint (vector,NULL );free_irq (vector, nvmeq); -/* Don't tell the adapter to delete the admin queue */ -if (qid) { +/* Don't tell the adapter to delete the admin queue. +* Don't tell a removed adapter to delete IO queues. */ +if (qid &&readl (&dev->bar->csts) !=-1 ) {adapter_delete_sq (dev, qid);adapter_delete_cq (dev, qid); }
@@ -2039,12 +2040,17 @@ static int nvme_dev_map(struct nvme_dev *dev) dev->bar =ioremap (pci_resource_start (pdev,0 ),8192 );if (!dev->bar) goto disable; - +if (readl (&dev->bar->csts) ==-1 ) { + result = -ENODEV ; + goto unmap; + } dev->db_stride =NVME_CAP_STRIDE (readq (&dev->bar->cap)); dev->dbs = ((void __iomem *)dev->bar) +4096 ;return 0 ; - + unmap: +iounmap (dev->bar); + dev->bar =NULL ; disable:pci_release_regions (pdev); disable_pci:
Signature · 1 line
1.7.10.4
PATCH 5/5 NVMe: Async IO queue deletion
In reply to Keith BuschThis attempts to delete all IO queues at the same time asynchronously on shutdown. This is necessary for a present device that is not responding; a shutdown operation previously would take 2 minutes per queue-pair to timeout before moving on to the next queue, making a device removal appear to take a very long time or "hung" as reported by users.
In the previous worst case, a removal may be stuck forever until a kill signal is given if there are more than 32 queue pairs since it would run out of admin command IDs after over an hour of timed out sync commands (admin queue depth is 64).
This patch will wait for the admin command timeout for all commands to complete, so the worst case now for an unresponsive controller is 60 seconds, though that still seems like a long time.
Since this adds another way to take queues offline, some duplicate code resulted so I moved these into more convienient functions.
--- drivers/block/nvme-core.c | 228 ++++++++++++++++++++++++++++++++++++++++++--- 1 file changed, 216 insertions(+), 12 deletions(-)
drivers/block/nvme-core.c +216 -12
@@ -62,6 +62,14 @@ static struct workqueue_struct *nvme_workq;static void nvme_reset_failed_dev (struct work_struct *ws); +struct async_cmd_info { +struct kthread_work work; +struct kthread_worker *worker; +u32 result; +int status; +void *ctx; +}; +/* * An NVM Express queue. Each device has at least two (one for admin * commands and one for I/O commands).
@@ -87,6 +95,7 @@ struct nvme_queue {u8 cq_phase;u8 cqe_seen;u8 q_suspended; +struct async_cmd_info cmdinfo;unsigned long cmdid_data[]; };
@@ -208,6 +217,15 @@ static void special_completion(struct nvme_dev *dev, void *ctx,dev_warn (&dev->pci_dev->dev,"Unknown special completion %p\n" , ctx); } +static void async_completion (struct nvme_dev *dev,void *ctx, +struct nvme_completion *cqe) +{ +struct async_cmd_info *cmdinfo = ctx; + cmdinfo->result =le32_to_cpup (&cqe->result); + cmdinfo->status =le16_to_cpup (&cqe->status) >>1 ; +queue_kthread_work (cmdinfo->worker, &cmdinfo->work); +} +/* * Called with local interrupts disabled and the q_lock held. May not sleep. */
@@ -898,12 +916,33 @@ int nvme_submit_sync_cmd(struct nvme_queue *nvmeq, struct nvme_command *cmd,return cmdinfo.status; } +int nvme_submit_async_cmd (struct nvme_queue *nvmeq,struct nvme_command *cmd, +struct async_cmd_info *cmdinfo, +unsigned timeout) +{ +int cmdid; + + cmdid =alloc_cmdid_killable (nvmeq, cmdinfo, async_completion, timeout); +if (cmdid <0 ) +return cmdid; + cmdinfo->status = -EINTR ; + cmd->common.command_id = cmdid; +nvme_submit_cmd (nvmeq, cmd); +return 0 ; +} +int nvme_submit_admin_cmd (struct nvme_dev *dev,struct nvme_command *cmd,u32 *result) {return nvme_submit_sync_cmd (dev->queues[0 ], cmd, result,ADMIN_TIMEOUT ); } +int nvme_submit_admin_cmd_async (struct nvme_dev *dev,struct nvme_command *cmd, +struct async_cmd_info *cmdinfo) +{ +return nvme_submit_async_cmd (dev->queues[0 ], cmd, cmdinfo,ADMIN_TIMEOUT ); +} +static int adapter_delete_queue (struct nvme_dev *dev,u8 opcode,u16 id) {int status;
@@ -1124,15 +1163,20 @@ static void nvme_free_queues(struct nvme_dev *dev) } } -static void nvme_disable_queue (struct nvme_dev *dev,int qid) +/** +* nvme_suspend_queue - put queue into suspended state +* @nvmeq - queue to suspend +* +* Returns 1 if already suspended, 0 otherwise. +*/ +static int nvme_suspend_queue (struct nvme_queue *nvmeq) { -struct nvme_queue *nvmeq = dev->queues[qid]; -int vector = dev->entry[nvmeq->cq_vector].vector; +int vector = nvmeq->dev->entry[nvmeq->cq_vector].vector;spin_lock_irq (&nvmeq->q_lock);if (nvmeq->q_suspended) {spin_unlock_irq (&nvmeq->q_lock); -return ; +return 1 ; } nvmeq->q_suspended =1 ;spin_unlock_irq (&nvmeq->q_lock);
@@ -1140,17 +1184,33 @@ static void nvme_disable_queue(struct nvme_dev *dev, int qid)irq_set_affinity_hint (vector,NULL );free_irq (vector, nvmeq); +return 0 ; +} + +static void nvme_clear_queue (struct nvme_queue *nvmeq) +{ +spin_lock_irq (&nvmeq->q_lock); +nvme_process_cq (nvmeq); +nvme_cancel_ios (nvmeq, false); +spin_unlock_irq (&nvmeq->q_lock); +} + +static void nvme_disable_queue (struct nvme_dev *dev,int qid) +{ +struct nvme_queue *nvmeq = dev->queues[qid]; + +if (!nvmeq) +return ; +if (nvme_suspend_queue (nvmeq)) +return ; +/* Don't tell the adapter to delete the admin queue. * Don't tell a removed adapter to delete IO queues. */ if (qid &&readl (&dev->bar->csts) !=-1 ) {adapter_delete_sq (dev, qid);adapter_delete_cq (dev, qid); } - -spin_lock_irq (&nvmeq->q_lock); -nvme_process_cq (nvmeq); -nvme_cancel_ios (nvmeq, false); -spin_unlock_irq (&nvmeq->q_lock); +nvme_clear_queue (nvmeq); } static struct nvme_queue *nvme_alloc_queue(struct nvme_dev *dev, int qid,
@@ -2075,20 +2135,164 @@ static void nvme_dev_unmap(struct nvme_dev *dev)pci_disable_device (dev->pci_dev); } +struct nvme_delq_ctx { +struct task_struct *waiter; +struct kthread_worker * worker; +atomic_t refcount; +}; + +static void nvme_wait_dq (struct nvme_delq_ctx *dq,struct nvme_dev *dev) +{ + dq->waiter = current; +mb (); + +for (;;) { +set_current_state (TASK_KILLABLE ); +if (!atomic_read (&dq->refcount)) +break ; +if (!schedule_timeout (ADMIN_TIMEOUT ) || +fatal_signal_pending (current)) { +set_current_state (TASK_RUNNING ); + +nvme_disable_ctrl (dev,readq (&dev->bar->cap)); +nvme_disable_queue (dev,0 ); + +send_sig (SIGKILL , dq->worker->task,1 ); +flush_kthread_worker (dq->worker); +return ; + } + } +set_current_state (TASK_RUNNING ); +} + +static void nvme_put_dq (struct nvme_delq_ctx *dq) +{ +atomic_dec (&dq->refcount); +if (dq->waiter) +wake_up_process (dq->waiter); +} + +static struct nvme_delq_ctx *nvme_get_dq (struct nvme_delq_ctx *dq) +{ +atomic_inc (&dq->refcount); +return dq; +} + +static void nvme_del_queue_end (struct nvme_queue *nvmeq) +{ +struct nvme_delq_ctx *dq = nvmeq->cmdinfo.ctx; + +nvme_clear_queue (nvmeq); +nvme_put_dq (dq); +} + +static int adapter_async_del_queue (struct nvme_queue *nvmeq,u8 opcode, +kthread_work_func_t fn) +{ +struct nvme_command c; + +memset (&c,0 ,sizeof (c)); + c.delete_queue.opcode = opcode; + c.delete_queue.qid =cpu_to_le16 (nvmeq->qid); + +init_kthread_work (&nvmeq->cmdinfo.work, fn); +return nvme_submit_admin_cmd_async (nvmeq->dev, &c, &nvmeq->cmdinfo); +} + +static void nvme_del_cq_work_handler (struct kthread_work *work) +{ + struct nvme_queue *nvmeq = container_of(work, struct nvme_queue, + cmdinfo.work); +nvme_del_queue_end (nvmeq); +} + +static int nvme_delete_cq (struct nvme_queue *nvmeq) +{ +return adapter_async_del_queue (nvmeq, nvme_admin_delete_cq, + nvme_del_cq_work_handler); +} + +static void nvme_del_sq_work_handler (struct kthread_work *work) +{ + struct nvme_queue *nvmeq = container_of(work, struct nvme_queue, + cmdinfo.work); +int status = nvmeq->cmdinfo.status; + +if (!status) + status =nvme_delete_cq (nvmeq); +if (status) +nvme_del_queue_end (nvmeq); +} + +static int nvme_delete_sq (struct nvme_queue *nvmeq) +{ +return adapter_async_del_queue (nvmeq, nvme_admin_delete_sq, + nvme_del_sq_work_handler); +} + +static void nvme_del_queue_start (struct kthread_work *work) +{ + struct nvme_queue *nvmeq = container_of(work, struct nvme_queue, + cmdinfo.work); +allow_signal (SIGKILL ); +if (nvme_delete_sq (nvmeq)) +nvme_del_queue_end (nvmeq); +} + +static void nvme_disable_io_queues (struct nvme_dev *dev) +{ +int i; +DEFINE_KTHREAD_WORKER_ONSTACK (worker); +struct nvme_delq_ctx dq; +struct task_struct *kworker_task =kthread_run (kthread_worker_fn, + &worker,"nvme%d" , dev->instance); + +if (IS_ERR (kworker_task)) { +dev_err (&dev->pci_dev->dev, +"Failed to create queue del task\n" ); +for (i = dev->queue_count -1 ; i >0 ; i--) +nvme_disable_queue (dev, i); +return ; + } + + dq.waiter =NULL ; +atomic_set (&dq.refcount,0 ); + dq.worker = &worker; +for (i = dev->queue_count -1 ; i >0 ; i--) { +struct nvme_queue *nvmeq = dev->queues[i]; + +if (nvme_suspend_queue (nvmeq)) +continue ; + nvmeq->cmdinfo.ctx =nvme_get_dq (&dq); + nvmeq->cmdinfo.worker = dq.worker; +init_kthread_work (&nvmeq->cmdinfo.work, nvme_del_queue_start); +queue_kthread_work (dq.worker, &nvmeq->cmdinfo.work); + } +nvme_wait_dq (&dq, dev); +kthread_stop (kworker_task); +} +static void nvme_dev_shutdown (struct nvme_dev *dev) {int i; dev->initialized =0 ; -for (i = dev->queue_count -1 ; i >=0 ; i--) -nvme_disable_queue (dev, i);spin_lock (&dev_list_lock);list_del_init (&dev->node);spin_unlock (&dev_list_lock); -if (dev->bar) +if (!dev->bar || (dev->bar &&readl (&dev->bar->csts) ==-1 )) { +for (i = dev->queue_count -1 ; i >=0 ; i--) { +struct nvme_queue *nvmeq = dev->queues[i]; +nvme_suspend_queue (nvmeq); +nvme_clear_queue (nvmeq); + } + }else { +nvme_disable_io_queues (dev);nvme_shutdown_ctrl (dev); +nvme_disable_queue (dev,0 ); + }nvme_dev_unmap (dev); }
Signature · 1 line
1.7.10.4