Skip to contentSkip to conversation

nvme-fc: fix module unloads while lports still pending

PATCH Superseded James Smart 3 participants 4 messages Jun 28, 2019 – Jun 29, 2019

Subsystems (2)

v2 of 3 Superseded Latest is v3 →

lore
Share & tools
lore
#0 lore raw

PATCH v2 1/1 nvme-fc: fix module unloads while lports still pending

Current code allows the module to be unloaded even if there are pending data structures, such as localports and controllers on the localports, that have yet to hit their reference counting to remove them.

Fix by having exit entrypoint explicitly delete every controller, which in turn will remove references on the remoteports and localports causing them to be deleted as well. The exit entrypoint, after initiating the deletes, will wait for the last localport to be deleted before continuing.

Signed-off-by James Smart <jsmart2021 at gmail.com>

---
V2: replaced self-module references with code that actually deletes
the outstanding structures.

---
 drivers/nvme/host/fc.c | 55 ++++++++++++++++++++++++++++++++++++++++++++++++--
 1 file changed, 53 insertions(+), 2 deletions(-)
drivers/nvme/host/fc.c +53 -2
index 9b497d785ed7..75a81d6c9fa6 100644
@@ -204,6 +204,9 @@ static DEFINE_IDA(nvme_fc_ctrl_cnt);
 
 static struct workqueue_struct *nvme_fc_wq;
 
+static bool nvme_fc_waiting_to_unload;
+struct completion nvme_fc_unload_proceed;
+
 /*
  * These items are short-term. They will eventually be moved into
  * a generic FC class. See comments in module init.
@@ -229,6 +232,8 @@ nvme_fc_free_lport(struct kref *ref)
 	/* remove from transport list */
 	spin_lock_irqsave(&nvme_fc_lock, flags);
 	list_del(&lport->port_list);
+	if (nvme_fc_waiting_to_unload && list_empty(&nvme_fc_lport_list))
+		complete(&nvme_fc_unload_proceed);
 	spin_unlock_irqrestore(&nvme_fc_lock, flags);
 
 	ida_simple_remove(&nvme_fc_local_port_cnt, lport->localport.port_num);
@@ -237,6 +242,7 @@ nvme_fc_free_lport(struct kref *ref)
 	put_device(lport->dev);
 
 	kfree(lport);
+
 }
 
 static void
@@ -3444,6 +3450,8 @@ static int __init nvme_fc_init_module(void)
 	if (ret)
 		goto out_destroy_device;
 
+	nvme_fc_waiting_to_unload = false;
+
 	return 0;
 
 out_destroy_device:
@@ -3456,11 +3464,54 @@ static int __init nvme_fc_init_module(void)
 	return ret;
 }
 
+static void
+nvme_fc_delete_controllers(struct nvme_fc_rport *rport)
+{
+	struct nvme_fc_ctrl *ctrl;
+
+	spin_lock(&rport->lock);
+	list_for_each_entry(ctrl, &rport->ctrl_list, ctrl_list) {
+		dev_warn(ctrl->ctrl.device,
+			"NVME-FC{%d}: transport unloading: deleting ctrl\n",
+			ctrl->cnum);
+		nvme_delete_ctrl(&ctrl->ctrl);
+	}
+	spin_unlock(&rport->lock);
+}
+
+static void
+nvme_fc_cleanup_for_unload(void)
+{
+	struct nvme_fc_lport *lport;
+	struct nvme_fc_rport *rport;
+	unsigned long flags;
+
+	spin_lock_irqsave(&nvme_fc_lock, flags);
+	list_for_each_entry(lport, &nvme_fc_lport_list, port_list) {
+		list_for_each_entry(rport, &lport->endp_list, endp_list) {
+			nvme_fc_delete_controllers(rport);
+		}
+	}
+	spin_unlock_irqrestore(&nvme_fc_lock, flags);
+
+	pr_info("%s: waiting for ctlr deletes\n", __func__);
+	wait_for_completion(&nvme_fc_unload_proceed);
+	pr_info("%s: ctrl deletes complete\n", __func__);
+}
+
 static void __exit nvme_fc_exit_module(void)
 {
-	/* sanity check - all lports should be removed */
+	unsigned long flags;
+	bool need_cleanup = false;
+
+	spin_lock_irqsave(&nvme_fc_lock, flags);
+	nvme_fc_waiting_to_unload = true;
+	init_completion(&nvme_fc_unload_proceed);
 	if (!list_empty(&nvme_fc_lport_list))
-		pr_warn("%s: localport list not empty\n", __func__);
+		need_cleanup = true;
+	spin_unlock_irqrestore(&nvme_fc_lock, flags);
+	if (need_cleanup)
+		nvme_fc_cleanup_for_unload();
 
 	nvmf_unregister_transport(&nvme_fc_transport);
 
Signature · 1 line
2.13.7
#1 lore raw

PATCH v2 1/1 nvme-fc: fix module unloads while lports still pending

drivers/nvme/host/fc.c nvme_fc_free_lport(struct kref *ref) · line 246

Quoted 6 lines · James Smart
drivers/nvme/host/fc.c +1 -0
@@ -237,6 +242,7 @@ nvme_fc_free_lport(struct kref *ref)
  	put_device(lport->dev);
 
  	kfree(lport);
+
  }

Spurious whitespace?

Quoted 25 lines
drivers/nvme/host/fc.c +22 -1
+static void
+nvme_fc_cleanup_for_unload(void)
+{
+	struct nvme_fc_lport *lport;
+	struct nvme_fc_rport *rport;
+	unsigned long flags;
+
+	spin_lock_irqsave(&nvme_fc_lock, flags);
+	list_for_each_entry(lport, &nvme_fc_lport_list, port_list) {
+		list_for_each_entry(rport, &lport->endp_list, endp_list) {
+			nvme_fc_delete_controllers(rport);
+		}
+	}
+	spin_unlock_irqrestore(&nvme_fc_lock, flags);
+
+	pr_info("%s: waiting for ctlr deletes\n", __func__);
+	wait_for_completion(&nvme_fc_unload_proceed);
+	pr_info("%s: ctrl deletes complete\n", __func__);
+}
+
  static void __exit nvme_fc_exit_module(void)
  {
-	/* sanity check - all lports should be removed */
+	unsigned long flags;
+	bool need_cleanup = false;

drivers/nvme/host/fc.c static int __init nvme_fc_init_module(void) · line 3514

Quoted 10 lines
drivers/nvme/host/fc.c +8 -1
+
+	spin_lock_irqsave(&nvme_fc_lock, flags);
+	nvme_fc_waiting_to_unload = true;
+	init_completion(&nvme_fc_unload_proceed);
  	if (!list_empty(&nvme_fc_lport_list))
-		pr_warn("%s: localport list not empty\n", __func__);
+		need_cleanup = true;
+	spin_unlock_irqrestore(&nvme_fc_lock, flags);
+	if (need_cleanup)
+		nvme_fc_cleanup_for_unload();

Can the nvme_fc_lport_list list become empty after nvme_fc_exit_module() has unlocked nvme_fc_lock and before nvme_fc_cleanup_for_unload() is called? If so, shouldn't nvme_fc_cleanup_for_unload() be called unconditionally and shouldn't the wait_for_completion(&nvme_fc_unload_proceed) call depend on whether or not nvme_fc_cleanup_for_unload() has deleted any controllers?

Thanks,

Bart.

#2 lore raw

PATCH v2 1/1 nvme-fc: fix module unloads while lports still pending

Quoted 8 lines · Bart Van Assche

Review nvme_fc_free_lport(struct kref *ref) · line 245

Quoted 5 lines · James Smart
(quoted diff) +1 -0
@@ -237,6 +242,7 @@ nvme_fc_free_lport(struct kref *ref)
 ????? put_device(lport->dev);
 ? ????? kfree(lport);
+
 ? }

Spurious whitespace?

yes

Quoted 35 lines
Quoted 35 lines
(quoted diff) +20 -0
+static void
+nvme_fc_cleanup_for_unload(void)
+{
+??? struct nvme_fc_lport *lport;
+??? struct nvme_fc_rport *rport;
+??? unsigned long flags;
+
+??? spin_lock_irqsave(&nvme_fc_lock, flags);
+??? list_for_each_entry(lport, &nvme_fc_lport_list, port_list) {
+??????? list_for_each_entry(rport, &lport->endp_list, endp_list) {
+??????????? nvme_fc_delete_controllers(rport);
+??????? }
+??? }
+??? spin_unlock_irqrestore(&nvme_fc_lock, flags);
+
+??? pr_info("%s: waiting for ctlr deletes\n", __func__);
+??? wait_for_completion(&nvme_fc_unload_proceed);
+??? pr_info("%s: ctrl deletes complete\n", __func__);
+}
+

? static void __exit nvme_fc_exit_module(void)
? {
-??? /* sanity check - all lports should be removed */ +??? unsigned long flags;
+??? bool need_cleanup = false;
+
+??? spin_lock_irqsave(&nvme_fc_lock, flags);
+??? nvme_fc_waiting_to_unload = true;
+??? init_completion(&nvme_fc_unload_proceed); ????? if (!list_empty(&nvme_fc_lport_list)) -??????? pr_warn("%s: localport list not empty\n", __func__); +??????? need_cleanup = true;
+??? spin_unlock_irqrestore(&nvme_fc_lock, flags); +??? if (need_cleanup)

Quoted 8 lines
Quoted 1 line

+??????? nvme_fc_cleanup_for_unload();

Can the nvme_fc_lport_list list become empty after nvme_fc_exit_module() has unlocked nvme_fc_lock and before nvme_fc_cleanup_for_unload() is called? If so, shouldn't nvme_fc_cleanup_for_unload() be called unconditionally and shouldn't the wait_for_completion(&nvme_fc_unload_proceed) call depend on whether or not nvme_fc_cleanup_for_unload() has deleted any controllers?

It's possible which is why I init'd the completion while under the lock.?? Therefore, the removal that would make it empty, which is also under lock, will trigger the completion. In this scenario, the cleanup routine would be an empty list, fall into the wait for completion, which immediately succeeds.

-- james

#3 lore raw

PATCH v2 1/1 nvme-fc: fix module unloads while lports still pending

Quoted 58 lines · James Smart
Quoted 8 lines · Bart Van Assche

Review nvme_fc_free_lport(struct kref *ref) · line 245

Quoted 5 lines · James Smart
(quoted diff) +1 -0
@@ -237,6 +242,7 @@ nvme_fc_free_lport(struct kref *ref)
 ????? put_device(lport->dev);
 ? ????? kfree(lport);
+
 ? }

Spurious whitespace?

yes

Quoted 35 lines
Quoted 35 lines
(quoted diff) +20 -0
+static void
+nvme_fc_cleanup_for_unload(void)
+{
+??? struct nvme_fc_lport *lport;
+??? struct nvme_fc_rport *rport;
+??? unsigned long flags;
+
+??? spin_lock_irqsave(&nvme_fc_lock, flags);
+??? list_for_each_entry(lport, &nvme_fc_lport_list, port_list) {
+??????? list_for_each_entry(rport, &lport->endp_list, endp_list) {
+??????????? nvme_fc_delete_controllers(rport);
+??????? }
+??? }
+??? spin_unlock_irqrestore(&nvme_fc_lock, flags);
+
+??? pr_info("%s: waiting for ctlr deletes\n", __func__);
+??? wait_for_completion(&nvme_fc_unload_proceed);
+??? pr_info("%s: ctrl deletes complete\n", __func__);
+}
+

? static void __exit nvme_fc_exit_module(void)
? {
-??? /* sanity check - all lports should be removed */ +??? unsigned long flags;
+??? bool need_cleanup = false;
+
+??? spin_lock_irqsave(&nvme_fc_lock, flags);
+??? nvme_fc_waiting_to_unload = true;
+??? init_completion(&nvme_fc_unload_proceed); ????? if (!list_empty(&nvme_fc_lport_list)) -??????? pr_warn("%s: localport list not empty\n", __func__); +??????? need_cleanup = true;
+??? spin_unlock_irqrestore(&nvme_fc_lock, flags); +??? if (need_cleanup)

Quoted 8 lines
Quoted 1 line

+??????? nvme_fc_cleanup_for_unload();

Can the nvme_fc_lport_list list become empty after nvme_fc_exit_module() has unlocked nvme_fc_lock and before nvme_fc_cleanup_for_unload() is called? If so, shouldn't nvme_fc_cleanup_for_unload() be called unconditionally and shouldn't the wait_for_completion(&nvme_fc_unload_proceed) call depend on whether or not nvme_fc_cleanup_for_unload() has deleted any controllers?

Quoted 5 lines

It's possible which is why I init'd the completion while under the lock.?? Therefore, the removal that would make it empty, which is also under lock, will trigger the completion. In this scenario, the cleanup routine would be an empty list, fall into the wait for completion, which immediately succeeds.

Hi James,

Have you considered to use DECLARE_COMPLETION()?

Thanks,

Bart.