Skip to contentSkip to conversation

nvme-multipath: fix leak on try_module_get failure

PATCH Superseded Keith Busch 5 participants 15 messages Feb 25 – Apr 15 NVMe

v1 of 2 Superseded Latest is v2 →

lore
Share & tools
lore
#0 lore raw

PATCH 1/2 nvme-multipath: fix leak on try_module_get failure

From: Keith Busch <[email protected]>

We need to fall back to the synchronous removal if we can't get a reference on the module needed for the deferred removal.

---
 drivers/nvme/host/multipath.c | 12 +++++-------
 1 file changed, 5 insertions(+), 7 deletions(-)
drivers/nvme/host/multipath.c +5 -7
index bfcc5904e6a26..fc6800a9f7f94 100644
@@ -1310,13 +1310,11 @@ void nvme_mpath_remove_disk(struct nvme_ns_head *head)
 	if (!list_empty(&head->list))
 		goto out;
 
-	if (head->delayed_removal_secs) {
-		/*
-		 * Ensure that no one could remove this module while the head
-		 * remove work is pending.
-		 */
-		if (!try_module_get(THIS_MODULE))
-			goto out;
+	/*
+	 * Ensure that no one could remove this module while the head
+	 * remove work is pending.
+	 */
+	if (head->delayed_removal_secs && try_module_get(THIS_MODULE)) {
 		mod_delayed_work(nvme_wq, &head->remove_work,
 				head->delayed_removal_secs * HZ);
 	} else {
Signature · 1 line
2.47.3
#1 lore raw

PATCH 2/2 nvme: fix unmatched id's under delayed path deletion

From: Keith Busch <[email protected]>

The NVMe controller is allowed to reuse an NSID for a new namespace after deleting the previous namespace that had been using it. The delayed removal may have the stale namespace head in the subsystem list pending the timer, which would cause the scan to falsely report an ID mismatch error for the new namespace. Flush the pending removal work and retry to resolve this.

---
 drivers/nvme/host/core.c | 18 ++++++++++++++++++
 1 file changed, 18 insertions(+)
drivers/nvme/host/core.c +18 -0
index 3de52f1d27234..e731d3182f095 100644
@@ -3966,6 +3966,7 @@ static int nvme_global_check_duplicate_ids(struct nvme_subsystem *this,
 
 static int nvme_init_ns_head(struct nvme_ns *ns, struct nvme_ns_info *info)
 {
+	bool retry = IS_ENABLED(CONFIG_NVME_MULTIPATH);
 	struct nvme_ctrl *ctrl = ns->ctrl;
 	struct nvme_ns_head *head = NULL;
 	int ret;
@@ -4008,6 +4009,7 @@ static int nvme_init_ns_head(struct nvme_ns *ns, struct nvme_ns_info *info)
 		ctrl->quirks |= NVME_QUIRK_BOGUS_NID;
 	}
 
+again:
 	mutex_lock(&ctrl->subsys->lock);
 	head = nvme_find_ns_head(ctrl, info->nsid);
 	if (!head) {
@@ -4033,6 +4035,22 @@ static int nvme_init_ns_head(struct nvme_ns *ns, struct nvme_ns_info *info)
 			goto out_put_ns_head;
 		}
 		if (!nvme_ns_ids_equal(&head->ids, &info->ids)) {
+			/*
+			 * A newly created namespace can reuse an NSID that was
+			 * previously deleted. If the head has no active paths,
+			 * it is pending delayed removal and still occupying
+			 * this NSID in the subsystem list. Flush the removal
+			 * work to clear the stale head and retry.
+			 */
+			if (retry && list_empty(&head->list)) {
+				mutex_unlock(&ctrl->subsys->lock);
+				flush_delayed_work(&head->remove_work);
+				nvme_put_ns_head(head);
+				retry = false;
+				goto again;
+			}
+
+			WARN_ONCE(list_empty(&head->list));
 			dev_err(ctrl->device,
 				"IDs don't match for shared namespace %d\n",
 					info->nsid);
Signature · 1 line
2.47.3
maintainer
#2 lore raw

Re: [PATCH 2/2] nvme: fix unmatched id's under delayed path deletion

drivers/nvme/host/core.c static int nvme_init_ns_head(struct nvme_ns *ns, struct nvme_ns_info *info) · line 4053

Quoted 2 lines · Keith Busch
drivers/nvme/host/core.c +2 -0
+
+			WARN_ONCE(list_empty(&head->list));

Ugh, I know this should be WARN_ON_ONCE()...

Reviewed-by on v1 2/2
#3 lore raw

Re: [PATCH 2/2] nvme: fix unmatched id's under delayed path deletion

In reply to Keith Busch
Quoted 43 lines · Keith Busch

From: Keith Busch <[email protected]>

The NVMe controller is allowed to reuse an NSID for a new namespace after deleting the previous namespace that had been using it. The delayed removal may have the stale namespace head in the subsystem list pending the timer, which would cause the scan to falsely report an ID mismatch error for the new namespace. Flush the pending removal work and retry to resolve this.

---
 drivers/nvme/host/core.c | 18 ++++++++++++++++++
 1 file changed, 18 insertions(+)
drivers/nvme/host/core.c +8 -0
index 3de52f1d27234..e731d3182f095 100644
@@ -3966,6 +3966,7 @@ static int nvme_global_check_duplicate_ids(struct nvme_subsystem *this,
 
 static int nvme_init_ns_head(struct nvme_ns *ns, struct nvme_ns_info *info)
 {
+	bool retry = IS_ENABLED(CONFIG_NVME_MULTIPATH);
 	struct nvme_ctrl *ctrl = ns->ctrl;
 	struct nvme_ns_head *head = NULL;
 	int ret;
@@ -4008,6 +4009,7 @@ static int nvme_init_ns_head(struct nvme_ns *ns, struct nvme_ns_info *info)
 		ctrl->quirks |= NVME_QUIRK_BOGUS_NID;
 	}
 
+again:
 	mutex_lock(&ctrl->subsys->lock);
 	head = nvme_find_ns_head(ctrl, info->nsid);
 	if (!head) {
@@ -4033,6 +4035,22 @@ static int nvme_init_ns_head(struct nvme_ns *ns, struct nvme_ns_info *info)
 			goto out_put_ns_head;
 		}
 		if (!nvme_ns_ids_equal(&head->ids, &info->ids)) {
+			/*
+			 * A newly created namespace can reuse an NSID that was
+			 * previously deleted. If the head has no active paths,
+			 * it is pending delayed removal and still occupying
+			 * this NSID in the subsystem list. Flush the removal
+			 * work to clear the stale head and retry.

drivers/nvme/host/core.c static int nvme_init_ns_head(struct nvme_ns *ns, struct nvme_ns_info *info) · line 4053

Quoted 10 lines
drivers/nvme/host/core.c +10 -0
+			 */
+			if (retry && list_empty(&head->list)) {
+				mutex_unlock(&ctrl->subsys->lock);
+				flush_delayed_work(&head->remove_work);
+				nvme_put_ns_head(head);
+				retry = false;
+				goto again;
+			}
+
+			WARN_ONCE(list_empty(&head->list));

We need to replace WARN_ONCE with WARN_ON_ONCE (as you already mentioned in another thread), so with that change applied, this looks good to me:

#4 lore raw

Re: [PATCH 2/2] nvme: fix unmatched id's under delayed path deletion

In reply to Keith Busch
Quoted 5 lines · Keith Busch

From: Keith Busch <[email protected]>

The NVMe controller is allowed to reuse an NSID for a new namespace after deleting the previous namespace that had been using it. The delayed removal may have the stale namespace head in the subsystem list pending the timer, which

Overlong lines.

Quoted 14 lines
drivers/nvme/host/core.c +2 -0
+	bool retry = IS_ENABLED(CONFIG_NVME_MULTIPATH);
 	struct nvme_ctrl *ctrl = ns->ctrl;
 	struct nvme_ns_head *head = NULL;
 	int ret;
@@ -4008,6 +4009,7 @@ static int nvme_init_ns_head(struct nvme_ns *ns, struct nvme_ns_info *info)
 		ctrl->quirks |= NVME_QUIRK_BOGUS_NID;
 	}
 
+again:
 	mutex_lock(&ctrl->subsys->lock);
 	head = nvme_find_ns_head(ctrl, info->nsid);
 	if (!head) {
@@ -4033,6 +4035,22 @@ static int nvme_init_ns_head(struct nvme_ns *ns, struct nvme_ns_info *info)
 			goto out_put_ns_head;

drivers/nvme/host/core.c static int nvme_init_ns_head(struct nvme_ns *ns, struct nvme_ns_info *info) · line 4045

Quoted 10 lines
drivers/nvme/host/core.c +8 -0
 		}
 		if (!nvme_ns_ids_equal(&head->ids, &info->ids)) {
+			/*
+			 * A newly created namespace can reuse an NSID that was
+			 * previously deleted. If the head has no active paths,
+			 * it is pending delayed removal and still occupying
+			 * this NSID in the subsystem list. Flush the removal
+			 * work to clear the stale head and retry.
+			 */
+			if (retry && list_empty(&head->list)) {

I find the retry logic a bit odd and different from other places do in similar areas. What I'd expected is either a "nr_retries" or "did_retry" variable initialized to 0/false, then checked here to be not set (plus the IS_ENABLED() for multipath) and incremented/set below.

But independent of that, the actual logic looks fine.

maintainer
#5 lore raw

Re: [PATCH 2/2] nvme: fix unmatched id's under delayed path deletion

Quoted 7 lines · Christoph Hellwig

I find the retry logic a bit odd and different from other places do in similar areas. What I'd expected is either a "nr_retries" or "did_retry" variable initialized to 0/false, then checked here to be not set (plus the IS_ENABLED() for multipath) and incremented/set below.

But independent of that, the actual logic looks fine.

I was able to test this, and it does work when we're specifically blocking on the delayed removal. But there's a different race this doesn't handle: controller A's scan_work may depend on controller B's scan_work to finish first to remove a final reference on the deleted namespace when A is trying to add a newly created namespace that recycled the NSID.

This is looking pretty tricky to resolve. The best solution I'm coming up with so far is to have the scan_work synthesize a NVME_AER_NOTICE_NS_CHANGED event for every controller in the subsystem, then re-kick their scan work if the scan_work removed anything.

#6 lore raw

Re: [PATCH 2/2] nvme: fix unmatched id's under delayed path deletion

Quoted 16 lines · Keith Busch
Quoted 7 lines · Christoph Hellwig

I find the retry logic a bit odd and different from other places do in similar areas. What I'd expected is either a "nr_retries" or "did_retry" variable initialized to 0/false, then checked here to be not set (plus the IS_ENABLED() for multipath) and incremented/set below.

But independent of that, the actual logic looks fine.

I was able to test this, and it does work when we're specifically blocking on the delayed removal. But there's a different race this doesn't handle: controller A's scan_work may depend on controller B's scan_work to finish first to remove a final reference on the deleted namespace when A is trying to add a newly created namespace that recycled the NSID.

Quoted 4 lines

This is looking pretty tricky to resolve. The best solution I'm coming up with so far is to have the scan_work synthesize a NVME_AER_NOTICE_NS_CHANGED event for every controller in the subsystem, then re-kick their scan work if the scan_work removed anything.

So does your disk when reuse NSID, changes ns ids such as NGUID/UUID/EUI64?

Thanks,
--Nilay

maintainer
#7 lore raw

Re: [PATCH 2/2] nvme: fix unmatched id's under delayed path deletion

Quoted 7 lines · Nilay Shroff
Quoted 4 lines · Keith Busch

This is looking pretty tricky to resolve. The best solution I'm coming up with so far is to have the scan_work synthesize a NVME_AER_NOTICE_NS_CHANGED event for every controller in the subsystem, then re-kick their scan work if the scan_work removed anything.

So does your disk when reuse NSID, changes ns ids such as NGUID/UUID/EUI64?

Yes. The NSID is put back into the available pool when it's deleted, but the UID associated with it is newly generated upon creation.

#8 lore raw

Re: [PATCH 2/2] nvme: fix unmatched id's under delayed path deletion

Quoted 11 lines · Keith Busch
Quoted 7 lines · Nilay Shroff
Quoted 4 lines · Keith Busch

This is looking pretty tricky to resolve. The best solution I'm coming up with so far is to have the scan_work synthesize a NVME_AER_NOTICE_NS_CHANGED event for every controller in the subsystem, then re-kick their scan work if the scan_work removed anything.

So does your disk when reuse NSID, changes ns ids such as NGUID/UUID/EUI64?

Yes. The NSID is put back into the available pool when it's deleted, but the UID associated with it is newly generated upon creation.

Yeah, that is one of the two allowed nvme behaviors, and there's a bit to detect if the persistent ids can be reused or not.

#12 lore raw
In reply to Keith Busch
Quoted 23 lines · Keith Busch

From: Keith Busch <[email protected]>

We need to fall back to the synchronous removal if we can't get a reference on the module needed for the deferred removal.

---
  drivers/nvme/host/multipath.c | 12 +++++-------
  1 file changed, 5 insertions(+), 7 deletions(-)
drivers/nvme/host/multipath.c +0 -5
index bfcc5904e6a26..fc6800a9f7f94 100644
@@ -1310,13 +1310,11 @@ void nvme_mpath_remove_disk(struct nvme_ns_head *head)
  	if (!list_empty(&head->list))
  		goto out;
 
-	if (head->delayed_removal_secs) {
-		/*
-		 * Ensure that no one could remove this module while the head
-		 * remove work is pending.
-		 */

drivers/nvme/host/multipath.c void nvme_mpath_remove_disk(struct nvme_ns_head *head) · line 1320

Quoted 10 lines
drivers/nvme/host/multipath.c +5 -2
-		if (!try_module_get(THIS_MODULE))
-			goto out;
+	/*
+	 * Ensure that no one could remove this module while the head
+	 * remove work is pending.
+	 */
+	if (head->delayed_removal_secs && try_module_get(THIS_MODULE)) {
  		mod_delayed_work(nvme_wq, &head->remove_work,
  				head->delayed_removal_secs * HZ);
  	} else {

Are we also missing the module_put() for when we cancel the timer? AFAICS, we only put the module for after the disk is removed.

Maybe we need something like this:

Subject: [PATCH] nvme-multipath: put module reference when delayed removal timer is canceled

The delayed removal timer is canceled when a NS appears. However, we do not put the module reference grabbed in nvme_mpath_remove_disk(), so fix that.

drivers/nvme/host/core.c +2 -1
index 1e33af94c24b..1b445830358e 100644
@@ -4083,7 +4083,8 @@ static int nvme_init_ns_head(struct nvme_ns *ns, struct nvme_ns_info *info)
 	mutex_unlock(&ctrl->subsys->lock);
 
 #ifdef CONFIG_NVME_MULTIPATH
-	cancel_delayed_work(&head->remove_work);
+	if (cancel_delayed_work_sync(&head->remove_work))
+		module_put(THIS_MODULE);
 #endif
 	return 0;
 
maintainer
#13 lore raw
Quoted 2 lines · John Garry

Are we also missing the module_put() for when we cancel the timer? AFAICS, we only put the module for after the disk is removed.

Yes, I believe you are correct.

Quoted 15 lines

Maybe we need something like this:

Subject: [PATCH] nvme-multipath: put module reference when delayed removal timer is canceled

The delayed removal timer is canceled when a NS appears. However, we do not put the module reference grabbed in nvme_mpath_remove_disk(), so fix that.

drivers/nvme/host/core.c +0 -0
index 1e33af94c24b..1b445830358e 100644

drivers/nvme/host/core.c static int nvme_init_ns_head(struct nvme_ns *ns, · line 4089

Quoted 8 lines
drivers/nvme/host/core.c +2 -1
@@ -4083,7 +4083,8 @@ static int nvme_init_ns_head(struct nvme_ns *ns,
 struct nvme_ns_info *info)
 	mutex_unlock(&ctrl->subsys->lock);
 
 #ifdef CONFIG_NVME_MULTIPATH
-	cancel_delayed_work(&head->remove_work);
+	if (cancel_delayed_work_sync(&head->remove_work))
+		module_put(THIS_MODULE);
 #endif

I don't think we need the _sync() variant here, but the check and module_put() look good.

#14 lore raw
Quoted 11 lines · Keith Busch

drivers/nvme/host/core.c static int nvme_init_ns_head(struct nvme_ns *ns, · line 4089

Quoted 9 lines
drivers/nvme/host/core.c +2 -1
@@ -4083,7 +4083,8 @@ static int nvme_init_ns_head(struct nvme_ns *ns,
 struct nvme_ns_info *info)
  	mutex_unlock(&ctrl->subsys->lock);
 
  #ifdef CONFIG_NVME_MULTIPATH
-	cancel_delayed_work(&head->remove_work);
+	if (cancel_delayed_work_sync(&head->remove_work))
+		module_put(THIS_MODULE);
  #endif

I don't think we need the _sync() variant here, but the check and module_put() look good.

Yeah, I don't think that the _sync is required either.

cheers