Skip to contentSkip to conversation

nvme-multipath: path selection fixes

PATCH Superseded Hannes Reinecke 3 participants 6 messages Jul 3, 2019 – Jul 4, 2019 NVMe

v1 of 2 Superseded Latest is v2 →

lore
Share & tools
lore
#0 lore raw

cover nvme-multipath: path selection fixes

Hi all,

during analysing a mysterious failure due to blk_queue_split() (again) I've found that the multipath path selection might return invalid namespaces under certain circumstances.
This patchset fixes those scenarios.

As usual, reviews and comments are welcome.

Hannes Reinecke (2):
nvme-multipath: check singular list in vme_round_robin_path()
nvme-multipath: do not select namespaces which are about to be removed

 drivers/nvme/host/multipath.c | 13 ++++++++++---
 1 file changed, 10 insertions(+), 3 deletions(-)
Signature · 1 line
2.16.4
#1 lore raw

PATCH 1/2 nvme-multipath: check singular list in vme_round_robin_path()

When we have a singular list in nvme_round_robin_path() we still need to check its validity.

Signed-off-by Hannes Reinecke <hare at suse.com>
---
 drivers/nvme/host/multipath.c | 6 +++++-
 1 file changed, 5 insertions(+), 1 deletion(-)
drivers/nvme/host/multipath.c +5 -1
index 499acf07d61a..c8cc82639327 100644
@@ -178,8 +178,12 @@ static struct nvme_ns *nvme_round_robin_path(struct nvme_ns_head *head,
 {
 	struct nvme_ns *ns, *found, *fallback = NULL;
 
-	if (list_is_singular(&head->list))
+	if (list_is_singular(&head->list)) {
+		if (old->ctrl->state != NVME_CTRL_LIVE ||
+		    test_bit(NVME_NS_ANA_PENDING, &old->flags))
+			return NULL;
 		return old;
+	}
 
 	for (ns = nvme_next_ns(head, old);
 	     ns != old;
Signature · 1 line
2.16.4
#2 lore raw

PATCH 2/2 nvme-multipath: do not select namespaces which are about to be removed

In reply to Hannes Reinecke

nvme_ns_remove() will first set the NVME_NS_REMOVING flag before removing it from the list at the very last step.
So to avoid selecting a namespace in nvme_find_path() which is about to be removed check the NVME_NS_REMOVING flag, too, when selecting a new path.

Signed-off-by Hannes Reinecke <hare at suse.com>
---
 drivers/nvme/host/multipath.c | 9 ++++++---
 1 file changed, 6 insertions(+), 3 deletions(-)
drivers/nvme/host/multipath.c +6 -3
index c8cc82639327..1f6105a5c596 100644
@@ -130,7 +130,8 @@ static struct nvme_ns *__nvme_find_path(struct nvme_ns_head *head, int node)
 
 	list_for_each_entry_rcu(ns, &head->list, siblings) {
 		if (ns->ctrl->state != NVME_CTRL_LIVE ||
-		    test_bit(NVME_NS_ANA_PENDING, &ns->flags))
+		    test_bit(NVME_NS_ANA_PENDING, &ns->flags) ||
+		    test_bit(NVME_NS_REMOVING, &ns->flags))
 			continue;
 
 		if (READ_ONCE(head->subsys->iopolicy) == NVME_IOPOLICY_NUMA)
@@ -180,7 +181,8 @@ static struct nvme_ns *nvme_round_robin_path(struct nvme_ns_head *head,
 
 	if (list_is_singular(&head->list)) {
 		if (old->ctrl->state != NVME_CTRL_LIVE ||
-		    test_bit(NVME_NS_ANA_PENDING, &old->flags))
+		    test_bit(NVME_NS_ANA_PENDING, &old->flags)||
+		    test_bit(NVME_NS_REMOVING, &old->flags))
 			return NULL;
 		return old;
 	}
@@ -189,7 +191,8 @@ static struct nvme_ns *nvme_round_robin_path(struct nvme_ns_head *head,
 	     ns != old;
 	     ns = nvme_next_ns(head, ns)) {
 		if (ns->ctrl->state != NVME_CTRL_LIVE ||
-		    test_bit(NVME_NS_ANA_PENDING, &ns->flags))
+		    test_bit(NVME_NS_ANA_PENDING, &ns->flags) ||
+		    test_bit(NVME_NS_REMOVING, &ns->flags))
 			continue;
 
 		if (ns->ana_state == NVME_ANA_OPTIMIZED) {
Signature · 1 line
2.16.4
#3 lore raw

PATCH 2/2 nvme-multipath: do not select namespaces which are about to be removed

Quoted 25 lines · Hannes Reinecke
drivers/nvme/host/multipath.c +4 -2
index c8cc82639327..1f6105a5c596 100644
@@ -130,7 +130,8 @@ static struct nvme_ns *__nvme_find_path(struct nvme_ns_head *head, int node)
 
 	list_for_each_entry_rcu(ns, &head->list, siblings) {
 		if (ns->ctrl->state != NVME_CTRL_LIVE ||
-		    test_bit(NVME_NS_ANA_PENDING, &ns->flags))
+		    test_bit(NVME_NS_ANA_PENDING, &ns->flags) ||
+		    test_bit(NVME_NS_REMOVING, &ns->flags))
 			continue;
 
 		if (READ_ONCE(head->subsys->iopolicy) == NVME_IOPOLICY_NUMA)
@@ -180,7 +181,8 @@ static struct nvme_ns *nvme_round_robin_path(struct nvme_ns_head *head,
 
 	if (list_is_singular(&head->list)) {
 		if (old->ctrl->state != NVME_CTRL_LIVE ||
-		    test_bit(NVME_NS_ANA_PENDING, &old->flags))
+		    test_bit(NVME_NS_ANA_PENDING, &old->flags)||
+		    test_bit(NVME_NS_REMOVING, &old->flags))
 			return NULL;
 		return old;
 	}

drivers/nvme/host/multipath.c static struct nvme_ns *nvme_round_robin_path(struct nvme_ns_head *head, · line 196

Quoted 7 lines
drivers/nvme/host/multipath.c +2 -1
@@ -189,7 +191,8 @@ static struct nvme_ns *nvme_round_robin_path(struct nvme_ns_head *head,
 	     ns != old;
 	     ns = nvme_next_ns(head, ns)) {
 		if (ns->ctrl->state != NVME_CTRL_LIVE ||
-		    test_bit(NVME_NS_ANA_PENDING, &ns->flags))
+		    test_bit(NVME_NS_ANA_PENDING, &ns->flags) ||
+		    test_bit(NVME_NS_REMOVING, &ns->flags))
 			continue;

I think we clearly need a patch before the two patches in your series that factors this check into a little helper with a descriptive name. Also doesn't nvme_path_is_optimized also need to have these checks while we are at it?

maintainer
#4 lore raw

PATCH 2/2 nvme-multipath: do not select namespaces which are about to be removed

Quoted 30 lines
Quoted 30 lines
drivers/nvme/host/multipath.c +4 -3
index c8cc82639327..1f6105a5c596 100644
@@ -130,7 +130,8 @@ static struct nvme_ns *__nvme_find_path(struct nvme_ns_head *head, int node)
 
  	list_for_each_entry_rcu(ns, &head->list, siblings) {
  		if (ns->ctrl->state != NVME_CTRL_LIVE ||
-		    test_bit(NVME_NS_ANA_PENDING, &ns->flags))
+		    test_bit(NVME_NS_ANA_PENDING, &ns->flags) ||
+		    test_bit(NVME_NS_REMOVING, &ns->flags))
  			continue;
 
  		if (READ_ONCE(head->subsys->iopolicy) == NVME_IOPOLICY_NUMA)
@@ -180,7 +181,8 @@ static struct nvme_ns *nvme_round_robin_path(struct nvme_ns_head *head,
 
  	if (list_is_singular(&head->list)) {
  		if (old->ctrl->state != NVME_CTRL_LIVE ||
-		    test_bit(NVME_NS_ANA_PENDING, &old->flags))
+		    test_bit(NVME_NS_ANA_PENDING, &old->flags)||
+		    test_bit(NVME_NS_REMOVING, &old->flags))
  			return NULL;
  		return old;
  	}
@@ -189,7 +191,8 @@ static struct nvme_ns *nvme_round_robin_path(struct nvme_ns_head *head,
  	     ns != old;
  	     ns = nvme_next_ns(head, ns)) {
  		if (ns->ctrl->state != NVME_CTRL_LIVE ||
-		    test_bit(NVME_NS_ANA_PENDING, &ns->flags))
Quoted 8 lines

drivers/nvme/host/multipath.c static struct nvme_ns *__nvme_find_path(struct nvme_ns_head *head, int node) · line 135

Quoted 3 lines
drivers/nvme/host/multipath.c +2 -0
+		    test_bit(NVME_NS_ANA_PENDING, &ns->flags) ||
+		    test_bit(NVME_NS_REMOVING, &ns->flags))
  			continue;

I think we clearly need a patch before the two patches in your series that factors this check into a little helper with a descriptive name. Also doesn't nvme_path_is_optimized also need to have these checks while we are at it?

I definitely agree here.

BTW, while we're on this, do we need the flags check to be atomic? or can we micro-optimize with a simple flag arithmetic?

#5 lore raw

PATCH 2/2 nvme-multipath: do not select namespaces which are about to be removed

Quoted 40 lines · Sagi Grimberg
Quoted 31 lines
Quoted 31 lines
drivers/nvme/host/multipath.c +0 -0
index c8cc82639327..1f6105a5c596 100644
@@ -130,7 +130,8 @@ static struct nvme_ns *__nvme_find_path(struct
 nvme_ns_head *head, int node)
 ? ????? list_for_each_entry_rcu(ns, &head->list, siblings) {

????????? if (ns->ctrl->state != NVME_CTRL_LIVE || -??????????? test_bit(NVME_NS_ANA_PENDING, &ns->flags)) +??????????? test_bit(NVME_NS_ANA_PENDING, &ns->flags) || +??????????? test_bit(NVME_NS_REMOVING, &ns->flags)) ????????????? continue;
? ????????? if (READ_ONCE(head->subsys->iopolicy) == NVME_IOPOLICY_NUMA)

drivers/nvme/host/multipath.c +0 -0
@@ -180,7 +181,8 @@ static struct nvme_ns
 *nvme_round_robin_path(struct nvme_ns_head *head,
 ? ????? if (list_is_singular(&head->list)) {

????????? if (old->ctrl->state != NVME_CTRL_LIVE || -??????????? test_bit(NVME_NS_ANA_PENDING, &old->flags)) +??????????? test_bit(NVME_NS_ANA_PENDING, &old->flags)|| +??????????? test_bit(NVME_NS_REMOVING, &old->flags)) ????????????? return NULL;
????????? return old;
????? }

drivers/nvme/host/multipath.c +0 -0
@@ -189,7 +191,8 @@ static struct nvme_ns
 *nvme_round_robin_path(struct nvme_ns_head *head,
 ?????????? ns != old;
 ?????????? ns = nvme_next_ns(head, ns)) {

????????? if (ns->ctrl->state != NVME_CTRL_LIVE || -??????????? test_bit(NVME_NS_ANA_PENDING, &ns->flags))

Quoted 8 lines
Quoted 3 lines

+??????????? test_bit(NVME_NS_ANA_PENDING, &ns->flags) || +??????????? test_bit(NVME_NS_REMOVING, &ns->flags)) ????????????? continue;

I think we clearly need a patch before the two patches in your series that factors this check into a little helper with a descriptive name.? Also doesn't nvme_path_is_optimized also need to have these checks while we are at it?

Quoted 4 lines

I definitely agree here.

BTW, while we're on this, do we need the flags check to be atomic? or can we micro-optimize with a simple flag arithmetic?

Depends on your definition of 'atomic' here.
No, the _combined_ check does not need to be atomic (ie we do not need to take a lock around the check here) as the flags are set in two wildly different places.
Yes, the check itself needs to be atomic (ie any update to the flags bitmap needs to be visible) as this check protect against a race condition. We could try to micro-optimize here with using bitmap_and(), but then this would probably amount to out-guess the compiler, and I'd rather not attempt it for the sake of readability.

Cheers,

Hannes

Signature · 5 lines
Dr. Hannes Reinecke		   Teamlead Storage & Networking
hare at suse.de			               +49 911 74053 688
SUSE LINUX GmbH, Maxfeldstr. 5, 90409 N?rnberg
GF: Felix Imend?rffer, Mary Higgins, Sri Rasiah
HRB 21284 (AG N?rnberg)