nvme-multipath: path selection fixes
cover nvme-multipath: path selection fixes
Hi all,
during analysing a mysterious failure due to blk_queue_split() (again) I've found that the multipath path selection might return invalid namespaces under certain circumstances.
This patchset fixes those scenarios.
As usual, reviews and comments are welcome.
Hannes Reinecke (2):
nvme-multipath: check singular list in vme_round_robin_path()
nvme-multipath: do not select namespaces which are about to be removed
drivers/nvme/host/multipath.c | 13 ++++++++++--- 1 file changed, 10 insertions(+), 3 deletions(-)
Signature · 1 line
2.16.4
PATCH 1/2 nvme-multipath: check singular list in vme_round_robin_path()
When we have a singular list in nvme_round_robin_path() we still need to check its validity.
--- drivers/nvme/host/multipath.c | 6 +++++- 1 file changed, 5 insertions(+), 1 deletion(-)
drivers/nvme/host/multipath.c +5 -1
@@ -178,8 +178,12 @@ static struct nvme_ns *nvme_round_robin_path(struct nvme_ns_head *head, {struct nvme_ns *ns, *found, *fallback =NULL ; -if (list_is_singular (&head->list)) +if (list_is_singular (&head->list)) { +if (old->ctrl->state !=NVME_CTRL_LIVE || +test_bit (NVME_NS_ANA_PENDING , &old->flags)) +return NULL ;return old; + } for (ns = nvme_next_ns(head, old); ns != old;
Signature · 1 line
2.16.4
PATCH 2/2 nvme-multipath: do not select namespaces which are about to be removed
In reply to Hannes Reineckenvme_ns_remove() will first set the NVME_NS_REMOVING flag before removing it from the list at the very last step.
So to avoid selecting a namespace in nvme_find_path() which is about to be removed check the NVME_NS_REMOVING flag, too, when selecting a new path.
--- drivers/nvme/host/multipath.c | 9 ++++++--- 1 file changed, 6 insertions(+), 3 deletions(-)
drivers/nvme/host/multipath.c +6 -3
@@ -130,7 +130,8 @@ static struct nvme_ns *__nvme_find_path(struct nvme_ns_head *head, int node)list_for_each_entry_rcu (ns, &head->list, siblings) {if (ns->ctrl->state !=NVME_CTRL_LIVE || -test_bit (NVME_NS_ANA_PENDING , &ns->flags)) +test_bit (NVME_NS_ANA_PENDING , &ns->flags) || +test_bit (NVME_NS_REMOVING , &ns->flags))continue ;if (READ_ONCE (head->subsys->iopolicy) ==NVME_IOPOLICY_NUMA )
@@ -180,7 +181,8 @@ static struct nvme_ns *nvme_round_robin_path(struct nvme_ns_head *head,if (list_is_singular (&head->list)) {if (old->ctrl->state !=NVME_CTRL_LIVE || -test_bit (NVME_NS_ANA_PENDING , &old->flags)) +test_bit (NVME_NS_ANA_PENDING , &old->flags)|| +test_bit (NVME_NS_REMOVING , &old->flags))return NULL ;return old; }
@@ -189,7 +191,8 @@ static struct nvme_ns *nvme_round_robin_path(struct nvme_ns_head *head, ns != old; ns = nvme_next_ns(head, ns)) {if (ns->ctrl->state !=NVME_CTRL_LIVE || -test_bit (NVME_NS_ANA_PENDING , &ns->flags)) +test_bit (NVME_NS_ANA_PENDING , &ns->flags) || +test_bit (NVME_NS_REMOVING , &ns->flags))continue ;if (ns->ana_state ==NVME_ANA_OPTIMIZED ) {
Signature · 1 line
2.16.4
Christoph Hellwig
maintainerPATCH 2/2 nvme-multipath: do not select namespaces which are about to be removed
Quoted 25 lines · Hannes Reinecke
drivers/nvme/host/multipath.c +4
-2@@ -130,7 +130,8 @@ static struct nvme_ns *__nvme_find_path(struct nvme_ns_head *head, int node)list_for_each_entry_rcu (ns, &head->list, siblings) {if (ns->ctrl->state !=NVME_CTRL_LIVE || -test_bit (NVME_NS_ANA_PENDING , &ns->flags)) +test_bit (NVME_NS_ANA_PENDING , &ns->flags) || +test_bit (NVME_NS_REMOVING , &ns->flags))continue ;if (READ_ONCE (head->subsys->iopolicy) ==NVME_IOPOLICY_NUMA )@@ -180,7 +181,8 @@ static struct nvme_ns *nvme_round_robin_path(struct nvme_ns_head *head,if (list_is_singular (&head->list)) {if (old->ctrl->state !=NVME_CTRL_LIVE || -test_bit (NVME_NS_ANA_PENDING , &old->flags)) +test_bit (NVME_NS_ANA_PENDING , &old->flags)|| +test_bit (NVME_NS_REMOVING , &old->flags))return NULL ;return old; }
drivers/nvme/host/multipath.c static struct nvme_ns *nvme_round_robin_path(struct nvme_ns_head *head, · line 196
Quoted 7 lines
drivers/nvme/host/multipath.c +2
-1@@ -189,7 +191,8 @@ static struct nvme_ns *nvme_round_robin_path(struct nvme_ns_head *head, ns != old; ns = nvme_next_ns(head, ns)) {if (ns->ctrl->state !=NVME_CTRL_LIVE || -test_bit (NVME_NS_ANA_PENDING , &ns->flags)) +test_bit (NVME_NS_ANA_PENDING , &ns->flags) || +test_bit (NVME_NS_REMOVING , &ns->flags))continue ;
I think we clearly need a patch before the two patches in your series that factors this check into a little helper with a descriptive name. Also doesn't nvme_path_is_optimized also need to have these checks while we are at it?
Sagi Grimberg
maintainerPATCH 2/2 nvme-multipath: do not select namespaces which are about to be removed
Quoted 30 lines
Quoted 30 lines
drivers/nvme/host/multipath.c +4
-3@@ -130,7 +130,8 @@ static struct nvme_ns *__nvme_find_path(struct nvme_ns_head *head, int node)list_for_each_entry_rcu (ns, &head->list, siblings) {if (ns->ctrl->state !=NVME_CTRL_LIVE || -test_bit (NVME_NS_ANA_PENDING , &ns->flags)) +test_bit (NVME_NS_ANA_PENDING , &ns->flags) || +test_bit (NVME_NS_REMOVING , &ns->flags))continue ;if (READ_ONCE (head->subsys->iopolicy) ==NVME_IOPOLICY_NUMA )@@ -180,7 +181,8 @@ static struct nvme_ns *nvme_round_robin_path(struct nvme_ns_head *head,if (list_is_singular (&head->list)) {if (old->ctrl->state !=NVME_CTRL_LIVE || -test_bit (NVME_NS_ANA_PENDING , &old->flags)) +test_bit (NVME_NS_ANA_PENDING , &old->flags)|| +test_bit (NVME_NS_REMOVING , &old->flags))return NULL ;return old; }@@ -189,7 +191,8 @@ static struct nvme_ns *nvme_round_robin_path(struct nvme_ns_head *head, ns != old; ns = nvme_next_ns(head, ns)) { if (ns->ctrl->state != NVME_CTRL_LIVE || -test_bit (NVME_NS_ANA_PENDING , &ns->flags))
Quoted 8 lines
drivers/nvme/host/multipath.c static struct nvme_ns *__nvme_find_path(struct nvme_ns_head *head, int node) · line 135
Quoted 3 lines
drivers/nvme/host/multipath.c +2
-0+test_bit (NVME_NS_ANA_PENDING , &ns->flags) || + test_bit(NVME_NS_REMOVING, &ns->flags)) continue;I think we clearly need a patch before the two patches in your series that factors this check into a little helper with a descriptive name. Also doesn't
nvme_path_is_optimizedalso need to have these checks while we are at it?
I definitely agree here.
BTW, while we're on this, do we need the flags check to be atomic? or can we micro-optimize with a simple flag arithmetic?
PATCH 2/2 nvme-multipath: do not select namespaces which are about to be removed
Quoted 40 lines · Sagi Grimberg
Quoted 31 lines
Quoted 31 lines
drivers/nvme/host/multipath.c +0
-0@@ -130,7 +130,8 @@ static struct nvme_ns *__nvme_find_path(struct nvme_ns_head *head, int node) ? ????? list_for_each_entry_rcu(ns, &head->list, siblings) {????????? if (
ns->ctrl->state!=NVME_CTRL_LIVE|| -???????????test_bit(NVME_NS_ANA_PENDING, &ns->flags)) +???????????test_bit(NVME_NS_ANA_PENDING, &ns->flags)|| +???????????test_bit(NVME_NS_REMOVING, &ns->flags)) ????????????? continue;
? ????????? if (READ_ONCE(head->subsys->iopolicy)==NVME_IOPOLICY_NUMA)drivers/nvme/host/multipath.c +0
-0@@ -180,7 +181,8 @@ static struct nvme_ns *nvme_round_robin_path(struct nvme_ns_head *head, ? ????? if (list_is_singular(&head->list)) {????????? if (
old->ctrl->state!=NVME_CTRL_LIVE|| -???????????test_bit(NVME_NS_ANA_PENDING, &old->flags)) +???????????test_bit(NVME_NS_ANA_PENDING, &old->flags)|| +???????????test_bit(NVME_NS_REMOVING, &old->flags)) ????????????? return NULL;
????????? return old;
????? }drivers/nvme/host/multipath.c +0
-0@@ -189,7 +191,8 @@ static struct nvme_ns *nvme_round_robin_path(struct nvme_ns_head *head, ?????????? ns != old; ?????????? ns = nvme_next_ns(head, ns)) {????????? if (
ns->ctrl->state!=NVME_CTRL_LIVE|| -???????????test_bit(NVME_NS_ANA_PENDING, &ns->flags))Quoted 8 lines
Quoted 3 lines
+???????????
test_bit(NVME_NS_ANA_PENDING, &ns->flags)|| +???????????test_bit(NVME_NS_REMOVING, &ns->flags)) ????????????? continue;I think we clearly need a patch before the two patches in your series that factors this check into a little helper with a descriptive name.? Also doesn't
nvme_path_is_optimizedalso need to have these checks while we are at it?
Quoted 4 lines
I definitely agree here.
BTW, while we're on this, do we need the flags check to be atomic? or can we micro-optimize with a simple flag arithmetic?
Depends on your definition of 'atomic' here.
No, the _combined_ check does not need to be atomic (ie we do not need to take a lock around the check here) as the flags are set in two wildly different places.
Yes, the check itself needs to be atomic (ie any update to the flags bitmap needs to be visible) as this check protect against a race condition. We could try to micro-optimize here with using bitmap_and(), but then this would probably amount to out-guess the compiler, and I'd rather not attempt it for the sake of readability.
Cheers,
Hannes
Signature · 5 lines
Dr. Hannes Reinecke Teamlead Storage & Networking hare at suse.de +49 911 74053 688 SUSE LINUX GmbH, Maxfeldstr. 5, 90409 N?rnberg GF: Felix Imend?rffer, Mary Higgins, Sri Rasiah HRB 21284 (AG N?rnberg)