Skip to contentSkip to conversation

nvme: switch to libmultipath

PATCH Superseded John Garry 1 participant 14 messages Apr 28

Subsystems (3)

v2 of 3 Superseded Latest is v3 →

lore
Share & tools
lore

John Garry

author
#0 lore raw

v2 cover nvme: switch to libmultipath

This switches the NVMe host driver to use libmultipath. That library is very heavily based on the NVMe multipath code, so the change over should hopefully be straightforward. There is often a direct replacement for functions.

The multipath functionality in nvme_ns_head and nvme_ns structures are replaced with the mpath_head and mpath_device structures.

It's hard to switch to libmulipath in a step-by-step fashion without breaking builds or functionality. To make the series reviewable, I took the approach of adding libmultipath-based code, which would initially be unused, and then finally making the full switch.

I think that more testing is required here and any help on that would be appreciated.

The series is based on 8658b6054439 (nvme/nvme-7.1) nvme-auth: Include SC_C in RVAL controller hash

Full series also available at
https://github.com/johnpgarry/linux/commits/scsi-multipath-pre-7.1-upstream-v2/

Differences to v1 (apart from porting changes for v2 libmultipath):
- always depend on LIBMULTIAPTH and drop nvme_ns_head.ns_count
- add nvme_add_ns() and nvme_delete_ns()
- init .drv_module (Nilay)
- condense code

John Garry (13):
nvme-multipath: pass NS head to nvme_mpath_revalidate_paths()
nvme-multipath: add initial support for using libmultipath
nvme-multipath: add nvme_mpath_available_path()
nvme-multipath: add nvme_mpath_{add, remove}_cdev()
nvme-multipath: add nvme_mpath_is_{disabled, optimised}
nvme-multipath: add nvme_mpath_cdev_ioctl()
nvme-multipath: add uring_cmd support
nvme-multipath: add nvme_mpath_get_iopolicy()
nvme-multipath: add nvme_mpath_synchronize()
nvme-multipath: add nvme_{add,delete}_ns()
nvme-multipath: add nvme_mpath_head_queue_if_no_path()
nvme-multipath: add nvme_mpath_get_nr_active()
nvme-multipath: switch to use libmultipath

 drivers/nvme/host/Kconfig     |   1 +
 drivers/nvme/host/core.c      |  88 ++--
 drivers/nvme/host/ioctl.c     | 108 ++--
 drivers/nvme/host/multipath.c | 897 +++++++---------------------------
 drivers/nvme/host/nvme.h      | 136 +++---
 drivers/nvme/host/pr.c        |  18 -
 drivers/nvme/host/sysfs.c     |  86 +---
 7 files changed, 337 insertions(+), 997 deletions(-)
Signature · 1 line
2.43.5
author
#1 lore raw

PATCH v2 1/13 nvme-multipath: pass NS head to nvme_mpath_revalidate_paths()

In nvme_mpath_revalidate_paths(), we are passed a NS pointer and use that to lookup the NS head and then as a iter variable.

It makes more sense pass the NS head and use a local variable for the NS iter.

---
 drivers/nvme/host/core.c      | 2 +-
 drivers/nvme/host/multipath.c | 4 ++--
 drivers/nvme/host/nvme.h      | 4 ++--
 3 files changed, 5 insertions(+), 5 deletions(-)
drivers/nvme/host/core.c +1 -1
index d62adebfdc68a..bb687295c2c67 100644
@@ -2559,7 +2559,7 @@ static int nvme_update_ns_info(struct nvme_ns *ns, struct nvme_ns_info *info)
 
 		set_capacity_and_notify(ns->head->disk, get_capacity(ns->disk));
 		set_disk_ro(ns->head->disk, nvme_ns_is_readonly(ns, info));
-		nvme_mpath_revalidate_paths(ns);
+		nvme_mpath_revalidate_paths(ns->head);
 
 		blk_mq_unfreeze_queue(ns->head->disk->queue, memflags);
 	}
drivers/nvme/host/multipath.c +2 -2
index 263161cb8ac06..e00e2842df307 100644
@@ -254,10 +254,10 @@ void nvme_mpath_clear_ctrl_paths(struct nvme_ctrl *ctrl)
 	srcu_read_unlock(&ctrl->srcu, srcu_idx);
 }
 
-void nvme_mpath_revalidate_paths(struct nvme_ns *ns)
+void nvme_mpath_revalidate_paths(struct nvme_ns_head *head)
 {
-	struct nvme_ns_head *head = ns->head;
 	sector_t capacity = get_capacity(head->disk);
+	struct nvme_ns *ns;
 	int node;
 	int srcu_idx;
 
drivers/nvme/host/nvme.h +2 -2
index ccd5e05dac98f..5de06c016b622 100644
@@ -1041,7 +1041,7 @@ void nvme_mpath_update(struct nvme_ctrl *ctrl);
 void nvme_mpath_uninit(struct nvme_ctrl *ctrl);
 void nvme_mpath_stop(struct nvme_ctrl *ctrl);
 bool nvme_mpath_clear_current_path(struct nvme_ns *ns);
-void nvme_mpath_revalidate_paths(struct nvme_ns *ns);
+void nvme_mpath_revalidate_paths(struct nvme_ns_head *head);
 void nvme_mpath_clear_ctrl_paths(struct nvme_ctrl *ctrl);
 void nvme_mpath_remove_disk(struct nvme_ns_head *head);
 void nvme_mpath_start_request(struct request *rq);
@@ -1106,7 +1106,7 @@ static inline bool nvme_mpath_clear_current_path(struct nvme_ns *ns)
 {
 	return false;
 }
-static inline void nvme_mpath_revalidate_paths(struct nvme_ns *ns)
+static inline void nvme_mpath_revalidate_paths(struct nvme_ns_head *head)
 {
 }
 static inline void nvme_mpath_clear_ctrl_paths(struct nvme_ctrl *ctrl)
Signature · 1 line
2.43.5
author
#2 lore raw

PATCH v2 2/13 nvme-multipath: add initial support for using libmultipath

In reply to John Garry

Add initial support, as follows:
- Add mpath_head_template
- Add mpath_device in nvme_ns
- Add mpath_disk pointer to head structure

Initially all the functionality which mpath_head_template points to will be unused, until the driver fully switches to libmultipath. Otherwise it's hard to do so in a step-wise fashion without breaking functionality.

Many of the libmultipath-based function added will reference the ns mpath_device, so add that now. Also add the NS head disk pointer for the same reason.

---
 drivers/nvme/host/Kconfig     | 1 +
 drivers/nvme/host/multipath.c | 4 ++++
 drivers/nvme/host/nvme.h      | 6 ++++++
 3 files changed, 11 insertions(+)
drivers/nvme/host/Kconfig +1 -0
index 31974c7dd20c9..1b3f76e781bad 100644
@@ -1,6 +1,7 @@
 # SPDX-License-Identifier: GPL-2.0-only
 config NVME_CORE
 	tristate
+	select LIBMULTIPATH
 
 config BLK_DEV_NVME
 	tristate "NVM Express block device"
drivers/nvme/host/multipath.c +4 -0
index e00e2842df307..b727d6b69f3df 100644
@@ -1388,3 +1388,7 @@ void nvme_mpath_uninit(struct nvme_ctrl *ctrl)
 	ctrl->ana_log_buf = NULL;
 	ctrl->ana_log_size = 0;
 }
+
+__maybe_unused
+static const struct mpath_head_template mpdt = {
+};
drivers/nvme/host/nvme.h +6 -0
index 5de06c016b622..f3026da0f39d9 100644
@@ -13,6 +13,7 @@
 #include <linux/blk-mq.h>
 #include <linux/sed-opal.h>
 #include <linux/fault-inject.h>
+#include <linux/multipath.h>
 #include <linux/rcupdate.h>
 #include <linux/wait.h>
 #include <linux/t10-pi.h>
@@ -554,6 +555,8 @@ struct nvme_ns_head {
 
 	u16			nr_plids;
 	u16			*plids;
+
+	struct mpath_head	*mpath_head;
 #ifdef CONFIG_NVME_MULTIPATH
 	struct bio_list		requeue_list;
 	spinlock_t		requeue_lock;
@@ -581,6 +584,7 @@ enum nvme_ns_features {
 };
 
 struct nvme_ns {
+	struct mpath_device mpath_device;
 	struct list_head list;
 
 	struct nvme_ctrl *ctrl;
@@ -607,6 +611,8 @@ struct nvme_ns {
 	struct nvme_fault_inject fault_inject;
 };
 
+#define nvme_mpath_to_ns(d) container_of(d, struct nvme_ns, mpath_device)
+
 /* NVMe ns supports metadata actions by the controller (generate/strip) */
 static inline bool nvme_ns_has_pi(struct nvme_ns_head *head)
 {
Signature · 1 line
2.43.5
author
#3 lore raw

PATCH v2 3/13 nvme-multipath: add nvme_mpath_available_path()

In reply to John Garry

This is for mpath_head_template.available_path callback.

Currently the same functionality is in nvme_available_path().

---
 drivers/nvme/host/multipath.c | 20 ++++++++++++++++++++
 1 file changed, 20 insertions(+)
drivers/nvme/host/multipath.c +20 -0
index b727d6b69f3df..cc5a3c6c272f7 100644
@@ -491,6 +491,25 @@ static bool nvme_available_path(struct nvme_ns_head *head)
 	return nvme_mpath_queue_if_no_path(head);
 }
 
+static bool nvme_mpath_available_path(struct mpath_device *mpath_device)
+{
+	struct nvme_ns *ns = nvme_mpath_to_ns(mpath_device);
+
+	if (test_bit(NVME_CTRL_FAILFAST_EXPIRED, &ns->ctrl->flags))
+		return false;
+
+	switch (nvme_ctrl_state(ns->ctrl)) {
+	case NVME_CTRL_LIVE:
+	case NVME_CTRL_RESETTING:
+	case NVME_CTRL_CONNECTING:
+		return true;
+	default:
+		break;
+	}
+
+	return false;
+}
+
 static void nvme_ns_head_submit_bio(struct bio *bio)
 {
 	struct nvme_ns_head *head = bio->bi_bdev->bd_disk->private_data;
@@ -1391,4 +1410,5 @@ void nvme_mpath_uninit(struct nvme_ctrl *ctrl)
 
 __maybe_unused
 static const struct mpath_head_template mpdt = {
+	.available_path = nvme_mpath_available_path,
 };
Signature · 1 line
2.43.5
author
#4 lore raw

PATCH v2 4/13 nvme-multipath: add nvme_mpath_{add, remove}_cdev()

In reply to John Garry

These are for mpath_head_template.add_cdev+del_cdev callbacks.

Currently the same functionality is in nvme_add_ns_cdev() and nvme_cdev_del().

---
 drivers/nvme/host/multipath.c | 22 ++++++++++++++++++++++
 1 file changed, 22 insertions(+)
drivers/nvme/host/multipath.c +22 -0
index cc5a3c6c272f7..0bdc7e0dbce50 100644
@@ -608,6 +608,26 @@ const struct block_device_operations nvme_ns_head_ops = {
 	.pr_ops		= &nvme_pr_ops,
 };
 
+static int nvme_mpath_add_cdev(struct mpath_head *mpath_head)
+{
+	struct nvme_ns_head *head = mpath_head->drvdata;
+	int ret;
+
+	mpath_head->cdev_device.parent = &head->subsys->dev;
+	ret = dev_set_name(&mpath_head->cdev_device, "ng%dn%d",
+			   head->subsys->instance, head->instance);
+	if (ret)
+		return ret;
+	ret = nvme_cdev_add(&mpath_head->cdev, &mpath_head->cdev_device,
+			    &mpath_chr_fops, THIS_MODULE);
+	return ret;
+}
+
+static void nvme_mpath_del_cdev(struct mpath_head *mpath_head)
+{
+	nvme_cdev_del(&mpath_head->cdev, &mpath_head->cdev_device);
+}
+
 static inline struct nvme_ns_head *cdev_to_ns_head(struct cdev *cdev)
 {
 	return container_of(cdev, struct nvme_ns_head, cdev);
@@ -1411,4 +1431,6 @@ void nvme_mpath_uninit(struct nvme_ctrl *ctrl)
 __maybe_unused
 static const struct mpath_head_template mpdt = {
 	.available_path = nvme_mpath_available_path,
+	.add_cdev = nvme_mpath_add_cdev,
+	.del_cdev = nvme_mpath_del_cdev,
 };
Signature · 1 line
2.43.5
author
#5 lore raw

PATCH v2 5/13 nvme-multipath: add nvme_mpath_is_{disabled, optimised}

In reply to John Garry

These are for mpath_head_template.is_{disabled, optimized} callbacks, and just call into nvme_path_is_disabled() and nvme_path_is_optimized(), respectively.

---
 drivers/nvme/host/multipath.c | 12 ++++++++++++
 1 file changed, 12 insertions(+)
drivers/nvme/host/multipath.c +12 -0
index 0bdc7e0dbce50..d9ab52a29ea8b 100644
@@ -291,6 +291,11 @@ static bool nvme_path_is_disabled(struct nvme_ns *ns)
 	return false;
 }
 
+static bool nvme_mpath_is_disabled(struct mpath_device *mpath_device)
+{
+	return nvme_path_is_disabled(nvme_mpath_to_ns(mpath_device));
+}
+
 static struct nvme_ns *__nvme_find_path(struct nvme_ns_head *head, int node)
 {
 	int found_distance = INT_MAX, fallback_distance = INT_MAX, distance;
@@ -433,6 +438,11 @@ static inline bool nvme_path_is_optimized(struct nvme_ns *ns)
 		ns->ana_state == NVME_ANA_OPTIMIZED;
 }
 
+static bool nvme_mpath_is_optimized(struct mpath_device *mpath_device)
+{
+	return nvme_path_is_optimized(nvme_mpath_to_ns(mpath_device));
+}
+
 static struct nvme_ns *nvme_numa_path(struct nvme_ns_head *head)
 {
 	int node = numa_node_id();
@@ -1433,4 +1443,6 @@ static const struct mpath_head_template mpdt = {
 	.available_path = nvme_mpath_available_path,
 	.add_cdev = nvme_mpath_add_cdev,
 	.del_cdev = nvme_mpath_del_cdev,
+	.is_disabled = nvme_mpath_is_disabled,
+	.is_optimized = nvme_mpath_is_optimized,
 };
Signature · 1 line
2.43.5
author
#6 lore raw

PATCH v2 6/13 nvme-multipath: add nvme_mpath_cdev_ioctl()

In reply to John Garry

Add nvme_mpath_cdev_ioctl(), which does the same as nvme_ns_head_chr_ioctl()

Also add nvme_mpath_ioctl_begin() and nvme_mpath_ioctl_finish() - they are special handling for how the SRCU read lock needs to be dropped for the controller command IOCTL handling. In this, nvme_mpath_ioctl_begin() takes a reference to the controller, and then mpath_chr_ioctl() will drop the SRCU read lock before calling nvme_mpath_cdev_ioctl() and finally the controller reference is dropped in nvme_mpath_ioctl_finish().

---
 drivers/nvme/host/ioctl.c     | 37 +++++++++++++++++++++++++++++++----
 drivers/nvme/host/multipath.c |  3 +++
 drivers/nvme/host/nvme.h      |  6 ++++++
 3 files changed, 42 insertions(+), 4 deletions(-)
drivers/nvme/host/ioctl.c +33 -4
index 8844bbd395159..ee99b8dbcdfff 100644
@@ -624,11 +624,9 @@ int nvme_ioctl(struct block_device *bdev, blk_mode_t mode,
 	return nvme_ns_ioctl(ns, cmd, argp, flags, open_for_write);
 }
 
-long nvme_ns_chr_ioctl(struct file *file, unsigned int cmd, unsigned long arg)
+static long _nvme_ns_chr_ioctl(struct nvme_ns *ns, unsigned int cmd,
+				unsigned long arg, bool open_for_write)
 {
-	struct nvme_ns *ns =
-		container_of(file_inode(file)->i_cdev, struct nvme_ns, cdev);
-	bool open_for_write = file->f_mode & FMODE_WRITE;
 	void __user *argp = (void __user *)arg;
 
 	if (is_ctrl_ioctl(cmd))
@@ -636,6 +634,14 @@ long nvme_ns_chr_ioctl(struct file *file, unsigned int cmd, unsigned long arg)
 	return nvme_ns_ioctl(ns, cmd, argp, 0, open_for_write);
 }
 
+long nvme_ns_chr_ioctl(struct file *file, unsigned int cmd, unsigned long arg)
+{
+	struct nvme_ns *ns =
+		container_of(file_inode(file)->i_cdev, struct nvme_ns, cdev);
+
+	return _nvme_ns_chr_ioctl(ns, cmd, arg, file->f_mode & FMODE_WRITE);
+}
+
 static int nvme_uring_cmd_checks(unsigned int issue_flags)
 {
 
@@ -690,6 +696,29 @@ int nvme_ns_chr_uring_cmd_iopoll(struct io_uring_cmd *ioucmd,
 	return 0;
 }
 #ifdef CONFIG_NVME_MULTIPATH
+long nvme_mpath_cdev_ioctl(struct mpath_device *mpath_device, unsigned int cmd,
+					unsigned long arg, bool open_for_write)
+{
+	return _nvme_ns_chr_ioctl(nvme_mpath_to_ns(mpath_device), cmd,
+			arg, open_for_write);
+}
+
+void nvme_mpath_ioctl_begin(struct mpath_device *mpath_device,
+		unsigned int cmd, void **data)
+{
+	struct nvme_ctrl *ctrl = nvme_mpath_to_ns(mpath_device)->ctrl;
+
+	if (is_ctrl_ioctl(cmd)) {
+		nvme_get_ctrl(ctrl);
+		*data = ctrl;
+	}
+}
+
+void nvme_mpath_ioctl_finish(void *opaque)
+{
+	nvme_put_ctrl(opaque);
+}
+
 static int nvme_ns_head_ctrl_ioctl(struct nvme_ns *ns, unsigned int cmd,
 		void __user *argp, struct nvme_ns_head *head, int srcu_idx,
 		bool open_for_write)
drivers/nvme/host/multipath.c +3 -0
index d9ab52a29ea8b..5e49cd716f859 100644
@@ -1445,4 +1445,7 @@ static const struct mpath_head_template mpdt = {
 	.del_cdev = nvme_mpath_del_cdev,
 	.is_disabled = nvme_mpath_is_disabled,
 	.is_optimized = nvme_mpath_is_optimized,
+	.cdev_ioctl = nvme_mpath_cdev_ioctl,
+	.ioctl_begin = nvme_mpath_ioctl_begin,
+	.ioctl_finish = nvme_mpath_ioctl_finish,
 };
drivers/nvme/host/nvme.h +6 -0
index f3026da0f39d9..1ec45cce05c9c 100644
@@ -1053,6 +1053,12 @@ void nvme_mpath_remove_disk(struct nvme_ns_head *head);
 void nvme_mpath_start_request(struct request *rq);
 void nvme_mpath_end_request(struct request *rq);
 
+long nvme_mpath_cdev_ioctl(struct mpath_device *mpath_device, unsigned int cmd,
+			unsigned long arg, bool open_for_write);
+void nvme_mpath_ioctl_begin(struct mpath_device *mpath_device,
+			unsigned int cmd, void **opaque);
+void nvme_mpath_ioctl_finish(void *opaque);
+
 static inline void nvme_trace_bio_complete(struct request *req)
 {
 	struct nvme_ns *ns = req->q->queuedata;
Signature · 1 line
2.43.5
author
#7 lore raw

PATCH v2 7/13 nvme-multipath: add uring_cmd support

In reply to John Garry

Add callback nvme_mpath_chr_uring_cmd, which is equivalent to nvme_ns_head_chr_uring_cmd().

Also fill in chr_uring_cmd_iopoll with same function as currently used, chr_uring_cmd_iopoll().

---
 drivers/nvme/host/ioctl.c     | 8 ++++++++
 drivers/nvme/host/multipath.c | 2 ++
 drivers/nvme/host/nvme.h      | 2 ++
 3 files changed, 12 insertions(+)
drivers/nvme/host/ioctl.c +8 -0
index ee99b8dbcdfff..07509a03d2ef4 100644
@@ -719,6 +719,14 @@ void nvme_mpath_ioctl_finish(void *opaque)
 	nvme_put_ctrl(opaque);
 }
 
+int nvme_mpath_chr_uring_cmd(struct mpath_device *mpath_device,
+		struct io_uring_cmd *ioucmd,
+		unsigned int issue_flags)
+{
+	return nvme_ns_uring_cmd(nvme_mpath_to_ns(mpath_device), ioucmd,
+					issue_flags);
+}
+
 static int nvme_ns_head_ctrl_ioctl(struct nvme_ns *ns, unsigned int cmd,
 		void __user *argp, struct nvme_ns_head *head, int srcu_idx,
 		bool open_for_write)
drivers/nvme/host/multipath.c +2 -0
index 5e49cd716f859..a51944ca56b1f 100644
@@ -1448,4 +1448,6 @@ static const struct mpath_head_template mpdt = {
 	.cdev_ioctl = nvme_mpath_cdev_ioctl,
 	.ioctl_begin = nvme_mpath_ioctl_begin,
 	.ioctl_finish = nvme_mpath_ioctl_finish,
+	.chr_uring_cmd = nvme_mpath_chr_uring_cmd,
+	.chr_uring_cmd_iopoll = nvme_ns_chr_uring_cmd_iopoll,
 };
drivers/nvme/host/nvme.h +2 -0
index 1ec45cce05c9c..efa868ba3fbf8 100644
@@ -1052,6 +1052,8 @@ void nvme_mpath_clear_ctrl_paths(struct nvme_ctrl *ctrl);
 void nvme_mpath_remove_disk(struct nvme_ns_head *head);
 void nvme_mpath_start_request(struct request *rq);
 void nvme_mpath_end_request(struct request *rq);
+int nvme_mpath_chr_uring_cmd(struct mpath_device *mpath_device,
+		struct io_uring_cmd *ioucmd, unsigned int issue_flags);
 
 long nvme_mpath_cdev_ioctl(struct mpath_device *mpath_device, unsigned int cmd,
 			unsigned long arg, bool open_for_write);
Signature · 1 line
2.43.5
author
#8 lore raw

PATCH v2 8/13 nvme-multipath: add nvme_mpath_get_iopolicy()

In reply to John Garry

Add a function to return the iopolicy for the head structure.

Since iopolicy for NVMe is currently per-subsysten, we add the mpath_iopolicy struct to the subsystem struct, and nvme_mpath_get_iopolicy() needs to access that member.

---
 drivers/nvme/host/multipath.c | 9 +++++++++
 drivers/nvme/host/nvme.h      | 1 +
 2 files changed, 10 insertions(+)
drivers/nvme/host/multipath.c +9 -0
index a51944ca56b1f..8025cf3270cdc 100644
@@ -1438,6 +1438,14 @@ void nvme_mpath_uninit(struct nvme_ctrl *ctrl)
 	ctrl->ana_log_size = 0;
 }
 
+static enum mpath_iopolicy_e nvme_mpath_get_iopolicy(
+				struct mpath_head *mpath_head)
+{
+	struct nvme_ns_head *head = mpath_head->drvdata;
+
+	return mpath_read_iopolicy(&head->subsys->mpath_iopolicy);
+}
+
 __maybe_unused
 static const struct mpath_head_template mpdt = {
 	.available_path = nvme_mpath_available_path,
@@ -1450,4 +1458,5 @@ static const struct mpath_head_template mpdt = {
 	.ioctl_finish = nvme_mpath_ioctl_finish,
 	.chr_uring_cmd = nvme_mpath_chr_uring_cmd,
 	.chr_uring_cmd_iopoll = nvme_ns_chr_uring_cmd_iopoll,
+	.get_iopolicy = nvme_mpath_get_iopolicy,
 };
drivers/nvme/host/nvme.h +1 -0
index efa868ba3fbf8..01a7a25bcd254 100644
@@ -502,6 +502,7 @@ struct nvme_subsystem {
 	struct ida		ns_ida;
 #ifdef CONFIG_NVME_MULTIPATH
 	enum nvme_iopolicy	iopolicy;
+	struct mpath_iopolicy	mpath_iopolicy;
 #endif
 };
 
Signature · 1 line
2.43.5
author
#9 lore raw

PATCH v2 9/13 nvme-multipath: add nvme_mpath_synchronize()

In reply to John Garry

Add a wrapper which calls into mpath_synchronize.

---
 drivers/nvme/host/nvme.h | 8 ++++++++
 1 file changed, 8 insertions(+)
drivers/nvme/host/nvme.h +8 -0
index 01a7a25bcd254..cc63ed8131c36 100644
@@ -1062,6 +1062,11 @@ void nvme_mpath_ioctl_begin(struct mpath_device *mpath_device,
 			unsigned int cmd, void **opaque);
 void nvme_mpath_ioctl_finish(void *opaque);
 
+static inline void nvme_mpath_synchronize(struct nvme_ns_head *head)
+{
+	mpath_synchronize(head->mpath_head);
+}
+
 static inline void nvme_trace_bio_complete(struct request *req)
 {
 	struct nvme_ns *ns = req->q->queuedata;
@@ -1094,6 +1099,9 @@ static inline bool nvme_ctrl_use_ana(struct nvme_ctrl *ctrl)
 {
 	return false;
 }
+static inline void nvme_mpath_synchronize(struct nvme_ns_head *head)
+{
+}
 static inline void nvme_failover_req(struct request *req)
 {
 }
Signature · 1 line
2.43.5
author
#10 lore raw

PATCH v2 10/13 nvme-multipath: add nvme_{add,delete}_ns()

In reply to John Garry

Add functions to call into the mpath_add_device() and mpath_delete_device() functions.

The per-NS gendisk pointer is used as the mpath_device disk pointer, which is used in libmultipath for references the per-path block device.

---
 drivers/nvme/host/nvme.h | 13 +++++++++++++
 1 file changed, 13 insertions(+)
drivers/nvme/host/nvme.h +13 -0
index cc63ed8131c36..b4fe7a7dd7f73 100644
@@ -1025,6 +1025,19 @@ extern const struct block_device_operations nvme_bdev_ops;
 
 void nvme_delete_ctrl_sync(struct nvme_ctrl *ctrl);
 struct nvme_ns *nvme_find_path(struct nvme_ns_head *head);
+
+static inline void nvme_add_ns(struct nvme_ns *ns)
+{
+	ns->mpath_device.disk = ns->disk;
+	ns->mpath_device.numa_node = ns->ctrl->numa_node;
+	mpath_add_device(ns->head->mpath_head, &ns->mpath_device);
+}
+
+static inline bool nvme_delete_ns(struct nvme_ns *ns)
+{
+	return mpath_delete_device(&ns->mpath_device);
+}
+
 #ifdef CONFIG_NVME_MULTIPATH
 static inline bool nvme_ctrl_use_ana(struct nvme_ctrl *ctrl)
 {
Signature · 1 line
2.43.5
author
#11 lore raw

PATCH v2 11/13 nvme-multipath: add nvme_mpath_head_queue_if_no_path()

In reply to John Garry

Add a wrapper to call into mpath_head_queue_if_no_path().

---
 drivers/nvme/host/nvme.h | 9 +++++++++
 1 file changed, 9 insertions(+)
drivers/nvme/host/nvme.h +9 -0
index b4fe7a7dd7f73..f96ad4f890797 100644
@@ -1080,6 +1080,11 @@ static inline void nvme_mpath_synchronize(struct nvme_ns_head *head)
 	mpath_synchronize(head->mpath_head);
 }
 
+static inline bool nvme_mpath_head_queue_if_no_path(struct nvme_ns_head *head)
+{
+	return mpath_head_queue_if_no_path(head->mpath_head);
+}
+
 static inline void nvme_trace_bio_complete(struct request *req)
 {
 	struct nvme_ns *ns = req->q->queuedata;
@@ -1200,6 +1205,10 @@ static inline bool nvme_mpath_queue_if_no_path(struct nvme_ns_head *head)
 {
 	return false;
 }
+static inline bool nvme_mpath_head_queue_if_no_path(struct nvme_ns_head *head)
+{
+	return false;
+}
 #endif /* CONFIG_NVME_MULTIPATH */
 
 int nvme_ns_get_unique_id(struct nvme_ns *ns, u8 id[16],
Signature · 1 line
2.43.5
author
#12 lore raw

PATCH v2 12/13 nvme-multipath: add nvme_mpath_get_nr_active()

In reply to John Garry

Add nvme_mpath_get_nr_active(), which gets the number of active requests per controller for an mpath_device.

---
 drivers/nvme/host/multipath.c | 6 ++++++
 1 file changed, 6 insertions(+)
drivers/nvme/host/multipath.c +6 -0
index 8025cf3270cdc..62e13a484ff5e 100644
@@ -1446,6 +1446,11 @@ static enum mpath_iopolicy_e nvme_mpath_get_iopolicy(
 	return mpath_read_iopolicy(&head->subsys->mpath_iopolicy);
 }
 
+static int nvme_mpath_get_nr_active(struct mpath_device *mpath_device)
+{
+	return atomic_read(&nvme_mpath_to_ns(mpath_device)->ctrl->nr_active);
+}
+
 __maybe_unused
 static const struct mpath_head_template mpdt = {
 	.available_path = nvme_mpath_available_path,
@@ -1459,4 +1464,5 @@ static const struct mpath_head_template mpdt = {
 	.chr_uring_cmd = nvme_mpath_chr_uring_cmd,
 	.chr_uring_cmd_iopoll = nvme_ns_chr_uring_cmd_iopoll,
 	.get_iopolicy = nvme_mpath_get_iopolicy,
+	.get_nr_active = nvme_mpath_get_nr_active,
 };
Signature · 1 line
2.43.5
author
#13 lore raw

PATCH v2 13/13 nvme-multipath: switch to use libmultipath

In reply to John Garry

Now that much unused libmulipath-based code has been added, do the full switch over.

The major change is that the multipath management is moved out of the nvme_ns_head structure and into the mpath_head structure.

The check for ns->head->disk is now replaced with a ns->head->mpath_head->disk check, to decide whether we are really in multipath mode.

---
 drivers/nvme/host/core.c      |  86 ++--
 drivers/nvme/host/ioctl.c     |  89 ----
 drivers/nvme/host/multipath.c | 873 +++++-----------------------------
 drivers/nvme/host/nvme.h      |  99 +---
 drivers/nvme/host/pr.c        |  18 -
 drivers/nvme/host/sysfs.c     |  86 +---
 6 files changed, 215 insertions(+), 1036 deletions(-)
drivers/nvme/host/core.c +44 -42
index bb687295c2c67..11349d2f8923d 100644
@@ -409,7 +409,7 @@ static inline enum nvme_disposition nvme_decide_disposition(struct request *req)
 	if ((nvme_req(req)->status & NVME_SCT_SC_MASK) == NVME_SC_AUTH_REQUIRED)
 		return AUTHENTICATE;
 
-	if (req->cmd_flags & REQ_NVME_MPATH) {
+	if (is_mpath_request(req)) {
 		if (nvme_is_path_error(nvme_req(req)->status) ||
 		    blk_queue_dying(req->q))
 			return FAILOVER;
@@ -442,7 +442,7 @@ static inline void __nvme_end_req(struct request *req)
 	}
 	nvme_end_req_zoned(req);
 	nvme_trace_bio_complete(req);
-	if (req->cmd_flags & REQ_NVME_MPATH)
+	if (is_mpath_request(req))
 		nvme_mpath_end_request(req);
 }
 
@@ -674,7 +674,7 @@ static void nvme_free_ns_head(struct kref *ref)
 
 	nvme_mpath_put_disk(head);
 	ida_free(&head->subsys->ns_ida, head->instance);
-	cleanup_srcu_struct(&head->srcu);
+	mpath_put_head(head->mpath_head);
 	nvme_put_subsystem(head->subsys);
 	kfree(head->plids);
 	kfree(head);
@@ -767,7 +767,7 @@ blk_status_t nvme_fail_nonready_command(struct nvme_ctrl *ctrl,
 	    state != NVME_CTRL_DELETING &&
 	    state != NVME_CTRL_DEAD &&
 	    !test_bit(NVME_CTRL_FAILFAST_EXPIRED, &ctrl->flags) &&
-	    !blk_noretry_request(rq) && !(rq->cmd_flags & REQ_NVME_MPATH))
+	    !blk_noretry_request(rq) && !is_mpath_request(rq))
 		return BLK_STS_RESOURCE;
 
 	if (!(rq->rq_flags & RQF_DONTPREP))
@@ -2523,11 +2523,12 @@ static int nvme_update_ns_info(struct nvme_ns *ns, struct nvme_ns_info *info)
 
 	if (!ret && nvme_ns_head_multipath(ns->head)) {
 		struct queue_limits *ns_lim = &ns->disk->queue->limits;
+		struct gendisk *disk = ns->head->mpath_head->disk;
 		struct queue_limits lim;
 		unsigned int memflags;
 
-		lim = queue_limits_start_update(ns->head->disk->queue);
-		memflags = blk_mq_freeze_queue(ns->head->disk->queue);
+		lim = queue_limits_start_update(disk->queue);
+		memflags = blk_mq_freeze_queue(disk->queue);
 		/*
 		 * queue_limits mixes values that are the hardware limitations
 		 * for bio splitting with what is the device configuration.
@@ -2548,20 +2549,20 @@ static int nvme_update_ns_info(struct nvme_ns *ns, struct nvme_ns_info *info)
 		lim.io_min = ns_lim->io_min;
 		lim.io_opt = ns_lim->io_opt;
 		queue_limits_stack_bdev(&lim, ns->disk->part0, 0,
-					ns->head->disk->disk_name);
+					disk->disk_name);
 		if (unsupported)
-			ns->head->disk->flags |= GENHD_FL_HIDDEN;
+			disk->flags |= GENHD_FL_HIDDEN;
 		else
 			nvme_init_integrity(ns->head, &lim, info);
 		lim.max_write_streams = ns_lim->max_write_streams;
 		lim.write_stream_granularity = ns_lim->write_stream_granularity;
-		ret = queue_limits_commit_update(ns->head->disk->queue, &lim);
+		ret = queue_limits_commit_update(disk->queue, &lim);
 
-		set_capacity_and_notify(ns->head->disk, get_capacity(ns->disk));
-		set_disk_ro(ns->head->disk, nvme_ns_is_readonly(ns, info));
+		set_capacity_and_notify(disk, get_capacity(ns->disk));
+		set_disk_ro(disk, nvme_ns_is_readonly(ns, info));
 		nvme_mpath_revalidate_paths(ns->head);
 
-		blk_mq_unfreeze_queue(ns->head->disk->queue, memflags);
+		blk_mq_unfreeze_queue(disk->queue, memflags);
 	}
 
 	return ret;
@@ -3917,14 +3918,11 @@ static int nvme_add_ns_cdev(struct nvme_ns *ns)
 static struct nvme_ns_head *nvme_alloc_ns_head(struct nvme_ctrl *ctrl,
 		struct nvme_ns_info *info)
 {
+	struct nvme_subsystem *subsys = ctrl->subsys;
 	struct nvme_ns_head *head;
 	size_t size = sizeof(*head);
 	int ret = -ENOMEM;
 
-#ifdef CONFIG_NVME_MULTIPATH
-	size += num_possible_nodes() * sizeof(struct nvme_ns *);
-#endif
-
 	head = kzalloc(size, GFP_KERNEL);
 	if (!head)
 		goto out;
@@ -3932,10 +3930,7 @@ static struct nvme_ns_head *nvme_alloc_ns_head(struct nvme_ctrl *ctrl,
 	if (ret < 0)
 		goto out_free_head;
 	head->instance = ret;
-	INIT_LIST_HEAD(&head->list);
-	ret = init_srcu_struct(&head->srcu);
-	if (ret)
-		goto out_ida_remove;
+
 	head->subsys = ctrl->subsys;
 	head->ns_id = info->nsid;
 	head->ids = info->ids;
@@ -3948,22 +3943,32 @@ static struct nvme_ns_head *nvme_alloc_ns_head(struct nvme_ctrl *ctrl,
 	if (head->ids.csi) {
 		ret = nvme_get_effects_log(ctrl, head->ids.csi, &head->effects);
 		if (ret)
-			goto out_cleanup_srcu;
+			goto out_ida_free;
 	} else
 		head->effects = ctrl->effects;
 
+	head->mpath_head = mpath_alloc_head();
+	if (IS_ERR(head->mpath_head))
+		goto out_ida_free;
+
+	head->mpath_head->drvdata = head;
+	head->mpath_head->drv_module = THIS_MODULE;
+	head->mpath_head->disk_groups = nvme_ns_attr_groups;
+	head->mpath_head->parent = &subsys->dev;
+
 	ret = nvme_mpath_alloc_disk(ctrl, head);
 	if (ret)
-		goto out_cleanup_srcu;
+		goto out_mpath_head_free;
 
 	list_add_tail(&head->entry, &ctrl->subsys->nsheads);
 
 	kref_get(&ctrl->subsys->ref);
 
 	return head;
-out_cleanup_srcu:
-	cleanup_srcu_struct(&head->srcu);
-out_ida_remove:
+
+out_mpath_head_free:
+	mpath_put_head(head->mpath_head);
+out_ida_free:
 	ida_free(&ctrl->subsys->ns_ida, head->instance);
 out_free_head:
 	kfree(head);
@@ -4002,7 +4007,7 @@ static int nvme_global_check_duplicate_ids(struct nvme_subsystem *this,
 static int nvme_init_ns_head(struct nvme_ns *ns, struct nvme_ns_info *info)
 {
 	struct nvme_ctrl *ctrl = ns->ctrl;
-	struct nvme_ns_head *head = NULL;
+	struct nvme_ns_head *head;
 	int ret;
 
 	ret = nvme_global_check_duplicate_ids(ctrl->subsys, &info->ids);
@@ -4061,7 +4066,7 @@ static int nvme_init_ns_head(struct nvme_ns *ns, struct nvme_ns_info *info)
 	} else {
 		ret = -EINVAL;
 		if ((!info->is_shared || !head->shared) &&
-		    !list_empty(&head->list)) {
+		    !mpath_head_devices_empty(head->mpath_head)) {
 			dev_err(ctrl->device,
 				"Duplicate unshared namespace %d\n",
 				info->nsid);
@@ -4083,14 +4088,10 @@ static int nvme_init_ns_head(struct nvme_ns *ns, struct nvme_ns_info *info)
 		}
 	}
 
-	list_add_tail_rcu(&ns->siblings, &head->list);
 	ns->head = head;
+	nvme_add_ns(ns);
 	mutex_unlock(&ctrl->subsys->lock);
 
-#ifdef CONFIG_NVME_MULTIPATH
-	if (cancel_delayed_work(&head->remove_work))
-		module_put(THIS_MODULE);
-#endif
 	return 0;
 
 out_put_ns_head:
@@ -4229,18 +4230,18 @@ static void nvme_alloc_ns(struct nvme_ctrl *ctrl, struct nvme_ns_info *info)
 	synchronize_srcu(&ctrl->srcu);
  out_unlink_ns:
 	mutex_lock(&ctrl->subsys->lock);
-	list_del_rcu(&ns->siblings);
-	if (list_empty(&ns->head->list)) {
+
+	if (nvme_delete_ns(ns)) {
 		list_del_init(&ns->head->entry);
 		/*
 		 * If multipath is not configured, we still create a namespace
-		 * head (nshead), but head->disk is not initialized in that
+		 * head (nshead), but mpath_head->disk is not initialized in that
 		 * case.  As a result, only a single reference to nshead is held
 		 * (via kref_init()) when it is created. Therefore, ensure that
 		 * we do not release the reference to nshead twice if head->disk
 		 * is not present.
 		 */
-		if (ns->head->disk)
+		if (ns->head->mpath_head->disk)
 			last_path = true;
 	}
 	mutex_unlock(&ctrl->subsys->lock);
@@ -4255,6 +4256,7 @@ static void nvme_alloc_ns(struct nvme_ctrl *ctrl, struct nvme_ns_info *info)
 
 static void nvme_ns_remove(struct nvme_ns *ns)
 {
+	struct nvme_ns_head *head = ns->head;
 	bool last_path = false;
 
 	if (test_and_set_bit(NVME_NS_REMOVING, &ns->flags))
@@ -4268,23 +4270,23 @@ static void nvme_ns_remove(struct nvme_ns *ns)
 	 * Ensure that !NVME_NS_READY is seen by other threads to prevent
 	 * this ns going back into current_path.
 	 */
-	synchronize_srcu(&ns->head->srcu);
+	nvme_mpath_synchronize(head);
 
 	/* wait for concurrent submissions */
 	if (nvme_mpath_clear_current_path(ns))
-		synchronize_srcu(&ns->head->srcu);
+		nvme_mpath_synchronize(head);
 
 	mutex_lock(&ns->ctrl->subsys->lock);
-	list_del_rcu(&ns->siblings);
-	if (list_empty(&ns->head->list)) {
-		if (!nvme_mpath_queue_if_no_path(ns->head))
+	if (nvme_delete_ns(ns)) {
+
+		if (!nvme_mpath_head_queue_if_no_path(ns->head))
 			list_del_init(&ns->head->entry);
 		last_path = true;
 	}
 	mutex_unlock(&ns->ctrl->subsys->lock);
 
 	/* guarantee not available in head->list */
-	synchronize_srcu(&ns->head->srcu);
+	nvme_mpath_synchronize(head);
 
 	if (!nvme_ns_head_multipath(ns->head))
 		nvme_cdev_del(&ns->cdev, &ns->cdev_device);
drivers/nvme/host/ioctl.c +0 -89

95 lines not on this page · raw message

drivers/nvme/host/multipath.c +124 -749

1,118 lines not on this page · raw message

drivers/nvme/host/nvme.h +25 -74

232 lines not on this page · raw message

drivers/nvme/host/pr.c +0 -18
index fe7dbe2648158..65e4dc74833e9 100644
@@ -49,22 +49,6 @@ static enum pr_type block_pr_type_from_nvme(enum nvme_pr_type type)
 	return 0;
 }
 
-static int nvme_send_ns_head_pr_command(struct block_device *bdev,
-		struct nvme_command *c, void *data, unsigned int data_len)
-{
-	struct nvme_ns_head *head = bdev->bd_disk->private_data;
-	int srcu_idx = srcu_read_lock(&head->srcu);
-	struct nvme_ns *ns = nvme_find_path(head);
-	int ret = -EWOULDBLOCK;
-
-	if (ns) {
-		c->common.nsid = cpu_to_le32(ns->head->ns_id);
-		ret = nvme_submit_sync_cmd(ns->queue, c, data, data_len);
-	}
-	srcu_read_unlock(&head->srcu, srcu_idx);
-	return ret;
-}
-
 static int nvme_send_ns_pr_command(struct nvme_ns *ns, struct nvme_command *c,
 		void *data, unsigned int data_len)
 {
@@ -101,8 +85,6 @@ static int __nvme_send_pr_command(struct block_device *bdev, u32 cdw10,
 	c.common.cdw10 = cpu_to_le32(cdw10);
 	c.common.cdw11 = cpu_to_le32(cdw11);
 
-	if (nvme_disk_is_ns_head(bdev->bd_disk))
-		return nvme_send_ns_head_pr_command(bdev, &c, data, data_len);
 	return nvme_send_ns_pr_command(bdev->bd_disk->private_data, &c,
 				data, data_len);
 }
drivers/nvme/host/sysfs.c +22 -64

136 lines not on this page · raw message

Signature · 1 line
2.43.5