Skip to contentSkip to conversation

nvme: multipath: Implemented new iopolicy "queue-depth"

PATCH In review Ewan D. Milne 6 participants 24 messages Nov 7, 2023 – May 9, 2024 NVMe

lore
Share & tools
lore
#0 lore raw

PATCH 1/3 nvme: multipath: Implemented new iopolicy "queue-depth"

The existing iopolicies are inefficient in some cases, such as the presence of a path with high latency. The round-robin policy would use that path equally with faster paths, which results in sub-optimal performance.

The queue-depth policy instead sends I/O requests down the path with the least amount of requests in its request queue. Paths with lower latency will clear requests more quickly and have less requests in their queues compared to "bad" paths. The aim is to use those paths the most to bring down overall latency.

This implementation adds an atomic variable to the nvme_ctrl struct to represent the queue depth. It is updated each time a request specific to that controller starts or ends.

[edm: patch developed by Thomas Song @ Pure Storage, fixed whitespace
and compilation warnings, updated MODULE_PARM description, and fixed potential issue with ->current_path[] being used]

---
 drivers/nvme/host/multipath.c | 59 +++++++++++++++++++++++++++++++++--
 drivers/nvme/host/nvme.h      |  2 ++
 2 files changed, 58 insertions(+), 3 deletions(-)
drivers/nvme/host/multipath.c +56 -3
index 0a88d7bdc5e3..4c2690cddef3 100644
@@ -17,6 +17,7 @@ MODULE_PARM_DESC(multipath,
 static const char *nvme_iopolicy_names[] = {
 	[NVME_IOPOLICY_NUMA]	= "numa",
 	[NVME_IOPOLICY_RR]	= "round-robin",
+	[NVME_IOPOLICY_QD]      = "queue-depth",
 };
 
 static int iopolicy = NVME_IOPOLICY_NUMA;
@@ -29,6 +30,8 @@ static int nvme_set_iopolicy(const char *val, const struct kernel_param *kp)
 		iopolicy = NVME_IOPOLICY_NUMA;
 	else if (!strncmp(val, "round-robin", 11))
 		iopolicy = NVME_IOPOLICY_RR;
+	else if (!strncmp(val, "queue-depth", 11))
+		iopolicy = NVME_IOPOLICY_QD;
 	else
 		return -EINVAL;
 
@@ -43,7 +46,7 @@ static int nvme_get_iopolicy(char *buf, const struct kernel_param *kp)
 module_param_call(iopolicy, nvme_set_iopolicy, nvme_get_iopolicy,
 	&iopolicy, 0644);
 MODULE_PARM_DESC(iopolicy,
-	"Default multipath I/O policy; 'numa' (default) or 'round-robin'");
+	"Default multipath I/O policy; 'numa' (default) , 'round-robin' or 'queue-depth'");
 
 void nvme_mpath_default_iopolicy(struct nvme_subsystem *subsys)
 {
@@ -130,6 +133,7 @@ void nvme_mpath_start_request(struct request *rq)
 	if (!blk_queue_io_stat(disk->queue) || blk_rq_is_passthrough(rq))
 		return;
 
+	atomic_inc(&ns->ctrl->nr_active);
 	nvme_req(rq)->flags |= NVME_MPATH_IO_STATS;
 	nvme_req(rq)->start_time = bdev_start_io_acct(disk->part0, req_op(rq),
 						      jiffies);
@@ -142,6 +146,8 @@ void nvme_mpath_end_request(struct request *rq)
 
 	if (!(nvme_req(rq)->flags & NVME_MPATH_IO_STATS))
 		return;
+
+	atomic_dec(&ns->ctrl->nr_active);
 	bdev_end_io_acct(ns->head->disk->part0, req_op(rq),
 			 blk_rq_bytes(rq) >> SECTOR_SHIFT,
 			 nvme_req(rq)->start_time);
@@ -329,6 +335,40 @@ static struct nvme_ns *nvme_round_robin_path(struct nvme_ns_head *head,
 	return found;
 }
 
+static struct nvme_ns *nvme_queue_depth_path(struct nvme_ns_head *head)
+{
+	struct nvme_ns *best_opt = NULL, *best_nonopt = NULL, *ns;
+	unsigned int min_depth_opt = UINT_MAX, min_depth_nonopt = UINT_MAX;
+	unsigned int depth;
+
+	list_for_each_entry_rcu(ns, &head->list, siblings) {
+		if (nvme_path_is_disabled(ns))
+			continue;
+
+		depth = atomic_read(&ns->ctrl->nr_active);
+
+		switch (ns->ana_state) {
+		case NVME_ANA_OPTIMIZED:
+			if (depth < min_depth_opt) {
+				min_depth_opt = depth;
+				best_opt = ns;
+			}
+			break;
+
+		case NVME_ANA_NONOPTIMIZED:
+			if (depth < min_depth_nonopt) {
+				min_depth_nonopt = depth;
+				best_nonopt = ns;
+			}
+			break;
+		default:
+			break;
+		}
+	}
+
+	return best_opt ? best_opt : best_nonopt;
+}
+
 static inline bool nvme_path_is_optimized(struct nvme_ns *ns)
 {
 	return ns->ctrl->state == NVME_CTRL_LIVE &&
@@ -337,15 +377,27 @@ static inline bool nvme_path_is_optimized(struct nvme_ns *ns)
 
 inline struct nvme_ns *nvme_find_path(struct nvme_ns_head *head)
 {
-	int node = numa_node_id();
+	int iopolicy = READ_ONCE(head->subsys->iopolicy);
+	int node;
 	struct nvme_ns *ns;
 
+	/*
+	 * queue-depth iopolicy does not need to reference ->current_path
+	 * but round-robin needs the last path used to advance to the
+	 * next one, and numa will continue to use the last path unless
+	 * it is or has become not optimized
+	 */
+	if (iopolicy == NVME_IOPOLICY_QD)
+		return nvme_queue_depth_path(head);
+
+	node = numa_node_id();
 	ns = srcu_dereference(head->current_path[node], &head->srcu);
 	if (unlikely(!ns))
 		return __nvme_find_path(head, node);
 
-	if (READ_ONCE(head->subsys->iopolicy) == NVME_IOPOLICY_RR)
+	if (iopolicy == NVME_IOPOLICY_RR)
 		return nvme_round_robin_path(head, node, ns);
+
 	if (unlikely(!nvme_path_is_optimized(ns)))
 		return __nvme_find_path(head, node);
 	return ns;
@@ -903,6 +955,7 @@ void nvme_mpath_init_ctrl(struct nvme_ctrl *ctrl)
 	mutex_init(&ctrl->ana_lock);
 	timer_setup(&ctrl->anatt_timer, nvme_anatt_timeout, 0);
 	INIT_WORK(&ctrl->ana_work, nvme_ana_work);
+	atomic_set(&ctrl->nr_active, 0);
 }
 
 int nvme_mpath_init_identify(struct nvme_ctrl *ctrl, struct nvme_id_ctrl *id)
drivers/nvme/host/nvme.h +2 -0
index 39a90b7cb125..f0f3fd8b4197 100644
@@ -347,6 +347,7 @@ struct nvme_ctrl {
 	size_t ana_log_size;
 	struct timer_list anatt_timer;
 	struct work_struct ana_work;
+	atomic_t nr_active;
 #endif
 
 #ifdef CONFIG_NVME_HOST_AUTH
@@ -390,6 +391,7 @@ struct nvme_ctrl {
 enum nvme_iopolicy {
 	NVME_IOPOLICY_NUMA,
 	NVME_IOPOLICY_RR,
+	NVME_IOPOLICY_QD,
 };
 
 struct nvme_subsystem {
Signature · 1 line
2.20.1
#1 lore raw

PATCH 2/3 nvme: multipath: only update ctrl->nr_active when using queue-depth iopolicy

The atomic updates of ctrl->nr_active are unnecessary when using numa or round-robin iopolicy, so avoid that cost on a per-request basis. Clear nr_active when changing iopolicy and do not decrement below zero. (This handles changing the iopolicy while requests are in flight.)

---
 drivers/nvme/host/core.c      |  2 +-
 drivers/nvme/host/multipath.c | 21 ++++++++++++++++++---
 drivers/nvme/host/nvme.h      |  2 ++
 3 files changed, 21 insertions(+), 4 deletions(-)
drivers/nvme/host/core.c +1 -1
index 75a1b58a7a43..9bc19755be77 100644
@@ -110,7 +110,7 @@ struct workqueue_struct *nvme_delete_wq;
 EXPORT_SYMBOL_GPL(nvme_delete_wq);
 
 static LIST_HEAD(nvme_subsystems);
-static DEFINE_MUTEX(nvme_subsystems_lock);
+DEFINE_MUTEX(nvme_subsystems_lock);
 
 static DEFINE_IDA(nvme_instance_ida);
 static dev_t nvme_ctrl_base_chr_devt;
drivers/nvme/host/multipath.c +18 -3
index 4c2690cddef3..e184e7c377bc 100644
@@ -133,7 +133,8 @@ void nvme_mpath_start_request(struct request *rq)
 	if (!blk_queue_io_stat(disk->queue) || blk_rq_is_passthrough(rq))
 		return;
 
-	atomic_inc(&ns->ctrl->nr_active);
+	if (READ_ONCE(ns->head->subsys->iopolicy) == NVME_IOPOLICY_QD)
+		atomic_inc(&ns->ctrl->nr_active);
 	nvme_req(rq)->flags |= NVME_MPATH_IO_STATS;
 	nvme_req(rq)->start_time = bdev_start_io_acct(disk->part0, req_op(rq),
 						      jiffies);
@@ -147,7 +148,8 @@ void nvme_mpath_end_request(struct request *rq)
 	if (!(nvme_req(rq)->flags & NVME_MPATH_IO_STATS))
 		return;
 
-	atomic_dec(&ns->ctrl->nr_active);
+	if (READ_ONCE(ns->head->subsys->iopolicy) == NVME_IOPOLICY_QD)
+		atomic_dec_if_positive(&ns->ctrl->nr_active);
 	bdev_end_io_acct(ns->head->disk->part0, req_op(rq),
 			 blk_rq_bytes(rq) >> SECTOR_SHIFT,
 			 nvme_req(rq)->start_time);
@@ -848,6 +850,19 @@ static ssize_t nvme_subsys_iopolicy_show(struct device *dev,
 			  nvme_iopolicy_names[READ_ONCE(subsys->iopolicy)]);
 }
 
+void nvme_subsys_iopolicy_update(struct nvme_subsystem *subsys, int iopolicy)
+{
+	struct nvme_ctrl *ctrl;
+
+	WRITE_ONCE(subsys->iopolicy, iopolicy);
+
+	mutex_lock(&nvme_subsystems_lock);
+	list_for_each_entry(ctrl, &subsys->ctrls, subsys_entry) {
+		atomic_set(&ctrl->nr_active, 0);
+	}
+	mutex_unlock(&nvme_subsystems_lock);
+}
+
 static ssize_t nvme_subsys_iopolicy_store(struct device *dev,
 		struct device_attribute *attr, const char *buf, size_t count)
 {
@@ -857,7 +872,7 @@ static ssize_t nvme_subsys_iopolicy_store(struct device *dev,
 
 	for (i = 0; i < ARRAY_SIZE(nvme_iopolicy_names); i++) {
 		if (sysfs_streq(buf, nvme_iopolicy_names[i])) {
-			WRITE_ONCE(subsys->iopolicy, i);
+			nvme_subsys_iopolicy_update(subsys, i);
 			return count;
 		}
 	}
drivers/nvme/host/nvme.h +2 -0
index f0f3fd8b4197..c4469bc38d89 100644
@@ -49,6 +49,8 @@ extern struct workqueue_struct *nvme_wq;
 extern struct workqueue_struct *nvme_reset_wq;
 extern struct workqueue_struct *nvme_delete_wq;
 
+extern struct mutex nvme_subsystems_lock;
+
 /*
  * List of workarounds for devices that required behavior not specified in
  * the standard.
Signature · 1 line
2.20.1
#2 lore raw

Re: [PATCH 2/3] nvme: multipath: only update ctrl->nr_active when using queue-depth iopolicy

drivers/nvme/host/multipath.c static ssize_t nvme_subsys_iopolicy_show(struct device *dev, · line 862

Quoted 10 lines
drivers/nvme/host/multipath.c +10 -0
+void nvme_subsys_iopolicy_update(struct nvme_subsystem *subsys, int iopolicy)
+{
+	struct nvme_ctrl *ctrl;
+
+	WRITE_ONCE(subsys->iopolicy, iopolicy);
+
+	mutex_lock(&nvme_subsystems_lock);
+	list_for_each_entry(ctrl, &subsys->ctrls, subsys_entry) {
+		atomic_set(&ctrl->nr_active, 0);
+	}

nit:- do we need braces for above loop ?

-ck

maintainer
#3 lore raw

Re: [PATCH 2/3] nvme: multipath: only update ctrl->nr_active when using queue-depth iopolicy

In reply to Ewan D. Milne
Quoted 4 lines · Ewan D. Milne

The atomic updates of ctrl->nr_active are unnecessary when using numa or round-robin iopolicy, so avoid that cost on a per-request basis. Clear nr_active when changing iopolicy and do not decrement below zero. (This handles changing the iopolicy while requests are in flight.)

Oh, here's restricting it to that policy. Any reason not to fold it in the first one?

drivers/nvme/host/multipath.c void nvme_mpath_start_request(struct request *rq) · line 137

Quoted 3 lines
drivers/nvme/host/multipath.c +2 -1
-	atomic_inc(&ns->ctrl->nr_active);
+	if (READ_ONCE(ns->head->subsys->iopolicy) == NVME_IOPOLICY_QD)
+		atomic_inc(&ns->ctrl->nr_active);

This all looks racy with the ability to change policies in flight. I think we need to introduce a new flag and do something like this:

	if (READ_ONCE(ns->head->subsys->iopolicy) == NVME_IOPOLICY_QD) {
		atomic_inc(&ns->ctrl->nr_active);
	  	nvme_req(rq)->flags |= NVME_MPATH_COUNT_ACTIVE;
	}

...

drivers/nvme/host/multipath.c void nvme_mpath_end_request(struct request *rq) · line 152

Quoted 7 lines
drivers/nvme/host/multipath.c +2 -1
@@ -147,7 +148,8 @@ void nvme_mpath_end_request(struct request *rq)
 	if (!(nvme_req(rq)->flags & NVME_MPATH_IO_STATS))
 		return;
 
-	atomic_dec(&ns->ctrl->nr_active);
+	if (READ_ONCE(ns->head->subsys->iopolicy) == NVME_IOPOLICY_QD)
+		atomic_dec_if_positive(&ns->ctrl->nr_active);

And then the above becomes:

	if (nvme_req(rq)->flags & NVME_MPATH_COUNT_ACTIVE)
		atomic_dec_if_positive(&ns->ctrl->nr_active);

...

drivers/nvme/host/multipath.c static ssize_t nvme_subsys_iopolicy_show(struct device *dev, · line 863

Quoted 11 lines
drivers/nvme/host/multipath.c +11 -0
+void nvme_subsys_iopolicy_update(struct nvme_subsystem *subsys, int iopolicy)
+{
+	struct nvme_ctrl *ctrl;
+
+	WRITE_ONCE(subsys->iopolicy, iopolicy);
+
+	mutex_lock(&nvme_subsystems_lock);
+	list_for_each_entry(ctrl, &subsys->ctrls, subsys_entry) {
+		atomic_set(&ctrl->nr_active, 0);
+	}
+	mutex_unlock(&nvme_subsystems_lock);

And then you don't need to do the above since atomic_dec's are always pairs with atomic_inc's. What do you think?

author
#4 lore raw

Re: [PATCH 2/3] nvme: multipath: only update ctrl->nr_active when using queue-depth iopolicy

I did 2/3 as a separate patch because Tomas wrote the first one and I thought it was a little more clear to have the optimization separated out, but I can combine them if we like.

-Ewan

Quoted 54 lines · Keith Busch
Quoted 4 lines · Ewan D. Milne

The atomic updates of ctrl->nr_active are unnecessary when using numa or round-robin iopolicy, so avoid that cost on a per-request basis. Clear nr_active when changing iopolicy and do not decrement below zero. (This handles changing the iopolicy while requests are in flight.)

Oh, here's restricting it to that policy. Any reason not to fold it in the first one?

drivers/nvme/host/multipath.c void nvme_mpath_start_request(struct request *rq) · line 137

Quoted 3 lines
drivers/nvme/host/multipath.c +2 -1
-     atomic_inc(&ns->ctrl->nr_active);
+     if (READ_ONCE(ns->head->subsys->iopolicy) == NVME_IOPOLICY_QD)
+             atomic_inc(&ns->ctrl->nr_active);

This all looks racy with the ability to change policies in flight. I think we need to introduce a new flag and do something like this:

        if (READ_ONCE(ns->head->subsys->iopolicy) == NVME_IOPOLICY_QD) {
                atomic_inc(&ns->ctrl->nr_active);
                nvme_req(rq)->flags |= NVME_MPATH_COUNT_ACTIVE;
        }

...

drivers/nvme/host/multipath.c void nvme_mpath_end_request(struct request *rq) · line 152

Quoted 7 lines
drivers/nvme/host/multipath.c +2 -1
@@ -147,7 +148,8 @@ void nvme_mpath_end_request(struct request *rq)
      if (!(nvme_req(rq)->flags & NVME_MPATH_IO_STATS))
              return;
 
-     atomic_dec(&ns->ctrl->nr_active);
+     if (READ_ONCE(ns->head->subsys->iopolicy) == NVME_IOPOLICY_QD)
+             atomic_dec_if_positive(&ns->ctrl->nr_active);

And then the above becomes:

        if (nvme_req(rq)->flags & NVME_MPATH_COUNT_ACTIVE)
                atomic_dec_if_positive(&ns->ctrl->nr_active);

...

drivers/nvme/host/multipath.c static ssize_t nvme_subsys_iopolicy_show(struct device *dev, · line 863

Quoted 11 lines
drivers/nvme/host/multipath.c +11 -0
+void nvme_subsys_iopolicy_update(struct nvme_subsystem *subsys, int iopolicy)
+{
+     struct nvme_ctrl *ctrl;
+
+     WRITE_ONCE(subsys->iopolicy, iopolicy);
+
+     mutex_lock(&nvme_subsystems_lock);
+     list_for_each_entry(ctrl, &subsys->ctrls, subsys_entry) {
+             atomic_set(&ctrl->nr_active, 0);
+     }
+     mutex_unlock(&nvme_subsystems_lock);

And then you don't need to do the above since atomic_dec's are always pairs with atomic_inc's. What do you think?

#5 lore raw

Re: [PATCH 2/3] nvme: multipath: only update ctrl->nr_active when using queue-depth iopolicy

In reply to Keith Busch
Quoted 8 lines · Keith Busch
Quoted 4 lines · Ewan D. Milne

The atomic updates of ctrl->nr_active are unnecessary when using numa or round-robin iopolicy, so avoid that cost on a per-request basis. Clear nr_active when changing iopolicy and do not decrement below zero. (This handles changing the iopolicy while requests are in flight.)

Oh, here's restricting it to that policy. Any reason not to fold it in the first one?

It should, and I agree with all the other comments.

But I'm also pretty deeply unhappy with the whole thing. This is a controller-wise atomic taken for every I/O. How slow are the subsystems people want to use it for? And is a global max active really the right measure, or would e a per-cpu, or at least batched per-cpu as used by the percpu counters by a better option?

#6 lore raw

Re: [PATCH 2/3] nvme: multipath: only update ctrl->nr_active when using queue-depth iopolicy

Quoted 1 line · Christoph Hellwig

But I'm also pretty deeply unhappy with the whole thing. This is a controller-wise atomic taken for every I/O. How slow are the subsystems people want to use it for?

This would never be useful with PCIe subsystems which all have latency/response times in the 10s of microseconds range.

This is really only useful with Fibre channel and TCP attached subsystems. We have several enterprise class storage arrays here in our lab at Red Hat - all with multiple 32GB FC and 100Gbps TCP connections. Based on our own private performance analysis of these devices they have latency/response times in 10s of milliseconds range... at best... and the performance bottle necks are in the storage arrays, not in the fabrics. Even some of the RDMA attached storage arrays are slow. These devices have trouble scaling up performance... so they they scale-out instead, adding more and more paths and controllers to different domains in their subsystems.

I am not concerned about the addition of one atomic counter with these use cases. In a grand scheme of things, when using an nvme-of attached storage array, the trade offs are worth it. Besides, if you've dived into the TCP socket layer you'll see there are plenty of atomic counters in the data path already. This is the main use case which needs QD scheduling - nvme/tcp - where latency can build up in the connection as well as in the storage array.

/John

author
#7 lore raw

Re: [PATCH 2/3] nvme: multipath: only update ctrl->nr_active when using queue-depth iopolicy

In reply to Christoph Hellwig

I did attempt an implementation with percpu counters but I did not see enough of a benefit. I also tried updating the counters for only a fraction of the requests, what I observed is that it does not work nearly as well as counting them all, unfortunately.

-Ewan

Quoted 20 lines · Christoph Hellwig
Quoted 8 lines · Keith Busch
Quoted 4 lines · Ewan D. Milne

The atomic updates of ctrl->nr_active are unnecessary when using numa or round-robin iopolicy, so avoid that cost on a per-request basis. Clear nr_active when changing iopolicy and do not decrement below zero. (This handles changing the iopolicy while requests are in flight.)

Oh, here's restricting it to that policy. Any reason not to fold it in the first one?

It should, and I agree with all the other comments.

But I'm also pretty deeply unhappy with the whole thing. This is a controller-wise atomic taken for every I/O. How slow are the subsystems people want to use it for? And is a global max active really the right measure, or would e a per-cpu, or at least batched per-cpu as used by the percpu counters by a better option?

#8 lore raw

Re: [PATCH 2/3] nvme: multipath: only update ctrl->nr_active when using queue-depth iopolicy

In reply to Ewan D. Milne
Quoted 4 lines · Ewan D. Milne

The atomic updates of ctrl->nr_active are unnecessary when using numa or round-robin iopolicy, so avoid that cost on a per-request basis. Clear nr_active when changing iopolicy and do not decrement below zero. (This handles changing the iopolicy while requests are in flight.)

Instead of trying to handle a changing iopolicy while requests are in flight, can we quiesce I/O when we change the iopolicy? That should let us simplify/speed up the logic in the I/O path a bit (atomic_dec_if_positive seems to hide a cmpxchg loop on most architectures, which can be slower than an atomic_dec).

Quoted 91 lines
---
 drivers/nvme/host/core.c      |  2 +-
 drivers/nvme/host/multipath.c | 21 ++++++++++++++++++---
 drivers/nvme/host/nvme.h      |  2 ++
 3 files changed, 21 insertions(+), 4 deletions(-)
drivers/nvme/host/core.c +1 -1
index 75a1b58a7a43..9bc19755be77 100644
@@ -110,7 +110,7 @@ struct workqueue_struct *nvme_delete_wq;
 EXPORT_SYMBOL_GPL(nvme_delete_wq);
 
 static LIST_HEAD(nvme_subsystems);
-static DEFINE_MUTEX(nvme_subsystems_lock);
+DEFINE_MUTEX(nvme_subsystems_lock);
 
 static DEFINE_IDA(nvme_instance_ida);
 static dev_t nvme_ctrl_base_chr_devt;
drivers/nvme/host/multipath.c +18 -3
index 4c2690cddef3..e184e7c377bc 100644
@@ -133,7 +133,8 @@ void nvme_mpath_start_request(struct request *rq)
 	if (!blk_queue_io_stat(disk->queue) || blk_rq_is_passthrough(rq))
 		return;
 
-	atomic_inc(&ns->ctrl->nr_active);
+	if (READ_ONCE(ns->head->subsys->iopolicy) == NVME_IOPOLICY_QD)
+		atomic_inc(&ns->ctrl->nr_active);
 	nvme_req(rq)->flags |= NVME_MPATH_IO_STATS;
 	nvme_req(rq)->start_time = bdev_start_io_acct(disk->part0, req_op(rq),
 						      jiffies);
@@ -147,7 +148,8 @@ void nvme_mpath_end_request(struct request *rq)
 	if (!(nvme_req(rq)->flags & NVME_MPATH_IO_STATS))
 		return;
 
-	atomic_dec(&ns->ctrl->nr_active);
+	if (READ_ONCE(ns->head->subsys->iopolicy) == NVME_IOPOLICY_QD)
+		atomic_dec_if_positive(&ns->ctrl->nr_active);
 	bdev_end_io_acct(ns->head->disk->part0, req_op(rq),
 			 blk_rq_bytes(rq) >> SECTOR_SHIFT,
 			 nvme_req(rq)->start_time);
@@ -848,6 +850,19 @@ static ssize_t nvme_subsys_iopolicy_show(struct device *dev,
 			  nvme_iopolicy_names[READ_ONCE(subsys->iopolicy)]);
 }
 
+void nvme_subsys_iopolicy_update(struct nvme_subsystem *subsys, int iopolicy)
+{
+	struct nvme_ctrl *ctrl;
+
+	WRITE_ONCE(subsys->iopolicy, iopolicy);
+
+	mutex_lock(&nvme_subsystems_lock);
+	list_for_each_entry(ctrl, &subsys->ctrls, subsys_entry) {
+		atomic_set(&ctrl->nr_active, 0);
+	}
+	mutex_unlock(&nvme_subsystems_lock);
+}
+
 static ssize_t nvme_subsys_iopolicy_store(struct device *dev,
 		struct device_attribute *attr, const char *buf, size_t count)
 {
@@ -857,7 +872,7 @@ static ssize_t nvme_subsys_iopolicy_store(struct device *dev,
 
 	for (i = 0; i < ARRAY_SIZE(nvme_iopolicy_names); i++) {
 		if (sysfs_streq(buf, nvme_iopolicy_names[i])) {
-			WRITE_ONCE(subsys->iopolicy, i);
+			nvme_subsys_iopolicy_update(subsys, i);
 			return count;
 		}
 	}
drivers/nvme/host/nvme.h +2 -0
index f0f3fd8b4197..c4469bc38d89 100644
@@ -49,6 +49,8 @@ extern struct workqueue_struct *nvme_wq;
 extern struct workqueue_struct *nvme_reset_wq;
 extern struct workqueue_struct *nvme_delete_wq;
 
+extern struct mutex nvme_subsystems_lock;
+
 /*
  * List of workarounds for devices that required behavior not specified in
  * the standard.
Signature · 1 line
2.20.1
author
#9 lore raw

Re: [PATCH 2/3] nvme: multipath: only update ctrl->nr_active when using queue-depth iopolicy

I think quiescing all the queues associated with the nvme_ctrl would be undesirable
just to handle changing the iopolicy, which only needs to affect I/O that has not yet
gone through the path selection. It would be better not to make the sysfs write have
to wait, as well (although this could be done with a work item, it seems like overkill).

Probably there is not enough contention for the atomic_dec_if_positive() to loop but I take your point, I'll look at it. The equipment I have here is likely not fast enough
to be sure.

-Ewan

Quoted 105 lines · Uday Shankar
Quoted 4 lines · Ewan D. Milne

The atomic updates of ctrl->nr_active are unnecessary when using numa or round-robin iopolicy, so avoid that cost on a per-request basis. Clear nr_active when changing iopolicy and do not decrement below zero. (This handles changing the iopolicy while requests are in flight.)

Instead of trying to handle a changing iopolicy while requests are in flight, can we quiesce I/O when we change the iopolicy? That should let us simplify/speed up the logic in the I/O path a bit (atomic_dec_if_positive seems to hide a cmpxchg loop on most architectures, which can be slower than an atomic_dec).

Quoted 91 lines
---
 drivers/nvme/host/core.c      |  2 +-
 drivers/nvme/host/multipath.c | 21 ++++++++++++++++++---
 drivers/nvme/host/nvme.h      |  2 ++
 3 files changed, 21 insertions(+), 4 deletions(-)
drivers/nvme/host/core.c +1 -1
index 75a1b58a7a43..9bc19755be77 100644
@@ -110,7 +110,7 @@ struct workqueue_struct *nvme_delete_wq;
 EXPORT_SYMBOL_GPL(nvme_delete_wq);
 
 static LIST_HEAD(nvme_subsystems);
-static DEFINE_MUTEX(nvme_subsystems_lock);
+DEFINE_MUTEX(nvme_subsystems_lock);
 
 static DEFINE_IDA(nvme_instance_ida);
 static dev_t nvme_ctrl_base_chr_devt;
drivers/nvme/host/multipath.c +18 -3
index 4c2690cddef3..e184e7c377bc 100644
@@ -133,7 +133,8 @@ void nvme_mpath_start_request(struct request *rq)
      if (!blk_queue_io_stat(disk->queue) || blk_rq_is_passthrough(rq))
              return;
 
-     atomic_inc(&ns->ctrl->nr_active);
+     if (READ_ONCE(ns->head->subsys->iopolicy) == NVME_IOPOLICY_QD)
+             atomic_inc(&ns->ctrl->nr_active);
      nvme_req(rq)->flags |= NVME_MPATH_IO_STATS;
      nvme_req(rq)->start_time = bdev_start_io_acct(disk->part0, req_op(rq),
                                                    jiffies);
@@ -147,7 +148,8 @@ void nvme_mpath_end_request(struct request *rq)
      if (!(nvme_req(rq)->flags & NVME_MPATH_IO_STATS))
              return;
 
-     atomic_dec(&ns->ctrl->nr_active);
+     if (READ_ONCE(ns->head->subsys->iopolicy) == NVME_IOPOLICY_QD)
+             atomic_dec_if_positive(&ns->ctrl->nr_active);
      bdev_end_io_acct(ns->head->disk->part0, req_op(rq),
                       blk_rq_bytes(rq) >> SECTOR_SHIFT,
                       nvme_req(rq)->start_time);
@@ -848,6 +850,19 @@ static ssize_t nvme_subsys_iopolicy_show(struct device *dev,
                        nvme_iopolicy_names[READ_ONCE(subsys->iopolicy)]);
 }
 
+void nvme_subsys_iopolicy_update(struct nvme_subsystem *subsys, int iopolicy)
+{
+     struct nvme_ctrl *ctrl;
+
+     WRITE_ONCE(subsys->iopolicy, iopolicy);
+
+     mutex_lock(&nvme_subsystems_lock);
+     list_for_each_entry(ctrl, &subsys->ctrls, subsys_entry) {
+             atomic_set(&ctrl->nr_active, 0);
+     }
+     mutex_unlock(&nvme_subsystems_lock);
+}
+
 static ssize_t nvme_subsys_iopolicy_store(struct device *dev,
              struct device_attribute *attr, const char *buf, size_t count)
 {
@@ -857,7 +872,7 @@ static ssize_t nvme_subsys_iopolicy_store(struct device *dev,
 
      for (i = 0; i < ARRAY_SIZE(nvme_iopolicy_names); i++) {
              if (sysfs_streq(buf, nvme_iopolicy_names[i])) {
-                     WRITE_ONCE(subsys->iopolicy, i);
+                     nvme_subsys_iopolicy_update(subsys, i);
                      return count;
              }
      }
drivers/nvme/host/nvme.h +2 -1
index f0f3fd8b4197..c4469bc38d89 100644
@@ -49,6 +49,8 @@ extern struct workqueue_struct *nvme_wq;
 extern struct workqueue_struct *nvme_reset_wq;
 extern struct workqueue_struct *nvme_delete_wq;
 
+extern struct mutex nvme_subsystems_lock;
+
 /*
  * List of workarounds for devices that required behavior not specified in
  * the standard.
--

2.20.1

#10 lore raw

PATCH 3/3 nvme: multipath: Invalidate current_path when changing iopolicy

In reply to Ewan D. Milne

When switching back to numa from round-robin, current_path may refer to a different path than the one numa would have selected, and it is desirable to have consistent behavior.

---
 drivers/nvme/host/multipath.c | 1 +
 1 file changed, 1 insertion(+)
drivers/nvme/host/multipath.c +1 -0
index e184e7c377bc..273940b6ca86 100644
@@ -859,6 +859,7 @@ void nvme_subsys_iopolicy_update(struct nvme_subsystem *subsys, int iopolicy)
 	mutex_lock(&nvme_subsystems_lock);
 	list_for_each_entry(ctrl, &subsys->ctrls, subsys_entry) {
 		atomic_set(&ctrl->nr_active, 0);
+		nvme_mpath_clear_ctrl_paths(ctrl);
 	}
 	mutex_unlock(&nvme_subsystems_lock);
 }
Signature · 1 line
2.20.1
Reviewed-by on v1 3/3
#11 lore raw

Re: [PATCH 3/3] nvme: multipath: Invalidate current_path when changing iopolicy

Quoted 3 lines · Ewan D. Milne

When switching back to numa from round-robin, current_path may refer to a different path than the one numa would have selected, and it is desirable to have consistent behavior.

In general invalidating the current path when switching the I/O policy seems like the right thing to do. So independent of the rest of the work we really should add this:

#12 lore raw
In reply to Ewan D. Milne
Quoted 10 lines
drivers/nvme/host/multipath.c +10 -0
+static struct nvme_ns *nvme_queue_depth_path(struct nvme_ns_head *head)
+{
+	struct nvme_ns *best_opt = NULL, *best_nonopt = NULL, *ns;
+	unsigned int min_depth_opt = UINT_MAX, min_depth_nonopt = UINT_MAX;
+	unsigned int depth;
+
+	list_for_each_entry_rcu(ns, &head->list, siblings) {
+		if (nvme_path_is_disabled(ns))
+			continue;
+

drivers/nvme/host/multipath.c static struct nvme_ns *nvme_round_robin_path(struct nvme_ns_head *head, · line 357

Quoted 10 lines
drivers/nvme/host/multipath.c +10 -0
+		depth = atomic_read(&ns->ctrl->nr_active);
+
+		switch (ns->ana_state) {
+		case NVME_ANA_OPTIMIZED:
+			if (depth < min_depth_opt) {
+				min_depth_opt = depth;
+				best_opt = ns;
+			}
+			break;
+

nit:- extra whiteline above after break ?

-ck

#13 lore raw
In reply to Ewan D. Milne
Quoted 4 lines · Ewan D. Milne

The existing iopolicies are inefficient in some cases, such as the presence of a path with high latency. The round-robin policy would use that path equally with faster paths, which results in sub-optimal performance.

do you have performance numbers for such case ?

Quoted 10 lines

The queue-depth policy instead sends I/O requests down the path with the least amount of requests in its request queue. Paths with lower latency will clear requests more quickly and have less requests in their queues compared to "bad" paths. The aim is to use those paths the most to bring down overall latency.

This implementation adds an atomic variable to the nvme_ctrl struct to represent the queue depth. It is updated each time a request specific to that controller starts or ends.

Quoted 8 lines

[edm: patch developed by Thomas Song @ Pure Storage, fixed whitespace
and compilation warnings, updated MODULE_PARM description, and fixed potential issue with ->current_path[] being used]

---

any performance comparison that shows the difference ?

-ck

author
#14 lore raw

Yes, we have some graphs. John M. presented them at ALPSS and there were some earlier ones at LSF/MM. I'll see if I can put up the latest set for download.

The basic issue is that with round-robin, requests for most/all of the tagset space
can end up on a path that is responding slowly, so we see a significant imbalance
in path utilization.

-Ewan

Quoted 33 lines · Chaitanya Kulkarni
Quoted 4 lines · Ewan D. Milne

The existing iopolicies are inefficient in some cases, such as the presence of a path with high latency. The round-robin policy would use that path equally with faster paths, which results in sub-optimal performance.

do you have performance numbers for such case ?

Quoted 10 lines

The queue-depth policy instead sends I/O requests down the path with the least amount of requests in its request queue. Paths with lower latency will clear requests more quickly and have less requests in their queues compared to "bad" paths. The aim is to use those paths the most to bring down overall latency.

This implementation adds an atomic variable to the nvme_ctrl struct to represent the queue depth. It is updated each time a request specific to that controller starts or ends.

Quoted 8 lines

[edm: patch developed by Thomas Song @ Pure Storage, fixed whitespace
and compilation warnings, updated MODULE_PARM description, and fixed potential issue with ->current_path[] being used]

---

any performance comparison that shows the difference ?

-ck

#15 lore raw

Here are the slides with the performance numbers I presented at ALPSS.

https://people.redhat.com/jmeneghi/ALPSS_2023/NVMe_QD_Multipathing.pdf

This is the same information I posted in:

https://lists.infradead.org/pipermail/linux-nvme/2023-September/042371.html

/John

Quoted 49 lines · Ewan Milne

Yes, we have some graphs. John M. presented them at ALPSS and there were some earlier ones at LSF/MM. I'll see if I can put up the latest set for download.

The basic issue is that with round-robin, requests for most/all of the tagset space
can end up on a path that is responding slowly, so we see a significant imbalance
in path utilization.

-Ewan

Quoted 33 lines · Chaitanya Kulkarni
Quoted 4 lines · Ewan D. Milne

The existing iopolicies are inefficient in some cases, such as the presence of a path with high latency. The round-robin policy would use that path equally with faster paths, which results in sub-optimal performance.

do you have performance numbers for such case ?

Quoted 10 lines

The queue-depth policy instead sends I/O requests down the path with the least amount of requests in its request queue. Paths with lower latency will clear requests more quickly and have less requests in their queues compared to "bad" paths. The aim is to use those paths the most to bring down overall latency.

This implementation adds an atomic variable to the nvme_ctrl struct to represent the queue depth. It is updated each time a request specific to that controller starts or ends.

Quoted 8 lines

[edm: patch developed by Thomas Song @ Pure Storage, fixed whitespace
and compilation warnings, updated MODULE_PARM description, and fixed potential issue with ->current_path[] being used]

---

any performance comparison that shows the difference ?

-ck

#16 lore raw
In reply to Ewan Milne
Quoted 4 lines · Ewan Milne

Yes, we have some graphs. John M. presented them at ALPSS and there were some earlier ones at LSF/MM. I'll see if I can put up the latest set for download.

Quoted 8 lines

The basic issue is that with round-robin, requests for most/all of the tagset space
can end up on a path that is responding slowly, so we see a significant imbalance
in path utilization.

-Ewan

Sounds good, please add a cover-letter on your next version and document those so everyone can see them, I'm pretty sure I'm not the only whose going to ask this question :) ...

Can you also please write a block test (cc me) for this under nvme category ? This needs to be tested every-time we send pull request.

-ck

maintainer
#17 lore raw
In reply to Ewan D. Milne

drivers/nvme/host/multipath.c void nvme_mpath_start_request(struct request *rq) · line 136

Quoted 7 lines · Ewan D. Milne

void nvme_mpath_default_iopolicy(struct nvme_subsystem *subsys) {

drivers/nvme/host/multipath.c +1 -0
@@ -130,6 +133,7 @@ void nvme_mpath_start_request(struct request *rq)
 	if (!blk_queue_io_stat(disk->queue) || blk_rq_is_passthrough(rq))
 		return;
 
+	atomic_inc(&ns->ctrl->nr_active);

Did you intend to skip counting nr_active if disk stats are not enabled, or if it's a passthough command?

Also, we should restrict the atomics for other new IO Depth policy.

drivers/nvme/host/multipath.c void nvme_mpath_end_request(struct request *rq) · line 150

Quoted 9 lines
drivers/nvme/host/multipath.c +2 -0
 	nvme_req(rq)->flags |= NVME_MPATH_IO_STATS;
 	nvme_req(rq)->start_time = bdev_start_io_acct(disk->part0, req_op(rq),
 						      jiffies);
@@ -142,6 +146,8 @@ void nvme_mpath_end_request(struct request *rq)
 
 	if (!(nvme_req(rq)->flags & NVME_MPATH_IO_STATS))
 		return;
+
+	atomic_dec(&ns->ctrl->nr_active);

And if disk stats were disabled, this dec isn't paired with an inc.

author
#18 lore raw

The patch uses the existing flag NVME_MPATH_IO_STATS in the request for simplicity to see if an atomic_inc() was performed before doing the _dec(). If we really need to keep the count for passthrough requests (maybe we do) then we would need another flag.

The reason to use the atomic_dec_if_positive() in the 2nd patch and set the nr_active to zero when changing the iopolicy is so that if e.g. someone changes the iopolicy repeatedly while requests are in-flight the counter will work properly.

(I have a test case and another path to make the nr_active counter visible in sysfs,
btw, if that would be useful.)

-Ewan

Quoted 27 lines · Keith Busch

drivers/nvme/host/multipath.c void nvme_mpath_start_request(struct request *rq) · line 136

Quoted 7 lines · Ewan D. Milne

void nvme_mpath_default_iopolicy(struct nvme_subsystem *subsys) {

drivers/nvme/host/multipath.c +1 -0
@@ -130,6 +133,7 @@ void nvme_mpath_start_request(struct request *rq)
      if (!blk_queue_io_stat(disk->queue) || blk_rq_is_passthrough(rq))
              return;
 
+     atomic_inc(&ns->ctrl->nr_active);

Did you intend to skip counting nr_active if disk stats are not enabled, or if it's a passthough command?

Also, we should restrict the atomics for other new IO Depth policy.

drivers/nvme/host/multipath.c void nvme_mpath_end_request(struct request *rq) · line 150

Quoted 9 lines
drivers/nvme/host/multipath.c +2 -0
      nvme_req(rq)->flags |= NVME_MPATH_IO_STATS;
      nvme_req(rq)->start_time = bdev_start_io_acct(disk->part0, req_op(rq),
                                                    jiffies);
@@ -142,6 +146,8 @@ void nvme_mpath_end_request(struct request *rq)
 
      if (!(nvme_req(rq)->flags & NVME_MPATH_IO_STATS))
              return;
+
+     atomic_dec(&ns->ctrl->nr_active);

And if disk stats were disabled, this dec isn't paired with an inc.

maintainer
#19 lore raw
Quoted 4 lines · Ewan Milne

The patch uses the existing flag NVME_MPATH_IO_STATS in the request for simplicity to see if an atomic_inc() was performed before doing the _dec(). If we really need to keep the count for passthrough requests (maybe we do) then we would need another flag.

Oh, I see.

I didn't think you wanted to tie path selection to whether or not the disk stats are enabled (and maybe you don't?), so that's why I thought we needed another flag for this.

#20 lore raw

PATCH v2 1/3 nvme: multipath: Implemented new iopolicy "queue-depth"

In reply to Ewan D. Milne

From: "Ewan D. Milne" <[email protected]>

The existing iopolicies are inefficient in some cases, such as the presence of a path with high latency. The round-robin policy would use that path equally with faster paths, which results in sub-optimal performance.

The queue-depth policy instead sends I/O requests down the path with the least amount of requests in its request queue. Paths with lower latency will clear requests more quickly and have less requests in their queues compared to "bad" paths. The aim is to use those paths the most to bring down overall latency.

This implementation adds an atomic variable to the nvme_ctrl struct to represent the queue depth. It is updated each time a request specific to that controller starts or ends.

[edm: patch developed by Thomas Song @ Pure Storage, fixed whitespace
and compilation warnings, updated MODULE_PARM description, and fixed potential issue with ->current_path[] being used]

---
 drivers/nvme/host/multipath.c | 59 +++++++++++++++++++++++++++++++++--
 drivers/nvme/host/nvme.h      |  2 ++
 2 files changed, 58 insertions(+), 3 deletions(-)
drivers/nvme/host/multipath.c +56 -3
index d16e976ae1a4..3603dd2c785f 100644
@@ -17,6 +17,7 @@ MODULE_PARM_DESC(multipath,
 static const char *nvme_iopolicy_names[] = {
 	[NVME_IOPOLICY_NUMA]	= "numa",
 	[NVME_IOPOLICY_RR]	= "round-robin",
+	[NVME_IOPOLICY_QD]      = "queue-depth",
 };
 
 static int iopolicy = NVME_IOPOLICY_NUMA;
@@ -29,6 +30,8 @@ static int nvme_set_iopolicy(const char *val, const struct kernel_param *kp)
 		iopolicy = NVME_IOPOLICY_NUMA;
 	else if (!strncmp(val, "round-robin", 11))
 		iopolicy = NVME_IOPOLICY_RR;
+	else if (!strncmp(val, "queue-depth", 11))
+		iopolicy = NVME_IOPOLICY_QD;
 	else
 		return -EINVAL;
 
@@ -43,7 +46,7 @@ static int nvme_get_iopolicy(char *buf, const struct kernel_param *kp)
 module_param_call(iopolicy, nvme_set_iopolicy, nvme_get_iopolicy,
 	&iopolicy, 0644);
 MODULE_PARM_DESC(iopolicy,
-	"Default multipath I/O policy; 'numa' (default) or 'round-robin'");
+	"Default multipath I/O policy; 'numa' (default) , 'round-robin' or 'queue-depth'");
 
 void nvme_mpath_default_iopolicy(struct nvme_subsystem *subsys)
 {
@@ -130,6 +133,7 @@ void nvme_mpath_start_request(struct request *rq)
 	if (!blk_queue_io_stat(disk->queue) || blk_rq_is_passthrough(rq))
 		return;
 
+	atomic_inc(&ns->ctrl->nr_active);
 	nvme_req(rq)->flags |= NVME_MPATH_IO_STATS;
 	nvme_req(rq)->start_time = bdev_start_io_acct(disk->part0, req_op(rq),
 						      jiffies);
@@ -142,6 +146,8 @@ void nvme_mpath_end_request(struct request *rq)
 
 	if (!(nvme_req(rq)->flags & NVME_MPATH_IO_STATS))
 		return;
+
+	atomic_dec(&ns->ctrl->nr_active);
 	bdev_end_io_acct(ns->head->disk->part0, req_op(rq),
 			 blk_rq_bytes(rq) >> SECTOR_SHIFT,
 			 nvme_req(rq)->start_time);
@@ -331,6 +337,40 @@ static struct nvme_ns *nvme_round_robin_path(struct nvme_ns_head *head,
 	return found;
 }
 
+static struct nvme_ns *nvme_queue_depth_path(struct nvme_ns_head *head)
+{
+	struct nvme_ns *best_opt = NULL, *best_nonopt = NULL, *ns;
+	unsigned int min_depth_opt = UINT_MAX, min_depth_nonopt = UINT_MAX;
+	unsigned int depth;
+
+	list_for_each_entry_rcu(ns, &head->list, siblings) {
+		if (nvme_path_is_disabled(ns))
+			continue;
+
+		depth = atomic_read(&ns->ctrl->nr_active);
+
+		switch (ns->ana_state) {
+		case NVME_ANA_OPTIMIZED:
+			if (depth < min_depth_opt) {
+				min_depth_opt = depth;
+				best_opt = ns;
+			}
+			break;
+
+		case NVME_ANA_NONOPTIMIZED:
+			if (depth < min_depth_nonopt) {
+				min_depth_nonopt = depth;
+				best_nonopt = ns;
+			}
+			break;
+		default:
+			break;
+		}
+	}
+
+	return best_opt ? best_opt : best_nonopt;
+}
+
 static inline bool nvme_path_is_optimized(struct nvme_ns *ns)
 {
 	return nvme_ctrl_state(ns->ctrl) == NVME_CTRL_LIVE &&
@@ -339,15 +379,27 @@ static inline bool nvme_path_is_optimized(struct nvme_ns *ns)
 
 inline struct nvme_ns *nvme_find_path(struct nvme_ns_head *head)
 {
-	int node = numa_node_id();
+	int iopolicy = READ_ONCE(head->subsys->iopolicy);
+	int node;
 	struct nvme_ns *ns;
 
+	/*
+	 * queue-depth iopolicy does not need to reference ->current_path
+	 * but round-robin needs the last path used to advance to the
+	 * next one, and numa will continue to use the last path unless
+	 * it is or has become not optimized
+	 */
+	if (iopolicy == NVME_IOPOLICY_QD)
+		return nvme_queue_depth_path(head);
+
+	node = numa_node_id();
 	ns = srcu_dereference(head->current_path[node], &head->srcu);
 	if (unlikely(!ns))
 		return __nvme_find_path(head, node);
 
-	if (READ_ONCE(head->subsys->iopolicy) == NVME_IOPOLICY_RR)
+	if (iopolicy == NVME_IOPOLICY_RR)
 		return nvme_round_robin_path(head, node, ns);
+
 	if (unlikely(!nvme_path_is_optimized(ns)))
 		return __nvme_find_path(head, node);
 	return ns;
@@ -906,6 +958,7 @@ void nvme_mpath_init_ctrl(struct nvme_ctrl *ctrl)
 	mutex_init(&ctrl->ana_lock);
 	timer_setup(&ctrl->anatt_timer, nvme_anatt_timeout, 0);
 	INIT_WORK(&ctrl->ana_work, nvme_ana_work);
+	atomic_set(&ctrl->nr_active, 0);
 }
 
 int nvme_mpath_init_identify(struct nvme_ctrl *ctrl, struct nvme_id_ctrl *id)
drivers/nvme/host/nvme.h +2 -0
index 05532c281177..e23e17ab94b9 100644
@@ -359,6 +359,7 @@ struct nvme_ctrl {
 	size_t ana_log_size;
 	struct timer_list anatt_timer;
 	struct work_struct ana_work;
+	atomic_t nr_active;
 #endif
 
 #ifdef CONFIG_NVME_HOST_AUTH
@@ -407,6 +408,7 @@ static inline enum nvme_ctrl_state nvme_ctrl_state(struct nvme_ctrl *ctrl)
 enum nvme_iopolicy {
 	NVME_IOPOLICY_NUMA,
 	NVME_IOPOLICY_RR,
+	NVME_IOPOLICY_QD,
 };
 
 struct nvme_subsystem {
Signature · 1 line
2.39.3
#21 lore raw

PATCH v2 2/3 nvme: multipath: only update ctrl->nr_active when using queue-depth iopolicy

In reply to Ewan D. Milne

From: "Ewan D. Milne" <[email protected]>

The atomic updates of ctrl->nr_active are unnecessary when using numa or round-robin iopolicy, so avoid that cost on a per-request basis. Clear nr_active when changing iopolicy and do not decrement below zero. (This handles changing the iopolicy while requests are in flight.)

---
 drivers/nvme/host/core.c      |  2 +-
 drivers/nvme/host/multipath.c | 21 ++++++++++++++++++---
 drivers/nvme/host/nvme.h      |  6 ++++++
 3 files changed, 25 insertions(+), 4 deletions(-)
drivers/nvme/host/core.c +1 -1
index 095f59e7aa93..ffc04d65c45e 100644
@@ -110,7 +110,7 @@ struct workqueue_struct *nvme_delete_wq;
 EXPORT_SYMBOL_GPL(nvme_delete_wq);
 
 static LIST_HEAD(nvme_subsystems);
-static DEFINE_MUTEX(nvme_subsystems_lock);
+DEFINE_MUTEX(nvme_subsystems_lock);
 
 static DEFINE_IDA(nvme_instance_ida);
 static dev_t nvme_ctrl_base_chr_devt;
drivers/nvme/host/multipath.c +18 -3
index 3603dd2c785f..02baadb45c82 100644
@@ -133,7 +133,8 @@ void nvme_mpath_start_request(struct request *rq)
 	if (!blk_queue_io_stat(disk->queue) || blk_rq_is_passthrough(rq))
 		return;
 
-	atomic_inc(&ns->ctrl->nr_active);
+	if (READ_ONCE(ns->head->subsys->iopolicy) == NVME_IOPOLICY_QD)
+		atomic_inc(&ns->ctrl->nr_active);
 	nvme_req(rq)->flags |= NVME_MPATH_IO_STATS;
 	nvme_req(rq)->start_time = bdev_start_io_acct(disk->part0, req_op(rq),
 						      jiffies);
@@ -147,7 +148,8 @@ void nvme_mpath_end_request(struct request *rq)
 	if (!(nvme_req(rq)->flags & NVME_MPATH_IO_STATS))
 		return;
 
-	atomic_dec(&ns->ctrl->nr_active);
+	if (READ_ONCE(ns->head->subsys->iopolicy) == NVME_IOPOLICY_QD)
+		atomic_dec_if_positive(&ns->ctrl->nr_active);
 	bdev_end_io_acct(ns->head->disk->part0, req_op(rq),
 			 blk_rq_bytes(rq) >> SECTOR_SHIFT,
 			 nvme_req(rq)->start_time);
@@ -851,6 +853,19 @@ static ssize_t nvme_subsys_iopolicy_show(struct device *dev,
 			  nvme_iopolicy_names[READ_ONCE(subsys->iopolicy)]);
 }
 
+void nvme_subsys_iopolicy_update(struct nvme_subsystem *subsys, int iopolicy)
+{
+	struct nvme_ctrl *ctrl;
+
+	WRITE_ONCE(subsys->iopolicy, iopolicy);
+
+	mutex_lock(&nvme_subsystems_lock);
+	list_for_each_entry(ctrl, &subsys->ctrls, subsys_entry) {
+		atomic_set(&ctrl->nr_active, 0);
+	}
+	mutex_unlock(&nvme_subsystems_lock);
+}
+
 static ssize_t nvme_subsys_iopolicy_store(struct device *dev,
 		struct device_attribute *attr, const char *buf, size_t count)
 {
@@ -860,7 +875,7 @@ static ssize_t nvme_subsys_iopolicy_store(struct device *dev,
 
 	for (i = 0; i < ARRAY_SIZE(nvme_iopolicy_names); i++) {
 		if (sysfs_streq(buf, nvme_iopolicy_names[i])) {
-			WRITE_ONCE(subsys->iopolicy, i);
+			nvme_subsys_iopolicy_update(subsys, i);
 			return count;
 		}
 	}
drivers/nvme/host/nvme.h +6 -0
index e23e17ab94b9..a557b4577c01 100644
@@ -50,6 +50,8 @@ extern struct workqueue_struct *nvme_wq;
 extern struct workqueue_struct *nvme_reset_wq;
 extern struct workqueue_struct *nvme_delete_wq;
 
+extern struct mutex nvme_subsystems_lock;
+
 /*
  * List of workarounds for devices that required behavior not specified in
  * the standard.
@@ -963,6 +965,7 @@ void nvme_mpath_clear_ctrl_paths(struct nvme_ctrl *ctrl);
 void nvme_mpath_shutdown_disk(struct nvme_ns_head *head);
 void nvme_mpath_start_request(struct request *rq);
 void nvme_mpath_end_request(struct request *rq);
+void nvme_subsys_iopolicy_update(struct nvme_subsystem *subsys, int iopolicy);
 
 static inline void nvme_trace_bio_complete(struct request *req)
 {
@@ -1062,6 +1065,9 @@ static inline bool nvme_disk_is_ns_head(struct gendisk *disk)
 {
 	return false;
 }
+static inline void nvme_subsys_iopolicy_update(struct nvme_subsystem *subsys, int iopolicy)
+{
+}
 #endif /* CONFIG_NVME_MULTIPATH */
 
 struct nvme_zone_info {
Signature · 1 line
2.39.3
#22 lore raw

v2 cover nvme: queue-depth multipath iopolicy

In reply to Ewan D. Milne

I'm re-issuing Ewan's queue-depth patches in preparation for LSFMM

These patches were first show at ALPSS 2023 where I shared the following graphs which measure the IO distribution across 4 active-optimized controllers using the round-robin verses queue-depth iopolicy.

https://people.redhat.com/jmeneghi/ALPSS_2023/NVMe_QD_Multipathing.pdf

Since that time we have continued testing these patches with a number of different nvme-of storage arrays and test bed configurations, and I've codified the tests and methods we use to measure IO distribution

All of my test results, together with the scripts I used to generate these graphs, are available at:

https://github.com/johnmeneghini/iopolicy

Please use the scripts in this repository to do your own testing.

These patches are based on nvme-v6.9

Ewan D. Milne (3):
nvme: multipath: Implemented new iopolicy "queue-depth"
nvme: multipath: only update ctrl->nr_active when using queue-depth
iopolicy
nvme: multipath: Invalidate current_path when changing iopolicy

 drivers/nvme/host/core.c      |  2 +-
 drivers/nvme/host/multipath.c | 77 +++++++++++++++++++++++++++++++++--
 drivers/nvme/host/nvme.h      |  8 ++++
 3 files changed, 82 insertions(+), 5 deletions(-)
Signature · 1 line
2.39.3
#23 lore raw

PATCH v2 3/3 nvme: multipath: Invalidate current_path when changing iopolicy

In reply to Ewan D. Milne

From: "Ewan D. Milne" <[email protected]>

When switching back to numa from round-robin, current_path may refer to a different path than the one numa would have selected, and it is desirable to have consistent behavior.

---
 drivers/nvme/host/multipath.c | 1 +
 1 file changed, 1 insertion(+)
drivers/nvme/host/multipath.c +1 -0
index 02baadb45c82..d916a5ddf5d4 100644
@@ -862,6 +862,7 @@ void nvme_subsys_iopolicy_update(struct nvme_subsystem *subsys, int iopolicy)
 	mutex_lock(&nvme_subsystems_lock);
 	list_for_each_entry(ctrl, &subsys->ctrls, subsys_entry) {
 		atomic_set(&ctrl->nr_active, 0);
+		nvme_mpath_clear_ctrl_paths(ctrl);
 	}
 	mutex_unlock(&nvme_subsystems_lock);
 }
Signature · 1 line
2.39.3