nvme: multipath: Implemented new iopolicy "queue-depth"
PATCH 1/3 nvme: multipath: Implemented new iopolicy "queue-depth"
The existing iopolicies are inefficient in some cases, such as the presence of a path with high latency. The round-robin policy would use that path equally with faster paths, which results in sub-optimal performance.
The queue-depth policy instead sends I/O requests down the path with the least amount of requests in its request queue. Paths with lower latency will clear requests more quickly and have less requests in their queues compared to "bad" paths. The aim is to use those paths the most to bring down overall latency.
This implementation adds an atomic variable to the nvme_ctrl struct to represent the queue depth. It is updated each time a request specific to that controller starts or ends.
[edm: patch developed by Thomas Song @ Pure Storage, fixed whitespace
and compilation warnings, updated MODULE_PARM description, and fixed potential issue with ->current_path[] being used]
--- drivers/nvme/host/multipath.c | 59 +++++++++++++++++++++++++++++++++-- drivers/nvme/host/nvme.h | 2 ++ 2 files changed, 58 insertions(+), 3 deletions(-)
drivers/nvme/host/multipath.c +56 -3
@@ -17,6 +17,7 @@ MODULE_PARM_DESC(multipath,static const char *nvme_iopolicy_names[] = { [NVME_IOPOLICY_NUMA ] ="numa" , [NVME_IOPOLICY_RR ] ="round-robin" , + [NVME_IOPOLICY_QD ] ="queue-depth" , };static int iopolicy =NVME_IOPOLICY_NUMA ;
@@ -29,6 +30,8 @@ static int nvme_set_iopolicy(const char *val, const struct kernel_param *kp) iopolicy =NVME_IOPOLICY_NUMA ; else if (!strncmp(val, "round-robin", 11)) iopolicy =NVME_IOPOLICY_RR ; + else if (!strncmp(val, "queue-depth", 11)) + iopolicy =NVME_IOPOLICY_QD ; else return -EINVAL;
@@ -43,7 +46,7 @@ static int nvme_get_iopolicy(char *buf, const struct kernel_param *kp)module_param_call (iopolicy, nvme_set_iopolicy, nvme_get_iopolicy, &iopolicy,0644 );MODULE_PARM_DESC (iopolicy, -"Default multipath I/O policy; 'numa' (default) or 'round-robin'" ); +"Default multipath I/O policy; 'numa' (default) , 'round-robin' or 'queue-depth'" );void nvme_mpath_default_iopolicy (struct nvme_subsystem *subsys) {
@@ -130,6 +133,7 @@ void nvme_mpath_start_request(struct request *rq)if (!blk_queue_io_stat (disk->queue) ||blk_rq_is_passthrough (rq))return ; +atomic_inc (&ns->ctrl->nr_active);nvme_req (rq)->flags |=NVME_MPATH_IO_STATS ;nvme_req (rq)->start_time =bdev_start_io_acct (disk->part0,req_op (rq), jiffies);
@@ -142,6 +146,8 @@ void nvme_mpath_end_request(struct request *rq)if (!(nvme_req (rq)->flags &NVME_MPATH_IO_STATS ))return ; + +atomic_dec (&ns->ctrl->nr_active);bdev_end_io_acct (ns->head->disk->part0,req_op (rq),blk_rq_bytes (rq) >>SECTOR_SHIFT ,nvme_req (rq)->start_time);
@@ -329,6 +335,40 @@ static struct nvme_ns *nvme_round_robin_path(struct nvme_ns_head *head,return found; } +static struct nvme_ns *nvme_queue_depth_path (struct nvme_ns_head *head) +{ +struct nvme_ns *best_opt =NULL , *best_nonopt =NULL , *ns; +unsigned int min_depth_opt =UINT_MAX , min_depth_nonopt =UINT_MAX ; +unsigned int depth; + +list_for_each_entry_rcu (ns, &head->list, siblings) { +if (nvme_path_is_disabled (ns)) +continue ; + + depth =atomic_read (&ns->ctrl->nr_active); + +switch (ns->ana_state) { +case NVME_ANA_OPTIMIZED : +if (depth < min_depth_opt) { + min_depth_opt = depth; + best_opt = ns; + } +break ; + +case NVME_ANA_NONOPTIMIZED : +if (depth < min_depth_nonopt) { + min_depth_nonopt = depth; + best_nonopt = ns; + } +break ; +default : +break ; + } + } + +return best_opt ? best_opt : best_nonopt; +} +static inline bool nvme_path_is_optimized (struct nvme_ns *ns) { return ns->ctrl->state == NVME_CTRL_LIVE &&
@@ -337,15 +377,27 @@ static inline bool nvme_path_is_optimized(struct nvme_ns *ns)inline struct nvme_ns *nvme_find_path (struct nvme_ns_head *head) { -int node =numa_node_id (); +int iopolicy =READ_ONCE (head->subsys->iopolicy); +int node;struct nvme_ns *ns; +/* +* queue-depth iopolicy does not need to reference ->current_path +* but round-robin needs the last path used to advance to the +* next one, and numa will continue to use the last path unless +* it is or has become not optimized +*/ +if (iopolicy ==NVME_IOPOLICY_QD ) +return nvme_queue_depth_path (head); + + node =numa_node_id (); ns =srcu_dereference (head->current_path[node], &head->srcu);if (unlikely (!ns))return __nvme_find_path (head, node); -if (READ_ONCE (head->subsys->iopolicy) ==NVME_IOPOLICY_RR ) +if (iopolicy ==NVME_IOPOLICY_RR )return nvme_round_robin_path (head, node, ns); +if (unlikely (!nvme_path_is_optimized (ns)))return __nvme_find_path (head, node);return ns;
@@ -903,6 +955,7 @@ void nvme_mpath_init_ctrl(struct nvme_ctrl *ctrl)mutex_init (&ctrl->ana_lock);timer_setup (&ctrl->anatt_timer, nvme_anatt_timeout,0 );INIT_WORK (&ctrl->ana_work, nvme_ana_work); +atomic_set (&ctrl->nr_active,0 ); }int nvme_mpath_init_identify (struct nvme_ctrl *ctrl,struct nvme_id_ctrl *id)
drivers/nvme/host/nvme.h +2 -0
@@ -347,6 +347,7 @@ struct nvme_ctrl {size_t ana_log_size;struct timer_list anatt_timer;struct work_struct ana_work; +atomic_t nr_active;#endif #ifdef CONFIG_NVME_HOST_AUTH
@@ -390,6 +391,7 @@ struct nvme_ctrl {enum nvme_iopolicy {NVME_IOPOLICY_NUMA ,NVME_IOPOLICY_RR , +NVME_IOPOLICY_QD , };struct nvme_subsystem {
Signature · 1 line
2.20.1
PATCH 2/3 nvme: multipath: only update ctrl->nr_active when using queue-depth iopolicy
The atomic updates of ctrl->nr_active are unnecessary when using numa or round-robin iopolicy, so avoid that cost on a per-request basis. Clear nr_active when changing iopolicy and do not decrement below zero. (This handles changing the iopolicy while requests are in flight.)
--- drivers/nvme/host/core.c | 2 +- drivers/nvme/host/multipath.c | 21 ++++++++++++++++++--- drivers/nvme/host/nvme.h | 2 ++ 3 files changed, 21 insertions(+), 4 deletions(-)
drivers/nvme/host/core.c +1 -1
@@ -110,7 +110,7 @@ struct workqueue_struct *nvme_delete_wq;EXPORT_SYMBOL_GPL (nvme_delete_wq);static LIST_HEAD (nvme_subsystems); -static DEFINE_MUTEX (nvme_subsystems_lock); +DEFINE_MUTEX (nvme_subsystems_lock);static DEFINE_IDA (nvme_instance_ida);static dev_t nvme_ctrl_base_chr_devt;
drivers/nvme/host/multipath.c +18 -3
@@ -133,7 +133,8 @@ void nvme_mpath_start_request(struct request *rq)if (!blk_queue_io_stat (disk->queue) ||blk_rq_is_passthrough (rq))return ; -atomic_inc (&ns->ctrl->nr_active); +if (READ_ONCE (ns->head->subsys->iopolicy) ==NVME_IOPOLICY_QD ) +atomic_inc (&ns->ctrl->nr_active);nvme_req (rq)->flags |=NVME_MPATH_IO_STATS ;nvme_req (rq)->start_time =bdev_start_io_acct (disk->part0,req_op (rq), jiffies);
@@ -147,7 +148,8 @@ void nvme_mpath_end_request(struct request *rq)if (!(nvme_req (rq)->flags &NVME_MPATH_IO_STATS ))return ; -atomic_dec (&ns->ctrl->nr_active); +if (READ_ONCE (ns->head->subsys->iopolicy) ==NVME_IOPOLICY_QD ) +atomic_dec_if_positive (&ns->ctrl->nr_active);bdev_end_io_acct (ns->head->disk->part0,req_op (rq),blk_rq_bytes (rq) >>SECTOR_SHIFT ,nvme_req (rq)->start_time);
@@ -848,6 +850,19 @@ static ssize_t nvme_subsys_iopolicy_show(struct device *dev, nvme_iopolicy_names[READ_ONCE(subsys->iopolicy)]); } +void nvme_subsys_iopolicy_update (struct nvme_subsystem *subsys,int iopolicy) +{ +struct nvme_ctrl *ctrl; + +WRITE_ONCE (subsys->iopolicy, iopolicy); + +mutex_lock (&nvme_subsystems_lock); +list_for_each_entry (ctrl, &subsys->ctrls, subsys_entry) { +atomic_set (&ctrl->nr_active,0 ); + } +mutex_unlock (&nvme_subsystems_lock); +} +static ssize_t nvme_subsys_iopolicy_store (struct device *dev,struct device_attribute *attr,const char *buf,size_t count) {
@@ -857,7 +872,7 @@ static ssize_t nvme_subsys_iopolicy_store(struct device *dev,for (i =0 ; i <ARRAY_SIZE (nvme_iopolicy_names); i++) {if (sysfs_streq (buf, nvme_iopolicy_names[i])) { -WRITE_ONCE (subsys->iopolicy, i); +nvme_subsys_iopolicy_update (subsys, i);return count; } }
drivers/nvme/host/nvme.h +2 -0
@@ -49,6 +49,8 @@ extern struct workqueue_struct *nvme_wq;extern struct workqueue_struct *nvme_reset_wq;extern struct workqueue_struct *nvme_delete_wq; +extern struct mutex nvme_subsystems_lock; +/* * List of workarounds for devices that required behavior not specified in * the standard.
Signature · 1 line
2.20.1
Re: [PATCH 2/3] nvme: multipath: only update ctrl->nr_active when using queue-depth iopolicy
drivers/nvme/host/multipath.c static ssize_t nvme_subsys_iopolicy_show(struct device *dev, · line 862
Quoted 10 lines
drivers/nvme/host/multipath.c +10
-0+void nvme_subsys_iopolicy_update (struct nvme_subsystem *subsys,int iopolicy) +{ +struct nvme_ctrl *ctrl; + +WRITE_ONCE (subsys->iopolicy, iopolicy); + +mutex_lock (&nvme_subsystems_lock); +list_for_each_entry (ctrl, &subsys->ctrls, subsys_entry) { +atomic_set (&ctrl->nr_active,0 ); + }
nit:- do we need braces for above loop ?
-ck
Keith Busch
maintainerRe: [PATCH 2/3] nvme: multipath: only update ctrl->nr_active when using queue-depth iopolicy
In reply to Ewan D. MilneQuoted 4 lines · Ewan D. Milne
The atomic updates of
ctrl->nr_activeare unnecessary when using numa or round-robin iopolicy, so avoid that cost on a per-request basis. Clearnr_activewhen changing iopolicy and do not decrement below zero. (This handles changing the iopolicy while requests are in flight.)
Oh, here's restricting it to that policy. Any reason not to fold it in the first one?
drivers/nvme/host/multipath.c void nvme_mpath_start_request(struct request *rq) · line 137
Quoted 3 lines
drivers/nvme/host/multipath.c +2
-1-atomic_inc (&ns->ctrl->nr_active); +if (READ_ONCE (ns->head->subsys->iopolicy) ==NVME_IOPOLICY_QD ) +atomic_inc (&ns->ctrl->nr_active);
This all looks racy with the ability to change policies in flight. I think we need to introduce a new flag and do something like this:
if (READ_ONCE(ns->head->subsys->iopolicy) == NVME_IOPOLICY_QD) {
atomic_inc(&ns->ctrl->nr_active);
nvme_req(rq)->flags |= NVME_MPATH_COUNT_ACTIVE;
}...
drivers/nvme/host/multipath.c void nvme_mpath_end_request(struct request *rq) · line 152
Quoted 7 lines
drivers/nvme/host/multipath.c +2
-1@@ -147,7 +148,8 @@ void nvme_mpath_end_request(struct request *rq)if (!(nvme_req (rq)->flags &NVME_MPATH_IO_STATS ))return ; -atomic_dec (&ns->ctrl->nr_active); +if (READ_ONCE (ns->head->subsys->iopolicy) ==NVME_IOPOLICY_QD ) +atomic_dec_if_positive (&ns->ctrl->nr_active);
And then the above becomes:
if (nvme_req(rq)->flags & NVME_MPATH_COUNT_ACTIVE) atomic_dec_if_positive(&ns->ctrl->nr_active);
...
drivers/nvme/host/multipath.c static ssize_t nvme_subsys_iopolicy_show(struct device *dev, · line 863
Quoted 11 lines
drivers/nvme/host/multipath.c +11
-0+void nvme_subsys_iopolicy_update (struct nvme_subsystem *subsys,int iopolicy) +{ +struct nvme_ctrl *ctrl; + +WRITE_ONCE (subsys->iopolicy, iopolicy); + +mutex_lock (&nvme_subsystems_lock); +list_for_each_entry (ctrl, &subsys->ctrls, subsys_entry) { +atomic_set (&ctrl->nr_active,0 ); + } +mutex_unlock (&nvme_subsystems_lock);
And then you don't need to do the above since atomic_dec's are always pairs with atomic_inc's. What do you think?
Re: [PATCH 2/3] nvme: multipath: only update ctrl->nr_active when using queue-depth iopolicy
I did 2/3 as a separate patch because Tomas wrote the first one and I thought it was a little more clear to have the optimization separated out, but I can combine them if we like.
-Ewan
Quoted 54 lines · Keith Busch
Quoted 4 lines · Ewan D. Milne
The atomic updates of
ctrl->nr_activeare unnecessary when using numa or round-robin iopolicy, so avoid that cost on a per-request basis. Clearnr_activewhen changing iopolicy and do not decrement below zero. (This handles changing the iopolicy while requests are in flight.)Oh, here's restricting it to that policy. Any reason not to fold it in the first one?
drivers/nvme/host/multipath.c void nvme_mpath_start_request(struct request *rq) · line 137
Quoted 3 lines
drivers/nvme/host/multipath.c +2
-1-atomic_inc (&ns->ctrl->nr_active); +if (READ_ONCE (ns->head->subsys->iopolicy) ==NVME_IOPOLICY_QD ) +atomic_inc (&ns->ctrl->nr_active);This all looks racy with the ability to change policies in flight. I think we need to introduce a new flag and do something like this:
if (READ_ONCE(ns->head->subsys->iopolicy) == NVME_IOPOLICY_QD) { atomic_inc(&ns->ctrl->nr_active); nvme_req(rq)->flags |= NVME_MPATH_COUNT_ACTIVE; }...
drivers/nvme/host/multipath.c void nvme_mpath_end_request(struct request *rq) · line 152
Quoted 7 lines
drivers/nvme/host/multipath.c +2
-1@@ -147,7 +148,8 @@ void nvme_mpath_end_request(struct request *rq)if (!(nvme_req (rq)->flags &NVME_MPATH_IO_STATS ))return ; -atomic_dec (&ns->ctrl->nr_active); +if (READ_ONCE (ns->head->subsys->iopolicy) ==NVME_IOPOLICY_QD ) +atomic_dec_if_positive (&ns->ctrl->nr_active);And then the above becomes:
if (nvme_req(rq)->flags & NVME_MPATH_COUNT_ACTIVE) atomic_dec_if_positive(&ns->ctrl->nr_active);...
drivers/nvme/host/multipath.c static ssize_t nvme_subsys_iopolicy_show(struct device *dev, · line 863
Quoted 11 lines
drivers/nvme/host/multipath.c +11
-0+void nvme_subsys_iopolicy_update (struct nvme_subsystem *subsys,int iopolicy) +{ +struct nvme_ctrl *ctrl; + +WRITE_ONCE (subsys->iopolicy, iopolicy); + +mutex_lock (&nvme_subsystems_lock); +list_for_each_entry (ctrl, &subsys->ctrls, subsys_entry) { +atomic_set (&ctrl->nr_active,0 ); + } +mutex_unlock (&nvme_subsystems_lock);And then you don't need to do the above since atomic_dec's are always pairs with atomic_inc's. What do you think?
Re: [PATCH 2/3] nvme: multipath: only update ctrl->nr_active when using queue-depth iopolicy
In reply to Keith BuschQuoted 8 lines · Keith Busch
Quoted 4 lines · Ewan D. Milne
The atomic updates of
ctrl->nr_activeare unnecessary when using numa or round-robin iopolicy, so avoid that cost on a per-request basis. Clearnr_activewhen changing iopolicy and do not decrement below zero. (This handles changing the iopolicy while requests are in flight.)Oh, here's restricting it to that policy. Any reason not to fold it in the first one?
It should, and I agree with all the other comments.
But I'm also pretty deeply unhappy with the whole thing. This is a controller-wise atomic taken for every I/O. How slow are the subsystems people want to use it for? And is a global max active really the right measure, or would e a per-cpu, or at least batched per-cpu as used by the percpu counters by a better option?
Re: [PATCH 2/3] nvme: multipath: only update ctrl->nr_active when using queue-depth iopolicy
Quoted 1 line · Christoph Hellwig
But I'm also pretty deeply unhappy with the whole thing. This is a controller-wise atomic taken for every I/O. How slow are the subsystems people want to use it for?
This would never be useful with PCIe subsystems which all have latency/response times in the 10s of microseconds range.
This is really only useful with Fibre channel and TCP attached subsystems. We have several enterprise class storage arrays here in our lab at Red Hat - all with multiple 32GB FC and 100Gbps TCP connections. Based on our own private performance analysis of these devices they have latency/response times in 10s of milliseconds range... at best... and the performance bottle necks are in the storage arrays, not in the fabrics. Even some of the RDMA attached storage arrays are slow. These devices have trouble scaling up performance... so they they scale-out instead, adding more and more paths and controllers to different domains in their subsystems.
I am not concerned about the addition of one atomic counter with these use cases. In a grand scheme of things, when using an nvme-of attached storage array, the trade offs are worth it. Besides, if you've dived into the TCP socket layer you'll see there are plenty of atomic counters in the data path already. This is the main use case which needs QD scheduling - nvme/tcp - where latency can build up in the connection as well as in the storage array.
/John
Re: [PATCH 2/3] nvme: multipath: only update ctrl->nr_active when using queue-depth iopolicy
In reply to Christoph HellwigI did attempt an implementation with percpu counters but I did not see enough of a benefit. I also tried updating the counters for only a fraction of the requests, what I observed is that it does not work nearly as well as counting them all, unfortunately.
-Ewan
Quoted 20 lines · Christoph Hellwig
Quoted 8 lines · Keith Busch
Quoted 4 lines · Ewan D. Milne
The atomic updates of
ctrl->nr_activeare unnecessary when using numa or round-robin iopolicy, so avoid that cost on a per-request basis. Clearnr_activewhen changing iopolicy and do not decrement below zero. (This handles changing the iopolicy while requests are in flight.)Oh, here's restricting it to that policy. Any reason not to fold it in the first one?
It should, and I agree with all the other comments.
But I'm also pretty deeply unhappy with the whole thing. This is a controller-wise atomic taken for every I/O. How slow are the subsystems people want to use it for? And is a global max active really the right measure, or would e a per-cpu, or at least batched per-cpu as used by the percpu counters by a better option?
Re: [PATCH 2/3] nvme: multipath: only update ctrl->nr_active when using queue-depth iopolicy
In reply to Ewan D. MilneQuoted 4 lines · Ewan D. Milne
The atomic updates of
ctrl->nr_activeare unnecessary when using numa or round-robin iopolicy, so avoid that cost on a per-request basis. Clearnr_activewhen changing iopolicy and do not decrement below zero. (This handles changing the iopolicy while requests are in flight.)
Instead of trying to handle a changing iopolicy while requests are in flight, can we quiesce I/O when we change the iopolicy? That should let us simplify/speed up the logic in the I/O path a bit (atomic_dec_if_positive seems to hide a cmpxchg loop on most architectures, which can be slower than an atomic_dec).
Quoted 91 lines
Signed-off-by Ewan D. Milne <[email protected]>--- drivers/nvme/host/core.c | 2 +- drivers/nvme/host/multipath.c | 21 ++++++++++++++++++--- drivers/nvme/host/nvme.h | 2 ++ 3 files changed, 21 insertions(+), 4 deletions(-)drivers/nvme/host/core.c +1
-1@@ -110,7 +110,7 @@ struct workqueue_struct *nvme_delete_wq;EXPORT_SYMBOL_GPL (nvme_delete_wq);static LIST_HEAD (nvme_subsystems); -static DEFINE_MUTEX (nvme_subsystems_lock); +DEFINE_MUTEX (nvme_subsystems_lock);static DEFINE_IDA (nvme_instance_ida);static dev_t nvme_ctrl_base_chr_devt;drivers/nvme/host/multipath.c +18
-3@@ -133,7 +133,8 @@ void nvme_mpath_start_request(struct request *rq)if (!blk_queue_io_stat (disk->queue) ||blk_rq_is_passthrough (rq))return ; -atomic_inc (&ns->ctrl->nr_active); +if (READ_ONCE (ns->head->subsys->iopolicy) ==NVME_IOPOLICY_QD ) +atomic_inc (&ns->ctrl->nr_active);nvme_req (rq)->flags |=NVME_MPATH_IO_STATS ;nvme_req (rq)->start_time =bdev_start_io_acct (disk->part0,req_op (rq), jiffies);@@ -147,7 +148,8 @@ void nvme_mpath_end_request(struct request *rq)if (!(nvme_req (rq)->flags &NVME_MPATH_IO_STATS ))return ; -atomic_dec (&ns->ctrl->nr_active); +if (READ_ONCE (ns->head->subsys->iopolicy) ==NVME_IOPOLICY_QD ) +atomic_dec_if_positive (&ns->ctrl->nr_active);bdev_end_io_acct (ns->head->disk->part0,req_op (rq),blk_rq_bytes (rq) >>SECTOR_SHIFT ,nvme_req (rq)->start_time);@@ -848,6 +850,19 @@ static ssize_t nvme_subsys_iopolicy_show(struct device *dev, nvme_iopolicy_names[READ_ONCE(subsys->iopolicy)]); } +void nvme_subsys_iopolicy_update (struct nvme_subsystem *subsys,int iopolicy) +{ +struct nvme_ctrl *ctrl; + +WRITE_ONCE (subsys->iopolicy, iopolicy); + +mutex_lock (&nvme_subsystems_lock); +list_for_each_entry (ctrl, &subsys->ctrls, subsys_entry) { +atomic_set (&ctrl->nr_active,0 ); + } +mutex_unlock (&nvme_subsystems_lock); +} +static ssize_t nvme_subsys_iopolicy_store (struct device *dev,struct device_attribute *attr,const char *buf,size_t count) {@@ -857,7 +872,7 @@ static ssize_t nvme_subsys_iopolicy_store(struct device *dev,for (i =0 ; i <ARRAY_SIZE (nvme_iopolicy_names); i++) {if (sysfs_streq (buf, nvme_iopolicy_names[i])) { -WRITE_ONCE (subsys->iopolicy, i); +nvme_subsys_iopolicy_update (subsys, i);return count; } }drivers/nvme/host/nvme.h +2
-0@@ -49,6 +49,8 @@ extern struct workqueue_struct *nvme_wq;extern struct workqueue_struct *nvme_reset_wq;extern struct workqueue_struct *nvme_delete_wq; +extern struct mutex nvme_subsystems_lock; +/* * List of workarounds for devices that required behavior not specified in * the standard. Signature · 1 line
2.20.1
Re: [PATCH 2/3] nvme: multipath: only update ctrl->nr_active when using queue-depth iopolicy
I think quiescing all the queues associated with the nvme_ctrl would be undesirable
just to handle changing the iopolicy, which only needs to affect I/O that has not yet
gone through the path selection. It would be better not to make the sysfs write have
to wait, as well (although this could be done with a work item, it seems like overkill).
Probably there is not enough contention for the atomic_dec_if_positive() to loop but I take your point, I'll look at it. The equipment I have here is likely not fast enough
to be sure.
-Ewan
Quoted 105 lines · Uday Shankar
Quoted 4 lines · Ewan D. Milne
The atomic updates of
ctrl->nr_activeare unnecessary when using numa or round-robin iopolicy, so avoid that cost on a per-request basis. Clearnr_activewhen changing iopolicy and do not decrement below zero. (This handles changing the iopolicy while requests are in flight.)Instead of trying to handle a changing iopolicy while requests are in flight, can we quiesce I/O when we change the iopolicy? That should let us simplify/speed up the logic in the I/O path a bit (
atomic_dec_if_positiveseems to hide a cmpxchg loop on most architectures, which can be slower than anatomic_dec).Quoted 91 lines
Signed-off-by Ewan D. Milne <[email protected]>--- drivers/nvme/host/core.c | 2 +- drivers/nvme/host/multipath.c | 21 ++++++++++++++++++--- drivers/nvme/host/nvme.h | 2 ++ 3 files changed, 21 insertions(+), 4 deletions(-)drivers/nvme/host/core.c +1
-1@@ -110,7 +110,7 @@ struct workqueue_struct *nvme_delete_wq;EXPORT_SYMBOL_GPL (nvme_delete_wq);static LIST_HEAD (nvme_subsystems); -static DEFINE_MUTEX (nvme_subsystems_lock); +DEFINE_MUTEX (nvme_subsystems_lock);static DEFINE_IDA (nvme_instance_ida);static dev_t nvme_ctrl_base_chr_devt;drivers/nvme/host/multipath.c +18
-3@@ -133,7 +133,8 @@ void nvme_mpath_start_request(struct request *rq)if (!blk_queue_io_stat (disk->queue) ||blk_rq_is_passthrough (rq))return ; -atomic_inc (&ns->ctrl->nr_active); +if (READ_ONCE (ns->head->subsys->iopolicy) ==NVME_IOPOLICY_QD ) +atomic_inc (&ns->ctrl->nr_active);nvme_req (rq)->flags |=NVME_MPATH_IO_STATS ;nvme_req (rq)->start_time =bdev_start_io_acct (disk->part0,req_op (rq), jiffies);@@ -147,7 +148,8 @@ void nvme_mpath_end_request(struct request *rq)if (!(nvme_req (rq)->flags &NVME_MPATH_IO_STATS ))return ; -atomic_dec (&ns->ctrl->nr_active); +if (READ_ONCE (ns->head->subsys->iopolicy) ==NVME_IOPOLICY_QD ) +atomic_dec_if_positive (&ns->ctrl->nr_active);bdev_end_io_acct (ns->head->disk->part0,req_op (rq),blk_rq_bytes (rq) >>SECTOR_SHIFT ,nvme_req (rq)->start_time);@@ -848,6 +850,19 @@ static ssize_t nvme_subsys_iopolicy_show(struct device *dev, nvme_iopolicy_names[READ_ONCE(subsys->iopolicy)]); } +void nvme_subsys_iopolicy_update (struct nvme_subsystem *subsys,int iopolicy) +{ +struct nvme_ctrl *ctrl; + +WRITE_ONCE (subsys->iopolicy, iopolicy); + +mutex_lock (&nvme_subsystems_lock); +list_for_each_entry (ctrl, &subsys->ctrls, subsys_entry) { +atomic_set (&ctrl->nr_active,0 ); + } +mutex_unlock (&nvme_subsystems_lock); +} +static ssize_t nvme_subsys_iopolicy_store (struct device *dev,struct device_attribute *attr,const char *buf,size_t count) {@@ -857,7 +872,7 @@ static ssize_t nvme_subsys_iopolicy_store(struct device *dev,for (i =0 ; i <ARRAY_SIZE (nvme_iopolicy_names); i++) {if (sysfs_streq (buf, nvme_iopolicy_names[i])) { -WRITE_ONCE (subsys->iopolicy, i); +nvme_subsys_iopolicy_update (subsys, i);return count; } }drivers/nvme/host/nvme.h +2
-1@@ -49,6 +49,8 @@ extern struct workqueue_struct *nvme_wq;extern struct workqueue_struct *nvme_reset_wq;extern struct workqueue_struct *nvme_delete_wq; +extern struct mutex nvme_subsystems_lock; +/* * List of workarounds for devices that required behavior not specified in * the standard. -- 2.20.1
PATCH 3/3 nvme: multipath: Invalidate current_path when changing iopolicy
In reply to Ewan D. MilneWhen switching back to numa from round-robin, current_path may refer to a different path than the one numa would have selected, and it is desirable to have consistent behavior.
--- drivers/nvme/host/multipath.c | 1 + 1 file changed, 1 insertion(+)
drivers/nvme/host/multipath.c +1 -0
@@ -859,6 +859,7 @@ void nvme_subsys_iopolicy_update(struct nvme_subsystem *subsys, int iopolicy)mutex_lock (&nvme_subsystems_lock);list_for_each_entry (ctrl, &subsys->ctrls, subsys_entry) {atomic_set (&ctrl->nr_active,0 ); +nvme_mpath_clear_ctrl_paths (ctrl); }mutex_unlock (&nvme_subsystems_lock); }
Signature · 1 line
2.20.1
Re: [PATCH 3/3] nvme: multipath: Invalidate current_path when changing iopolicy
Quoted 3 lines · Ewan D. Milne
When switching back to numa from round-robin,
current_pathmay refer to a different path than the one numa would have selected, and it is desirable to have consistent behavior.
In general invalidating the current path when switching the I/O policy seems like the right thing to do. So independent of the rest of the work we really should add this:
Quoted 10 lines
drivers/nvme/host/multipath.c +10
-0+static struct nvme_ns *nvme_queue_depth_path (struct nvme_ns_head *head) +{ +struct nvme_ns *best_opt =NULL , *best_nonopt =NULL , *ns; +unsigned int min_depth_opt =UINT_MAX , min_depth_nonopt =UINT_MAX ; +unsigned int depth; + +list_for_each_entry_rcu (ns, &head->list, siblings) { +if (nvme_path_is_disabled (ns)) +continue ; +
drivers/nvme/host/multipath.c static struct nvme_ns *nvme_round_robin_path(struct nvme_ns_head *head, · line 357
Quoted 10 lines
drivers/nvme/host/multipath.c +10
-0+ depth =atomic_read (&ns->ctrl->nr_active); + +switch (ns->ana_state) { +case NVME_ANA_OPTIMIZED : +if (depth < min_depth_opt) { + min_depth_opt = depth; + best_opt = ns; + } +break ; +
nit:- extra whiteline above after break ?
-ck
Quoted 4 lines · Ewan D. Milne
The existing iopolicies are inefficient in some cases, such as the presence of a path with high latency. The round-robin policy would use that path equally with faster paths, which results in sub-optimal performance.
do you have performance numbers for such case ?
Quoted 10 lines
The queue-depth policy instead sends I/O requests down the path with the least amount of requests in its request queue. Paths with lower latency will clear requests more quickly and have less requests in their queues compared to "bad" paths. The aim is to use those paths the most to bring down overall latency.
This implementation adds an atomic variable to the
nvme_ctrlstruct to represent the queue depth. It is updated each time a request specific to that controller starts or ends.
Quoted 8 lines
[edm: patch developed by Thomas Song @ Pure Storage, fixed whitespace
and compilation warnings, updatedMODULE_PARMdescription, and fixed potential issue with ->current_path[] being used]Co-developed-by Thomas Song <[email protected]>Signed-off-by Ewan D. Milne <[email protected]>---
any performance comparison that shows the difference ?
-ck
Yes, we have some graphs. John M. presented them at ALPSS and there were some earlier ones at LSF/MM. I'll see if I can put up the latest set for download.
The basic issue is that with round-robin, requests for most/all of the tagset space
can end up on a path that is responding slowly, so we see a significant imbalance
in path utilization.
-Ewan
Quoted 33 lines · Chaitanya Kulkarni
Quoted 4 lines · Ewan D. Milne
The existing iopolicies are inefficient in some cases, such as the presence of a path with high latency. The round-robin policy would use that path equally with faster paths, which results in sub-optimal performance.
do you have performance numbers for such case ?
Quoted 10 lines
The queue-depth policy instead sends I/O requests down the path with the least amount of requests in its request queue. Paths with lower latency will clear requests more quickly and have less requests in their queues compared to "bad" paths. The aim is to use those paths the most to bring down overall latency.
This implementation adds an atomic variable to the
nvme_ctrlstruct to represent the queue depth. It is updated each time a request specific to that controller starts or ends.Quoted 8 lines
[edm: patch developed by Thomas Song @ Pure Storage, fixed whitespace
and compilation warnings, updatedMODULE_PARMdescription, and fixed potential issue with ->current_path[] being used]Co-developed-by Thomas Song <[email protected]>Signed-off-by Ewan D. Milne <[email protected]>---
any performance comparison that shows the difference ?
-ck
Here are the slides with the performance numbers I presented at ALPSS.
https://people.redhat.com/jmeneghi/ALPSS_2023/NVMe_QD_Multipathing.pdf
This is the same information I posted in:
https://lists.infradead.org/pipermail/linux-nvme/2023-September/042371.html
/John
Quoted 49 lines · Ewan Milne
Yes, we have some graphs. John M. presented them at ALPSS and there were some earlier ones at LSF/MM. I'll see if I can put up the latest set for download.
The basic issue is that with round-robin, requests for most/all of the tagset space
can end up on a path that is responding slowly, so we see a significant imbalance
in path utilization.-Ewan
Quoted 33 lines · Chaitanya Kulkarni
Quoted 4 lines · Ewan D. Milne
The existing iopolicies are inefficient in some cases, such as the presence of a path with high latency. The round-robin policy would use that path equally with faster paths, which results in sub-optimal performance.
do you have performance numbers for such case ?
Quoted 10 lines
The queue-depth policy instead sends I/O requests down the path with the least amount of requests in its request queue. Paths with lower latency will clear requests more quickly and have less requests in their queues compared to "bad" paths. The aim is to use those paths the most to bring down overall latency.
This implementation adds an atomic variable to the
nvme_ctrlstruct to represent the queue depth. It is updated each time a request specific to that controller starts or ends.Quoted 8 lines
[edm: patch developed by Thomas Song @ Pure Storage, fixed whitespace
and compilation warnings, updatedMODULE_PARMdescription, and fixed potential issue with ->current_path[] being used]Co-developed-by Thomas Song <[email protected]>Signed-off-by Ewan D. Milne <[email protected]>---
any performance comparison that shows the difference ?
-ck
Quoted 4 lines · Ewan Milne
Yes, we have some graphs. John M. presented them at ALPSS and there were some earlier ones at LSF/MM. I'll see if I can put up the latest set for download.
Quoted 8 lines
The basic issue is that with round-robin, requests for most/all of the tagset space
can end up on a path that is responding slowly, so we see a significant imbalance
in path utilization.-Ewan
Sounds good, please add a cover-letter on your next version and document those so everyone can see them, I'm pretty sure I'm not the only whose going to ask this question :) ...
Can you also please write a block test (cc me) for this under nvme category ? This needs to be tested every-time we send pull request.
-ck
Keith Busch
maintainerdrivers/nvme/host/multipath.c void nvme_mpath_start_request(struct request *rq) · line 136
Quoted 7 lines · Ewan D. Milne
void
nvme_mpath_default_iopolicy(struct nvme_subsystem *subsys){drivers/nvme/host/multipath.c +1
-0@@ -130,6 +133,7 @@ void nvme_mpath_start_request(struct request *rq)if (!blk_queue_io_stat (disk->queue) ||blk_rq_is_passthrough (rq))return ; +atomic_inc (&ns->ctrl->nr_active);
Did you intend to skip counting nr_active if disk stats are not enabled, or if it's a passthough command?
Also, we should restrict the atomics for other new IO Depth policy.
drivers/nvme/host/multipath.c void nvme_mpath_end_request(struct request *rq) · line 150
Quoted 9 lines
drivers/nvme/host/multipath.c +2
-0nvme_req (rq)->flags |=NVME_MPATH_IO_STATS ;nvme_req (rq)->start_time =bdev_start_io_acct (disk->part0,req_op (rq), jiffies);@@ -142,6 +146,8 @@ void nvme_mpath_end_request(struct request *rq)if (!(nvme_req (rq)->flags &NVME_MPATH_IO_STATS ))return ; + +atomic_dec (&ns->ctrl->nr_active);
And if disk stats were disabled, this dec isn't paired with an inc.
The patch uses the existing flag NVME_MPATH_IO_STATS in the request for simplicity to see if an atomic_inc() was performed before doing the _dec(). If we really need to keep the count for passthrough requests (maybe we do) then we would need another flag.
The reason to use the atomic_dec_if_positive() in the 2nd patch and set the nr_active to zero when changing the iopolicy is so that if e.g. someone changes the iopolicy repeatedly while requests are in-flight the counter will work properly.
(I have a test case and another path to make the nr_active counter visible in sysfs,
btw, if that would be useful.)
-Ewan
Quoted 27 lines · Keith Busch
drivers/nvme/host/multipath.c void nvme_mpath_start_request(struct request *rq) · line 136
Quoted 7 lines · Ewan D. Milne
void
nvme_mpath_default_iopolicy(struct nvme_subsystem *subsys){drivers/nvme/host/multipath.c +1
-0@@ -130,6 +133,7 @@ void nvme_mpath_start_request(struct request *rq)if (!blk_queue_io_stat (disk->queue) ||blk_rq_is_passthrough (rq))return ; +atomic_inc (&ns->ctrl->nr_active);Did you intend to skip counting
nr_activeif disk stats are not enabled, or if it's a passthough command?Also, we should restrict the atomics for other new IO Depth policy.
drivers/nvme/host/multipath.c void nvme_mpath_end_request(struct request *rq) · line 150
Quoted 9 lines
drivers/nvme/host/multipath.c +2
-0nvme_req (rq)->flags |=NVME_MPATH_IO_STATS ;nvme_req (rq)->start_time =bdev_start_io_acct (disk->part0,req_op (rq), jiffies);@@ -142,6 +146,8 @@ void nvme_mpath_end_request(struct request *rq)if (!(nvme_req (rq)->flags &NVME_MPATH_IO_STATS ))return ; + +atomic_dec (&ns->ctrl->nr_active);And if disk stats were disabled, this dec isn't paired with an inc.
Keith Busch
maintainerQuoted 4 lines · Ewan Milne
The patch uses the existing flag
NVME_MPATH_IO_STATSin the request for simplicity to see if anatomic_inc()was performed before doing the_dec(). If we really need to keep the count for passthrough requests (maybe we do) then we would need another flag.
Oh, I see.
I didn't think you wanted to tie path selection to whether or not the disk stats are enabled (and maybe you don't?), so that's why I thought we needed another flag for this.
PATCH v2 1/3 nvme: multipath: Implemented new iopolicy "queue-depth"
In reply to Ewan D. MilneFrom: "Ewan D. Milne" <[email protected]>
The existing iopolicies are inefficient in some cases, such as the presence of a path with high latency. The round-robin policy would use that path equally with faster paths, which results in sub-optimal performance.
The queue-depth policy instead sends I/O requests down the path with the least amount of requests in its request queue. Paths with lower latency will clear requests more quickly and have less requests in their queues compared to "bad" paths. The aim is to use those paths the most to bring down overall latency.
This implementation adds an atomic variable to the nvme_ctrl struct to represent the queue depth. It is updated each time a request specific to that controller starts or ends.
[edm: patch developed by Thomas Song @ Pure Storage, fixed whitespace
and compilation warnings, updated MODULE_PARM description, and fixed potential issue with ->current_path[] being used]
--- drivers/nvme/host/multipath.c | 59 +++++++++++++++++++++++++++++++++-- drivers/nvme/host/nvme.h | 2 ++ 2 files changed, 58 insertions(+), 3 deletions(-)
drivers/nvme/host/multipath.c +56 -3
@@ -17,6 +17,7 @@ MODULE_PARM_DESC(multipath,static const char *nvme_iopolicy_names[] = { [NVME_IOPOLICY_NUMA ] ="numa" , [NVME_IOPOLICY_RR ] ="round-robin" , + [NVME_IOPOLICY_QD ] ="queue-depth" , };static int iopolicy =NVME_IOPOLICY_NUMA ;
@@ -29,6 +30,8 @@ static int nvme_set_iopolicy(const char *val, const struct kernel_param *kp) iopolicy =NVME_IOPOLICY_NUMA ; else if (!strncmp(val, "round-robin", 11)) iopolicy =NVME_IOPOLICY_RR ; + else if (!strncmp(val, "queue-depth", 11)) + iopolicy =NVME_IOPOLICY_QD ; else return -EINVAL;
@@ -43,7 +46,7 @@ static int nvme_get_iopolicy(char *buf, const struct kernel_param *kp)module_param_call (iopolicy, nvme_set_iopolicy, nvme_get_iopolicy, &iopolicy,0644 );MODULE_PARM_DESC (iopolicy, -"Default multipath I/O policy; 'numa' (default) or 'round-robin'" ); +"Default multipath I/O policy; 'numa' (default) , 'round-robin' or 'queue-depth'" );void nvme_mpath_default_iopolicy (struct nvme_subsystem *subsys) {
@@ -130,6 +133,7 @@ void nvme_mpath_start_request(struct request *rq)if (!blk_queue_io_stat (disk->queue) ||blk_rq_is_passthrough (rq))return ; +atomic_inc (&ns->ctrl->nr_active);nvme_req (rq)->flags |=NVME_MPATH_IO_STATS ;nvme_req (rq)->start_time =bdev_start_io_acct (disk->part0,req_op (rq), jiffies);
@@ -142,6 +146,8 @@ void nvme_mpath_end_request(struct request *rq)if (!(nvme_req (rq)->flags &NVME_MPATH_IO_STATS ))return ; + +atomic_dec (&ns->ctrl->nr_active);bdev_end_io_acct (ns->head->disk->part0,req_op (rq),blk_rq_bytes (rq) >>SECTOR_SHIFT ,nvme_req (rq)->start_time);
@@ -331,6 +337,40 @@ static struct nvme_ns *nvme_round_robin_path(struct nvme_ns_head *head,return found; } +static struct nvme_ns *nvme_queue_depth_path (struct nvme_ns_head *head) +{ +struct nvme_ns *best_opt =NULL , *best_nonopt =NULL , *ns; +unsigned int min_depth_opt =UINT_MAX , min_depth_nonopt =UINT_MAX ; +unsigned int depth; + +list_for_each_entry_rcu (ns, &head->list, siblings) { +if (nvme_path_is_disabled (ns)) +continue ; + + depth =atomic_read (&ns->ctrl->nr_active); + +switch (ns->ana_state) { +case NVME_ANA_OPTIMIZED : +if (depth < min_depth_opt) { + min_depth_opt = depth; + best_opt = ns; + } +break ; + +case NVME_ANA_NONOPTIMIZED : +if (depth < min_depth_nonopt) { + min_depth_nonopt = depth; + best_nonopt = ns; + } +break ; +default : +break ; + } + } + +return best_opt ? best_opt : best_nonopt; +} +static inline bool nvme_path_is_optimized (struct nvme_ns *ns) { return nvme_ctrl_state(ns->ctrl) == NVME_CTRL_LIVE &&
@@ -339,15 +379,27 @@ static inline bool nvme_path_is_optimized(struct nvme_ns *ns)inline struct nvme_ns *nvme_find_path (struct nvme_ns_head *head) { -int node =numa_node_id (); +int iopolicy =READ_ONCE (head->subsys->iopolicy); +int node;struct nvme_ns *ns; +/* +* queue-depth iopolicy does not need to reference ->current_path +* but round-robin needs the last path used to advance to the +* next one, and numa will continue to use the last path unless +* it is or has become not optimized +*/ +if (iopolicy ==NVME_IOPOLICY_QD ) +return nvme_queue_depth_path (head); + + node =numa_node_id (); ns =srcu_dereference (head->current_path[node], &head->srcu);if (unlikely (!ns))return __nvme_find_path (head, node); -if (READ_ONCE (head->subsys->iopolicy) ==NVME_IOPOLICY_RR ) +if (iopolicy ==NVME_IOPOLICY_RR )return nvme_round_robin_path (head, node, ns); +if (unlikely (!nvme_path_is_optimized (ns)))return __nvme_find_path (head, node);return ns;
@@ -906,6 +958,7 @@ void nvme_mpath_init_ctrl(struct nvme_ctrl *ctrl)mutex_init (&ctrl->ana_lock);timer_setup (&ctrl->anatt_timer, nvme_anatt_timeout,0 );INIT_WORK (&ctrl->ana_work, nvme_ana_work); +atomic_set (&ctrl->nr_active,0 ); }int nvme_mpath_init_identify (struct nvme_ctrl *ctrl,struct nvme_id_ctrl *id)
drivers/nvme/host/nvme.h +2 -0
@@ -359,6 +359,7 @@ struct nvme_ctrl {size_t ana_log_size;struct timer_list anatt_timer;struct work_struct ana_work; +atomic_t nr_active;#endif #ifdef CONFIG_NVME_HOST_AUTH
@@ -407,6 +408,7 @@ static inline enum nvme_ctrl_state nvme_ctrl_state(struct nvme_ctrl *ctrl)enum nvme_iopolicy {NVME_IOPOLICY_NUMA ,NVME_IOPOLICY_RR , +NVME_IOPOLICY_QD , };struct nvme_subsystem {
Signature · 1 line
2.39.3
PATCH v2 2/3 nvme: multipath: only update ctrl->nr_active when using queue-depth iopolicy
In reply to Ewan D. MilneFrom: "Ewan D. Milne" <[email protected]>
The atomic updates of ctrl->nr_active are unnecessary when using numa or round-robin iopolicy, so avoid that cost on a per-request basis. Clear nr_active when changing iopolicy and do not decrement below zero. (This handles changing the iopolicy while requests are in flight.)
--- drivers/nvme/host/core.c | 2 +- drivers/nvme/host/multipath.c | 21 ++++++++++++++++++--- drivers/nvme/host/nvme.h | 6 ++++++ 3 files changed, 25 insertions(+), 4 deletions(-)
drivers/nvme/host/core.c +1 -1
@@ -110,7 +110,7 @@ struct workqueue_struct *nvme_delete_wq;EXPORT_SYMBOL_GPL (nvme_delete_wq);static LIST_HEAD (nvme_subsystems); -static DEFINE_MUTEX (nvme_subsystems_lock); +DEFINE_MUTEX (nvme_subsystems_lock);static DEFINE_IDA (nvme_instance_ida);static dev_t nvme_ctrl_base_chr_devt;
drivers/nvme/host/multipath.c +18 -3
@@ -133,7 +133,8 @@ void nvme_mpath_start_request(struct request *rq)if (!blk_queue_io_stat (disk->queue) ||blk_rq_is_passthrough (rq))return ; -atomic_inc (&ns->ctrl->nr_active); +if (READ_ONCE (ns->head->subsys->iopolicy) ==NVME_IOPOLICY_QD ) +atomic_inc (&ns->ctrl->nr_active);nvme_req (rq)->flags |=NVME_MPATH_IO_STATS ;nvme_req (rq)->start_time =bdev_start_io_acct (disk->part0,req_op (rq), jiffies);
@@ -147,7 +148,8 @@ void nvme_mpath_end_request(struct request *rq)if (!(nvme_req (rq)->flags &NVME_MPATH_IO_STATS ))return ; -atomic_dec (&ns->ctrl->nr_active); +if (READ_ONCE (ns->head->subsys->iopolicy) ==NVME_IOPOLICY_QD ) +atomic_dec_if_positive (&ns->ctrl->nr_active);bdev_end_io_acct (ns->head->disk->part0,req_op (rq),blk_rq_bytes (rq) >>SECTOR_SHIFT ,nvme_req (rq)->start_time);
@@ -851,6 +853,19 @@ static ssize_t nvme_subsys_iopolicy_show(struct device *dev, nvme_iopolicy_names[READ_ONCE(subsys->iopolicy)]); } +void nvme_subsys_iopolicy_update (struct nvme_subsystem *subsys,int iopolicy) +{ +struct nvme_ctrl *ctrl; + +WRITE_ONCE (subsys->iopolicy, iopolicy); + +mutex_lock (&nvme_subsystems_lock); +list_for_each_entry (ctrl, &subsys->ctrls, subsys_entry) { +atomic_set (&ctrl->nr_active,0 ); + } +mutex_unlock (&nvme_subsystems_lock); +} +static ssize_t nvme_subsys_iopolicy_store (struct device *dev,struct device_attribute *attr,const char *buf,size_t count) {
@@ -860,7 +875,7 @@ static ssize_t nvme_subsys_iopolicy_store(struct device *dev,for (i =0 ; i <ARRAY_SIZE (nvme_iopolicy_names); i++) {if (sysfs_streq (buf, nvme_iopolicy_names[i])) { -WRITE_ONCE (subsys->iopolicy, i); +nvme_subsys_iopolicy_update (subsys, i);return count; } }
drivers/nvme/host/nvme.h +6 -0
@@ -50,6 +50,8 @@ extern struct workqueue_struct *nvme_wq;extern struct workqueue_struct *nvme_reset_wq;extern struct workqueue_struct *nvme_delete_wq; +extern struct mutex nvme_subsystems_lock; +/* * List of workarounds for devices that required behavior not specified in * the standard.
@@ -963,6 +965,7 @@ void nvme_mpath_clear_ctrl_paths(struct nvme_ctrl *ctrl);void nvme_mpath_shutdown_disk (struct nvme_ns_head *head);void nvme_mpath_start_request (struct request *rq);void nvme_mpath_end_request (struct request *rq); +void nvme_subsys_iopolicy_update (struct nvme_subsystem *subsys,int iopolicy);static inline void nvme_trace_bio_complete (struct request *req) {
@@ -1062,6 +1065,9 @@ static inline bool nvme_disk_is_ns_head(struct gendisk *disk) {return false; } +static inline void nvme_subsys_iopolicy_update (struct nvme_subsystem *subsys,int iopolicy) +{ +}#endif /* CONFIG_NVME_MULTIPATH */ struct nvme_zone_info {
Signature · 1 line
2.39.3
v2 cover nvme: queue-depth multipath iopolicy
In reply to Ewan D. MilneI'm re-issuing Ewan's queue-depth patches in preparation for LSFMM
These patches were first show at ALPSS 2023 where I shared the following graphs which measure the IO distribution across 4 active-optimized controllers using the round-robin verses queue-depth iopolicy.
https://people.redhat.com/jmeneghi/ALPSS_2023/NVMe_QD_Multipathing.pdf
Since that time we have continued testing these patches with a number of different nvme-of storage arrays and test bed configurations, and I've codified the tests and methods we use to measure IO distribution
All of my test results, together with the scripts I used to generate these graphs, are available at:
https://github.com/johnmeneghini/iopolicy
Please use the scripts in this repository to do your own testing.
These patches are based on nvme-v6.9
Ewan D. Milne (3):
nvme: multipath: Implemented new iopolicy "queue-depth"
nvme: multipath: only update ctrl->nr_active when using queue-depth
iopolicy
nvme: multipath: Invalidate current_path when changing iopolicy
drivers/nvme/host/core.c | 2 +- drivers/nvme/host/multipath.c | 77 +++++++++++++++++++++++++++++++++-- drivers/nvme/host/nvme.h | 8 ++++ 3 files changed, 82 insertions(+), 5 deletions(-)
Signature · 1 line
2.39.3
PATCH v2 3/3 nvme: multipath: Invalidate current_path when changing iopolicy
In reply to Ewan D. MilneFrom: "Ewan D. Milne" <[email protected]>
When switching back to numa from round-robin, current_path may refer to a different path than the one numa would have selected, and it is desirable to have consistent behavior.
--- drivers/nvme/host/multipath.c | 1 + 1 file changed, 1 insertion(+)
drivers/nvme/host/multipath.c +1 -0
@@ -862,6 +862,7 @@ void nvme_subsys_iopolicy_update(struct nvme_subsystem *subsys, int iopolicy)mutex_lock (&nvme_subsystems_lock);list_for_each_entry (ctrl, &subsys->ctrls, subsys_entry) {atomic_set (&ctrl->nr_active,0 ); +nvme_mpath_clear_ctrl_paths (ctrl); }mutex_unlock (&nvme_subsystems_lock); }
Signature · 1 line
2.39.3