improve quiesce time for large amount of namespaces
Subsystems (2)
Sagi Grimberg
maintainercover improve quiesce time for large amount of namespaces
This set attempts to improve the quiesce time when using a large set of namespaces, which also improves I/O failover time in a multipath environment.
We improve for both non-blocking hctxs (e.g. pci, fc, rdma) and blocking hctxs (e.g. tcp).
The original patch from Chao was targeted to rdma, hence Patch #4 is just for testing purposes in case testing with nvme-tcp is an issue.
Chao Leng (1):
nvme-core: reduce io failover time
Sagi Grimberg (3):
blk-mq: add async quiesce interface for blocking hw queues
nvme: improve quiesce for blocking queues
nvme-rdma: use blocking quiesce interface
block/blk-mq.c | 31 +++++++++++++++++++++++++++++++ drivers/nvme/host/core.c | 20 +++++++++++++++++++- drivers/nvme/host/nvme.h | 1 + drivers/nvme/host/rdma.c | 5 +++-- drivers/nvme/host/tcp.c | 1 + include/linux/blk-mq.h | 4 ++++ 6 files changed, 59 insertions(+), 3 deletions(-)
Signature · 7 lines
2.25.1 _______________________________________________ Linux-nvme mailing list [email protected] http://lists.infradead.org/mailman/listinfo/linux-nvme
Sagi Grimberg
maintainerRFC PATCH 1/4 blk-mq: add async quiesce interface for blocking hw queues
Drivers that use blocking hw queues may have to quiesce a large amount of request queues at once (e.g. controller or adapter reset). These drivers would benefit from an async quiesce interface such that the can trigger quiesce asynchronously and wait for all in parallel.
This leaves the synchronization responsibility to the driver, but adds a convenient interface to quiesce async and wait in a single pass.
--- block/blk-mq.c | 31 +++++++++++++++++++++++++++++++ include/linux/blk-mq.h | 4 ++++ 2 files changed, 35 insertions(+)
block/blk-mq.c +31 -0
@@ -209,6 +209,37 @@ void blk_mq_quiesce_queue_nowait(struct request_queue *q) }EXPORT_SYMBOL_GPL (blk_mq_quiesce_queue_nowait); +void blk_mq_quiesce_blocking_queue_async (struct request_queue *q) +{ +struct blk_mq_hw_ctx *hctx; +unsigned int i; + +blk_mq_quiesce_queue_nowait (q); + +queue_for_each_hw_ctx (q, hctx, i) { +if (!(hctx->flags &BLK_MQ_F_BLOCKING )) +continue ; +init_completion (&hctx->rcu_sync.completion); +init_rcu_head (&hctx->rcu_sync.head); +call_srcu (hctx->srcu, &hctx->rcu_sync.head, wakeme_after_rcu); + } +} +EXPORT_SYMBOL_GPL (blk_mq_quiesce_blocking_queue_async); + +void blk_mq_quiesce_blocking_queue_async_wait (struct request_queue *q) +{ +struct blk_mq_hw_ctx *hctx; +unsigned int i; + +queue_for_each_hw_ctx (q, hctx, i) { +if (!(hctx->flags &BLK_MQ_F_BLOCKING )) +continue ; +wait_for_completion (&hctx->rcu_sync.completion); +destroy_rcu_head (&hctx->rcu_sync.head); + } +} +EXPORT_SYMBOL_GPL (blk_mq_quiesce_blocking_queue_async_wait); +/** * blk_mq_quiesce_queue() - wait until all ongoing dispatches have finished * @q: request queue.
include/linux/blk-mq.h +4 -0
@@ -5,6 +5,7 @@#include <linux/blkdev.h> #include <linux/sbitmap.h> #include <linux/srcu.h> +#include <linux/rcupdate_wait.h> struct blk_mq_tags ;struct blk_flush_queue ;
@@ -170,6 +171,7 @@ struct blk_mq_hw_ctx {*/ struct list_head hctx_list; +struct rcu_synchronize rcu_sync;/** * @srcu: Sleepable RCU. Use as lock when type of the hardware queue is * blocking (BLK_MQ_F_BLOCKING). Must be the last member - see also
@@ -532,6 +534,8 @@ int blk_mq_map_queues(struct blk_mq_queue_map *qmap);void blk_mq_update_nr_hw_queues (struct blk_mq_tag_set *set,int nr_hw_queues);void blk_mq_quiesce_queue_nowait (struct request_queue *q); +void blk_mq_quiesce_blocking_queue_async (struct request_queue *q); +void blk_mq_quiesce_blocking_queue_async_wait (struct request_queue *q);unsigned int blk_mq_rq_cpu (struct request *rq);
Signature · 7 lines
2.25.1 _______________________________________________ Linux-nvme mailing list [email protected] http://lists.infradead.org/mailman/listinfo/linux-nvme
Sagi Grimberg
maintainerRFC PATCH 2/4 nvme: improve quiesce for blocking queues
In reply to Sagi Grimbergnvme transports that use blocking hw queues (e.g. nvme-tcp) currently will synchronize queue quiesce for each namespace at once. This can slow down failover time (which first quiesce all ns queues) if we have a large amount of namespaces. Instead, we want to use an async interface and do the namespaces quiesce in parallel rather than serially.
Introduce nvme_stop_blocking_queues for transports that use blocking hw queues and convert nvme-tcp to use the new interface.
--- drivers/nvme/host/core.c | 13 +++++++++++++ drivers/nvme/host/nvme.h | 1 + drivers/nvme/host/tcp.c | 1 + 3 files changed, 15 insertions(+)
drivers/nvme/host/core.c +13 -0
@@ -4529,6 +4529,19 @@ void nvme_start_freeze(struct nvme_ctrl *ctrl) }EXPORT_SYMBOL_GPL (nvme_start_freeze); +void nvme_stop_blocking_queues (struct nvme_ctrl *ctrl) +{ +struct nvme_ns *ns; + +down_read (&ctrl->namespaces_rwsem); +list_for_each_entry (ns, &ctrl->namespaces, list) +_blk_mq_quiesce_blocking_queue_async (ns->queue); +list_for_each_entry (ns, &ctrl->namespaces, list) +_blk_mq_quiesce_blocking_queue_async_wait (ns->queue); +up_read (&ctrl->namespaces_rwsem); +} +EXPORT_SYMBOL_GPL (nvme_stop_blocking_queues); +void nvme_stop_queues (struct nvme_ctrl *ctrl) {struct nvme_ns *ns;
drivers/nvme/host/nvme.h +1 -0
@@ -568,6 +568,7 @@ int nvme_sec_submit(void *data, u16 spsp, u8 secp, void *buffer, size_t len,void nvme_complete_async_event (struct nvme_ctrl *ctrl,__le16 status,volatile union nvme_result *res); +void nvme_stop_blocking_queues (struct nvme_ctrl *ctrl);void nvme_stop_queues (struct nvme_ctrl *ctrl);void nvme_start_queues (struct nvme_ctrl *ctrl);void nvme_kill_queues (struct nvme_ctrl *ctrl);
drivers/nvme/host/tcp.c +1 -0
@@ -1887,6 +1887,7 @@ static void nvme_tcp_teardown_io_queues(struct nvme_ctrl *ctrl, {if (ctrl->queue_count <=1 )return ; +nvme_start_freeze (ctrl);nvme_stop_queues (ctrl);nvme_tcp_stop_io_queues (ctrl);if (ctrl->tagset) {
Signature · 7 lines
2.25.1 _______________________________________________ Linux-nvme mailing list [email protected] http://lists.infradead.org/mailman/listinfo/linux-nvme
Sagi Grimberg
maintainerRe: [PATCH rfc 2/4] nvme: improve quiesce for blocking queues
drivers/nvme/host/tcp.c static void nvme_tcp_teardown_io_queues(struct nvme_ctrl *ctrl, · line 1890
Quoted 10 lines
drivers/nvme/host/tcp.c +1
-0@@ -1887,6 +1887,7 @@ static void nvme_tcp_teardown_io_queues(struct nvme_ctrl *ctrl, {if (ctrl->queue_count <=1 )return ; +nvme_start_freeze (ctrl);
Woops... wrong patch...
_______________________________________________
Linux-nvme mailing list
[email protected]
http://lists.infradead.org/mailman/listinfo/linux-nvme
Sagi Grimberg
maintainerRFC PATCH 3/4 nvme-core: reduce io failover time
In reply to Sagi GrimbergFrom: Chao Leng <[email protected]>
We test nvme over roce fail over with multipath when 1000 namespaces configured, io pause more than 10 seconds. The reason: nvme_stop_queues will quiesce all queues for each namespace when io timeout cause path error. Quiesce queue wait all ongoing dispatches finished through synchronize_rcu, need more than 10 milliseconds for each wait, thus io pause more than 10 seconds.
To reduce io pause time, nvme_stop_queues use blk_mq_quiesce_queue_nowait to quiesce the queue, nvme_stop_queues wait all ongoing dispatches completed after all queues has been quiesced.
--- drivers/nvme/host/core.c | 7 ++++++- 1 file changed, 6 insertions(+), 1 deletion(-)
drivers/nvme/host/core.c +6 -1
@@ -4548,8 +4548,13 @@ void nvme_stop_queues(struct nvme_ctrl *ctrl)down_read (&ctrl->namespaces_rwsem);list_for_each_entry (ns, &ctrl->namespaces, list) -blk_mq_quiesce_queue (ns->queue); +blk_mq_quiesce_queue_nowait (ns->queue);up_read (&ctrl->namespaces_rwsem); +/* +* BLK_MQ_F_BLOCKING drivers should never call us +*/ +WARN_ON_ONCE (ctrl->tagset.flags &BLK_MQ_F_BLOCKING ); +synchronize_rcu (); }EXPORT_SYMBOL_GPL (nvme_stop_queues);
Signature · 7 lines
2.25.1 _______________________________________________ Linux-nvme mailing list [email protected] http://lists.infradead.org/mailman/listinfo/linux-nvme
Sagi Grimberg
maintainerPATCH 4/4 nvme-rdma: use blocking quiesce interface
In reply to Sagi Grimberg--- drivers/nvme/host/rdma.c | 5 +++-- 1 file changed, 3 insertions(+), 2 deletions(-)
drivers/nvme/host/rdma.c +3 -2
@@ -793,6 +793,7 @@ static struct blk_mq_tag_set *nvme_rdma_alloc_tagset(struct nvme_ctrl *nctrl, set->queue_depth =NVME_AQ_MQ_TAG_DEPTH ; set->reserved_tags =2 ;/* connect + keep-alive */ set->numa_node = nctrl->numa_node; + set->flags =BLK_MQ_F_BLOCKING ; set->cmd_size =sizeof (struct nvme_rdma_request ) +NVME_RDMA_DATA_SGL_SIZE ; set->driver_data = ctrl;
@@ -806,7 +807,7 @@ static struct blk_mq_tag_set *nvme_rdma_alloc_tagset(struct nvme_ctrl *nctrl, set->queue_depth = nctrl->sqsize +1 ; set->reserved_tags =1 ;/* fabric connect */ set->numa_node = nctrl->numa_node; - set->flags =BLK_MQ_F_SHOULD_MERGE ; + set->flags =BLK_MQ_F_SHOULD_MERGE |BLK_MQ_F_BLOCKING ; set->cmd_size =sizeof (struct nvme_rdma_request ) +NVME_RDMA_DATA_SGL_SIZE ;if (nctrl->max_integrity_segments)
@@ -1008,7 +1009,7 @@ static void nvme_rdma_teardown_io_queues(struct nvme_rdma_ctrl *ctrl, bool remove) {if (ctrl->ctrl.queue_count >1 ) { -nvme_stop_queues (&ctrl->ctrl); +nvme_stop_blocking_queues (&ctrl->ctrl);nvme_rdma_stop_io_queues (ctrl);if (ctrl->ctrl.tagset) { blk_mq_tagset_busy_iter(ctrl->ctrl.tagset,
Signature · 7 lines
2.25.1 _______________________________________________ Linux-nvme mailing list [email protected] http://lists.infradead.org/mailman/listinfo/linux-nvme