improve quiesce time for large amount of namespaces
Subsystems (3)
Sagi Grimberg
maintainerv2 cover improve quiesce time for large amount of namespaces
This set attempts to improve the quiesce time when using a large set of namespaces, which also improves I/O failover time in a multipath environment.
We improve for both non-blocking hctxs (e.g. pci, fc, rdma) and blocking hctxs (e.g. tcp).
The original patch from Chao was targeted to rdma, hence Patch #4 is just for testing purposes in case testing with nvme-tcp is an issue.
Changes from v1:
- fixed patch #2 wrong leftovers (start_freeze)
Chao Leng (1):
nvme-core: reduce io failover time
Sagi Grimberg (3):
blk-mq: add async quiesce interface for blocking hw queues
nvme: improve quiesce for blocking queues
nvme-rdma: use blocking quiesce interface
block/blk-mq.c | 31 +++++++++++++++++++++++++++++++ drivers/nvme/host/core.c | 20 +++++++++++++++++++- drivers/nvme/host/nvme.h | 1 + drivers/nvme/host/rdma.c | 5 +++-- drivers/nvme/host/tcp.c | 2 +- include/linux/blk-mq.h | 4 ++++ 6 files changed, 59 insertions(+), 4 deletions(-)
Signature · 7 lines
2.25.1 _______________________________________________ Linux-nvme mailing list [email protected] http://lists.infradead.org/mailman/listinfo/linux-nvme
Sagi Grimberg
maintainerRFC PATCH v2 1/4 blk-mq: add async quiesce interface for blocking hw queues
Drivers that use blocking hw queues may have to quiesce a large amount of request queues at once (e.g. controller or adapter reset). These drivers would benefit from an async quiesce interface such that the can trigger quiesce asynchronously and wait for all in parallel.
This leaves the synchronization responsibility to the driver, but adds a convenient interface to quiesce async and wait in a single pass.
--- block/blk-mq.c | 31 +++++++++++++++++++++++++++++++ include/linux/blk-mq.h | 4 ++++ 2 files changed, 35 insertions(+)
block/blk-mq.c +31 -0
@@ -209,6 +209,37 @@ void blk_mq_quiesce_queue_nowait(struct request_queue *q) }EXPORT_SYMBOL_GPL (blk_mq_quiesce_queue_nowait); +void blk_mq_quiesce_blocking_queue_async (struct request_queue *q) +{ +struct blk_mq_hw_ctx *hctx; +unsigned int i; + +blk_mq_quiesce_queue_nowait (q); + +queue_for_each_hw_ctx (q, hctx, i) { +if (!(hctx->flags &BLK_MQ_F_BLOCKING )) +continue ; +init_completion (&hctx->rcu_sync.completion); +init_rcu_head (&hctx->rcu_sync.head); +call_srcu (hctx->srcu, &hctx->rcu_sync.head, wakeme_after_rcu); + } +} +EXPORT_SYMBOL_GPL (blk_mq_quiesce_blocking_queue_async); + +void blk_mq_quiesce_blocking_queue_async_wait (struct request_queue *q) +{ +struct blk_mq_hw_ctx *hctx; +unsigned int i; + +queue_for_each_hw_ctx (q, hctx, i) { +if (!(hctx->flags &BLK_MQ_F_BLOCKING )) +continue ; +wait_for_completion (&hctx->rcu_sync.completion); +destroy_rcu_head (&hctx->rcu_sync.head); + } +} +EXPORT_SYMBOL_GPL (blk_mq_quiesce_blocking_queue_async_wait); +/** * blk_mq_quiesce_queue() - wait until all ongoing dispatches have finished * @q: request queue.
include/linux/blk-mq.h +4 -0
@@ -5,6 +5,7 @@#include <linux/blkdev.h> #include <linux/sbitmap.h> #include <linux/srcu.h> +#include <linux/rcupdate_wait.h> struct blk_mq_tags ;struct blk_flush_queue ;
@@ -170,6 +171,7 @@ struct blk_mq_hw_ctx {*/ struct list_head hctx_list; +struct rcu_synchronize rcu_sync;/** * @srcu: Sleepable RCU. Use as lock when type of the hardware queue is * blocking (BLK_MQ_F_BLOCKING). Must be the last member - see also
@@ -532,6 +534,8 @@ int blk_mq_map_queues(struct blk_mq_queue_map *qmap);void blk_mq_update_nr_hw_queues (struct blk_mq_tag_set *set,int nr_hw_queues);void blk_mq_quiesce_queue_nowait (struct request_queue *q); +void blk_mq_quiesce_blocking_queue_async (struct request_queue *q); +void blk_mq_quiesce_blocking_queue_async_wait (struct request_queue *q);unsigned int blk_mq_rq_cpu (struct request *rq);
Signature · 7 lines
2.25.1 _______________________________________________ Linux-nvme mailing list [email protected] http://lists.infradead.org/mailman/listinfo/linux-nvme
Re: [PATCH rfc v2 1/4] blk-mq: add async quiesce interface for blocking hw queues
Looks great. One suggest: srcu provide the batch sync mechanism, it may be more generic. Weakness: for the same srcu, concurrent batch waiting is not supported. The code just for TINY_SRCU:
---
block/blk-mq.c | 24 ++++++++++++++++++++++++ include/linux/srcu.h | 2 ++ include/linux/srcutiny.h | 1 + kernel/rcu/srcutiny.c | 16 ++++++++++++++++
4 files changed, 43 insertions(+)
block/blk-mq.c +0 -0
@@ -235,6 +235,30 @@ void blk_mq_quiesce_queue(struct request_queue *q)
}
EXPORT_SYMBOL_GPL(blk_mq_quiesce_queue);
+void blk_mq_quiesce_blocking_queue_async(struct request_queue *q) +{
+ struct blk_mq_hw_ctx *hctx;
+ unsigned int i;
+
+ blk_mq_quiesce_queue_nowait(q);
+
+ queue_for_each_hw_ctx(q, hctx, i)
+ if (hctx->flags & BLK_MQ_F_BLOCKING)
+ synchronize_srcu_async(hctx->srcu);
+}
+EXPORT_SYMBOL_GPL(blk_mq_quiesce_blocking_queue_async);
+
+void blk_mq_quiesce_blocking_queue_async_wait(struct request_queue *q) +{
+ struct blk_mq_hw_ctx *hctx;
+ unsigned int i;
+
+ queue_for_each_hw_ctx(q, hctx, i)
+ if (hctx->flags & BLK_MQ_F_BLOCKING)
+ synchronize_srcu_async_wait(hctx->srcu);
+} +EXPORT_SYMBOL_GPL(blk_mq_quiesce_blocking_queue_async_wait); +
/*
* blk_mq_unquiesce_queue() - counterpart of blk_mq_quiesce_queue()
* @q: request queue.
include/linux/srcu.h +0 -0
@@ -60,6 +60,8 @@ void cleanup_srcu_struct(struct srcu_struct *ssp);
int __srcu_read_lock(struct srcu_struct *ssp) __acquires(ssp); void __srcu_read_unlock(struct srcu_struct *ssp, int idx) __releases(ssp); void synchronize_srcu(struct srcu_struct *ssp);
+void synchronize_srcu_async(struct srcu_struct *ssp);
+void synchronize_srcu_async_wait(struct srcu_struct *ssp);
#ifdef CONFIG_DEBUG_LOCK_ALLOC
include/linux/srcutiny.h +0 -0
@@ -23,6 +23,7 @@ struct srcu_struct {
struct rcu_head *srcu_cb_head; /* Pending callbacks: Head. */ struct rcu_head **srcu_cb_tail; /* Pending callbacks: Tail. */ struct work_struct srcu_work; /* For driving grace periods. */ + struct rcu_synchronize rcu_sync;
#ifdef CONFIG_DEBUG_LOCK_ALLOC
struct lockdep_map dep_map;
#endif /* #ifdef CONFIG_DEBUG_LOCK_ALLOC */
kernel/rcu/srcutiny.c +0 -0
@@ -190,6 +190,22 @@ void synchronize_srcu(struct srcu_struct *ssp)
}
EXPORT_SYMBOL_GPL(synchronize_srcu);
+void synchronize_srcu_async(struct srcu_struct *ssp) +{
+ init_rcu_head(&ssp->rcu_sync.head);
+ init_completion(&ssp->rcu_sync.completion);
+ call_srcu(ssp, &ssp->rcu_sync.head, wakeme_after_rcu_batch); +
+}
+EXPORT_SYMBOL_GPL(synchronize_srcu_async);
+
+void synchronize_srcu_async_wait(struct srcu_struct *ssp) +{
+ wait_for_completion(&ssp->rcu_sync.completion);
+ destroy_rcu_head(&ssp->rcu_sync.head);
+}
+EXPORT_SYMBOL_GPL(synchronize_srcu_async_wait);
+
/* Lockdep diagnostics. */
void __init rcu_scheduler_starting(void)
{
Quoted 77 lines · Sagi Grimberg
Drivers that use blocking hw queues may have to quiesce a large amount of request queues at once (e.g. controller or adapter reset). These drivers would benefit from an async quiesce interface such that the can trigger quiesce asynchronously and wait for all in parallel.
This leaves the synchronization responsibility to the driver, but adds a convenient interface to quiesce async and wait in a single pass.
Signed-off-by Sagi Grimberg <[email protected]>--- block/blk-mq.c | 31 +++++++++++++++++++++++++++++++ include/linux/blk-mq.h | 4 ++++ 2 files changed, 35 insertions(+)block/blk-mq.c +31
-0@@ -209,6 +209,37 @@ void blk_mq_quiesce_queue_nowait(struct request_queue *q) }EXPORT_SYMBOL_GPL (blk_mq_quiesce_queue_nowait); +void blk_mq_quiesce_blocking_queue_async (struct request_queue *q) +{ +struct blk_mq_hw_ctx *hctx; +unsigned int i; + +blk_mq_quiesce_queue_nowait (q); + +queue_for_each_hw_ctx (q, hctx, i) { +if (!(hctx->flags &BLK_MQ_F_BLOCKING )) +continue ; +init_completion (&hctx->rcu_sync.completion); +init_rcu_head (&hctx->rcu_sync.head); +call_srcu (hctx->srcu, &hctx->rcu_sync.head, wakeme_after_rcu); + } +} +EXPORT_SYMBOL_GPL (blk_mq_quiesce_blocking_queue_async); + +void blk_mq_quiesce_blocking_queue_async_wait (struct request_queue *q) +{ +struct blk_mq_hw_ctx *hctx; +unsigned int i; + +queue_for_each_hw_ctx (q, hctx, i) { +if (!(hctx->flags &BLK_MQ_F_BLOCKING )) +continue ; +wait_for_completion (&hctx->rcu_sync.completion); +destroy_rcu_head (&hctx->rcu_sync.head); + } +} +EXPORT_SYMBOL_GPL (blk_mq_quiesce_blocking_queue_async_wait); +/** * blk_mq_quiesce_queue() - wait until all ongoing dispatches have finished * @q: request queue. include/linux/blk-mq.h +2
-0@@ -5,6 +5,7 @@#include <linux/blkdev.h> #include <linux/sbitmap.h> #include <linux/srcu.h> +#include <linux/rcupdate_wait.h> struct blk_mq_tags ;struct blk_flush_queue ;@@ -170,6 +171,7 @@ struct blk_mq_hw_ctx {*/ struct list_head hctx_list; +struct rcu_synchronize rcu_sync;/** * @srcu: Sleepable RCU. Use as lock when type of the hardware queue is * blocking (BLK_MQ_F_BLOCKING). Must be the last member - see also
include/linux/blk-mq.h int blk_mq_map_queues(struct blk_mq_queue_map *qmap); · line 541
Quoted 8 lines
include/linux/blk-mq.h +2
-0@@ -532,6 +534,8 @@ int blk_mq_map_queues(struct blk_mq_queue_map *qmap);void blk_mq_update_nr_hw_queues (struct blk_mq_tag_set *set,int nr_hw_queues);void blk_mq_quiesce_queue_nowait (struct request_queue *q); +void blk_mq_quiesce_blocking_queue_async (struct request_queue *q); +void blk_mq_quiesce_blocking_queue_async_wait (struct request_queue *q);unsigned int blk_mq_rq_cpu (struct request *rq);
_______________________________________________
Linux-nvme mailing list
[email protected]
http://lists.infradead.org/mailman/listinfo/linux-nvme
Sagi Grimberg
maintainerRe: [PATCH rfc v2 1/4] blk-mq: add async quiesce interface for blocking hw queues
Quoted 4 lines
Looks great. One suggest: srcu provide the batch sync mechanism, it may be more generic. Weakness: for the same srcu, concurrent batch waiting is not supported. The code just for
TINY_SRCU:
Doesn't make sense to me...
But what I was thinking is not to do this only for blocking queues, we can easily use call_rcu for non-blocking queues as well..
i.e.
--
void blk_mq_quiesce_queue_async(struct request_queue *q)
{
struct blk_mq_hw_ctx *hctx;
unsigned int i;
blk_mq_quiesce_queue_nowait(q);
queue_for_each_hw_ctx(q, hctx, i) {
init_completion(&hctx->rcu_sync.completion);
init_rcu_head(&hctx->rcu_sync.head);
if (hctx->flags & BLK_MQ_F_BLOCKING)
call_srcu(hctx->srcu, &hctx->rcu_sync.head,
wakeme_after_rcu);
else
call_rcu(hctx->srcu, &hctx->rcu_sync.head,
wakeme_after_rcu);
}
}EXPORT_SYMBOL_GPL(blk_mq_quiesce_blocking_queue_async);
void blk_mq_quiesce_queue_async_wait(struct request_queue *q)
{
struct blk_mq_hw_ctx *hctx;
unsigned int i;
queue_for_each_hw_ctx(q, hctx, i) {
wait_for_completion(&hctx->rcu_sync.completion);
destroy_rcu_head(&hctx->rcu_sync.head);
}
}EXPORT_SYMBOL_GPL(blk_mq_quiesce_blocking_queue_async_wait); --
And then in nvme we always use that...
Thoughts?
_______________________________________________
Linux-nvme mailing list
[email protected]
http://lists.infradead.org/mailman/listinfo/linux-nvme
Sagi Grimberg
maintainerRFC PATCH v2 2/4 nvme: improve quiesce for blocking queues
In reply to Sagi Grimbergnvme transports that use blocking hw queues (e.g. nvme-tcp) currently will synchronize queue quiesce for each namespace at once. This can slow down failover time (which first quiesce all ns queues) if we have a large amount of namespaces. Instead, we want to use an async interface and do the namespaces quiesce in parallel rather than serially.
Introduce nvme_stop_blocking_queues for transports that use blocking hw queues and convert nvme-tcp to use the new interface.
--- drivers/nvme/host/core.c | 13 +++++++++++++ drivers/nvme/host/nvme.h | 1 + drivers/nvme/host/tcp.c | 2 +- 3 files changed, 15 insertions(+), 1 deletion(-)
drivers/nvme/host/core.c +13 -0
@@ -4529,6 +4529,19 @@ void nvme_start_freeze(struct nvme_ctrl *ctrl) }EXPORT_SYMBOL_GPL (nvme_start_freeze); +void nvme_stop_blocking_queues (struct nvme_ctrl *ctrl) +{ +struct nvme_ns *ns; + +down_read (&ctrl->namespaces_rwsem); +list_for_each_entry (ns, &ctrl->namespaces, list) +blk_mq_quiesce_blocking_queue_async (ns->queue); +list_for_each_entry (ns, &ctrl->namespaces, list) +blk_mq_quiesce_blocking_queue_async_wait (ns->queue); +up_read (&ctrl->namespaces_rwsem); +} +EXPORT_SYMBOL_GPL (nvme_stop_blocking_queues); +void nvme_stop_queues (struct nvme_ctrl *ctrl) {struct nvme_ns *ns;
drivers/nvme/host/nvme.h +1 -0
@@ -568,6 +568,7 @@ int nvme_sec_submit(void *data, u16 spsp, u8 secp, void *buffer, size_t len,void nvme_complete_async_event (struct nvme_ctrl *ctrl,__le16 status,volatile union nvme_result *res); +void nvme_stop_blocking_queues (struct nvme_ctrl *ctrl);void nvme_stop_queues (struct nvme_ctrl *ctrl);void nvme_start_queues (struct nvme_ctrl *ctrl);void nvme_kill_queues (struct nvme_ctrl *ctrl);
drivers/nvme/host/tcp.c +1 -1
@@ -1887,7 +1887,7 @@ static void nvme_tcp_teardown_io_queues(struct nvme_ctrl *ctrl, {if (ctrl->queue_count <=1 )return ; -nvme_stop_queues (ctrl); +nvme_stop_blocking_queues (ctrl);nvme_tcp_stop_io_queues (ctrl);if (ctrl->tagset) { blk_mq_tagset_busy_iter(ctrl->tagset,
Signature · 7 lines
2.25.1 _______________________________________________ Linux-nvme mailing list [email protected] http://lists.infradead.org/mailman/listinfo/linux-nvme
Sagi Grimberg
maintainerRFC PATCH v2 3/4 nvme-core: reduce io failover time
In reply to Sagi GrimbergFrom: Chao Leng <[email protected]>
We test nvme over roce fail over with multipath when 1000 namespaces configured, io pause more than 10 seconds. The reason: nvme_stop_queues will quiesce all queues for each namespace when io timeout cause path error. Quiesce queue wait all ongoing dispatches finished through synchronize_rcu, need more than 10 milliseconds for each wait, thus io pause more than 10 seconds.
To reduce io pause time, nvme_stop_queues use blk_mq_quiesce_queue_nowait to quiesce the queue, nvme_stop_queues wait all ongoing dispatches completed after all queues has been quiesced.
--- drivers/nvme/host/core.c | 7 ++++++- 1 file changed, 6 insertions(+), 1 deletion(-)
drivers/nvme/host/core.c +6 -1
@@ -4548,8 +4548,13 @@ void nvme_stop_queues(struct nvme_ctrl *ctrl)down_read (&ctrl->namespaces_rwsem);list_for_each_entry (ns, &ctrl->namespaces, list) -blk_mq_quiesce_queue (ns->queue); +blk_mq_quiesce_queue_nowait (ns->queue);up_read (&ctrl->namespaces_rwsem); +/* +* BLK_MQ_F_BLOCKING drivers should never call us +*/ +WARN_ON_ONCE (ctrl->tagset.flags &BLK_MQ_F_BLOCKING ); +synchronize_rcu (); }EXPORT_SYMBOL_GPL (nvme_stop_queues);
Signature · 7 lines
2.25.1 _______________________________________________ Linux-nvme mailing list [email protected] http://lists.infradead.org/mailman/listinfo/linux-nvme
Sagi Grimberg
maintainerRe: [PATCH rfc v2 3/4] nvme-core: reduce io failover time
Quoted 6 lines
drivers/nvme/host/core.c +0
-0
drivers/nvme/host/core.c void nvme_stop_queues(struct nvme_ctrl *ctrl) · line 4556
Quoted 10 lines
drivers/nvme/host/core.c +5
-1@@ -4548,8 +4548,13 @@ void nvme_stop_queues(struct nvme_ctrl *ctrl)down_read (&ctrl->namespaces_rwsem);list_for_each_entry (ns, &ctrl->namespaces, list) -blk_mq_quiesce_queue (ns->queue); +blk_mq_quiesce_queue_nowait (ns->queue);up_read (&ctrl->namespaces_rwsem); +/* +* BLK_MQ_F_BLOCKING drivers should never call us +*/ +WARN_ON_ONCE (ctrl->tagset.flags &BLK_MQ_F_BLOCKING );
Woops^2 tagset is a pointer... will resend v3 after I get some feedback...
_______________________________________________
Linux-nvme mailing list
[email protected]
http://lists.infradead.org/mailman/listinfo/linux-nvme
Re: [PATCH rfc v2 3/4] nvme-core: reduce io failover time
In reply to Sagi GrimbergQuoted 24 lines · Sagi Grimberg
From: Chao Leng <[email protected]>
We test nvme over roce fail over with multipath when 1000 namespaces configured, io pause more than 10 seconds. The reason:
nvme_stop_queueswill quiesce all queues for each namespace when io timeout cause path error. Quiesce queue wait all ongoing dispatches finished throughsynchronize_rcu, need more than 10 milliseconds for each wait, thus io pause more than 10 seconds.To reduce io pause time,
nvme_stop_queuesuseblk_mq_quiesce_queue_nowaitto quiesce the queue,nvme_stop_queueswait all ongoing dispatches completed after all queues has been quiesced.Signed-off-by Chao Leng <[email protected]>Signed-off-by Sagi Grimberg <[email protected]>--- drivers/nvme/host/core.c | 7 ++++++- 1 file changed, 6 insertions(+), 1 deletion(-)drivers/nvme/host/core.c +0
-0
drivers/nvme/host/core.c void nvme_stop_queues(struct nvme_ctrl *ctrl) · line 4556
Quoted 10 lines
drivers/nvme/host/core.c +5
-1@@ -4548,8 +4548,13 @@ void nvme_stop_queues(struct nvme_ctrl *ctrl)down_read (&ctrl->namespaces_rwsem);list_for_each_entry (ns, &ctrl->namespaces, list) -blk_mq_quiesce_queue (ns->queue); +blk_mq_quiesce_queue_nowait (ns->queue);up_read (&ctrl->namespaces_rwsem); +/* +* BLK_MQ_F_BLOCKING drivers should never call us +*/ +WARN_ON_ONCE (ctrl->tagset.flags &BLK_MQ_F_BLOCKING );
There is a little compile bug, ctrl->tagset is a pointer. it should:
WARN_ON_ONCE(ctrl->tagset->flags & BLK_MQ_F_BLOCKING);
drivers/nvme/host/core.c void nvme_stop_queues(struct nvme_ctrl *ctrl) · line 4560
Quoted 4 lines
drivers/nvme/host/core.c +1
-0+synchronize_rcu (); }EXPORT_SYMBOL_GPL (nvme_stop_queues);
_______________________________________________
Linux-nvme mailing list
[email protected]
http://lists.infradead.org/mailman/listinfo/linux-nvme
Sagi Grimberg
maintainerPATCH v2 4/4 nvme-rdma: use blocking quiesce interface
In reply to Sagi Grimberg--- drivers/nvme/host/rdma.c | 5 +++-- 1 file changed, 3 insertions(+), 2 deletions(-)
drivers/nvme/host/rdma.c +3 -2
@@ -793,6 +793,7 @@ static struct blk_mq_tag_set *nvme_rdma_alloc_tagset(struct nvme_ctrl *nctrl, set->queue_depth =NVME_AQ_MQ_TAG_DEPTH ; set->reserved_tags =2 ;/* connect + keep-alive */ set->numa_node = nctrl->numa_node; + set->flags =BLK_MQ_F_BLOCKING ; set->cmd_size =sizeof (struct nvme_rdma_request ) +NVME_RDMA_DATA_SGL_SIZE ; set->driver_data = ctrl;
@@ -806,7 +807,7 @@ static struct blk_mq_tag_set *nvme_rdma_alloc_tagset(struct nvme_ctrl *nctrl, set->queue_depth = nctrl->sqsize +1 ; set->reserved_tags =1 ;/* fabric connect */ set->numa_node = nctrl->numa_node; - set->flags =BLK_MQ_F_SHOULD_MERGE ; + set->flags =BLK_MQ_F_SHOULD_MERGE |BLK_MQ_F_BLOCKING ; set->cmd_size =sizeof (struct nvme_rdma_request ) +NVME_RDMA_DATA_SGL_SIZE ;if (nctrl->max_integrity_segments)
@@ -1008,7 +1009,7 @@ static void nvme_rdma_teardown_io_queues(struct nvme_rdma_ctrl *ctrl, bool remove) {if (ctrl->ctrl.queue_count >1 ) { -nvme_stop_queues (&ctrl->ctrl); +nvme_stop_blocking_queues (&ctrl->ctrl);nvme_rdma_stop_io_queues (ctrl);if (ctrl->ctrl.tagset) { blk_mq_tagset_busy_iter(ctrl->ctrl.tagset,
Signature · 7 lines
2.25.1 _______________________________________________ Linux-nvme mailing list [email protected] http://lists.infradead.org/mailman/listinfo/linux-nvme