Skip to contentSkip to conversation

blk-mq: implement queue quiesce via percpu_ref for BLK_MQ_F_BLOCKING

PATCH Superseded Ming Lei 3 participants 10 messages Sep 9, 2020 – Sep 10, 2020

Subsystems (2)

v4 of 8 Superseded Latest is v8 →

lore
Share & tools
lore

Ming Lei

author
#0 lore raw

v4 cover blk-mq: implement queue quiesce via percpu_ref for BLK_MQ_F_BLOCKING

Hi Jens,

The 1st patch add .mq_quiesce_mutex for serializing quiesce/unquiesce, and prepares for replacing srcu with percpu_ref.

The 2nd patch replaces srcu with percpu_ref.

The 3rd patch adds tagset quiesce interface.

The 4th patch applies tagset quiesce interface for NVMe subsystem.

V4:
- remove .mq_quiesce_mutex, and switch to test_and_[set|clear] for avoiding duplicated quiesce action
- pass blktests(block, nvme)

V3:
- add tagset quiesce interface
- apply tagset quiesce interface for NVMe
- pass blktests(block, nvme)

V2:
- add .mq_quiesce_lock
- add comment on patch 2 wrt. handling hctx_lock() failure
- trivial patch style change

Ming Lei (3):
block: use test_and_{clear|test}_bit to set/clear QUEUE_FLAG_QUIESCED
blk-mq: implement queue quiesce via percpu_ref for BLK_MQ_F_BLOCKING
blk-mq: add tagset quiesce interface

Sagi Grimberg (1):
nvme: use blk_mq_[un]quiesce_tagset

 block/blk-core.c         |  13 +++
 block/blk-mq-sysfs.c     |   2 -
 block/blk-mq.c           | 173 +++++++++++++++++++++++++--------------
 block/blk-sysfs.c        |   6 +-
 block/blk.h              |   2 +
 drivers/nvme/host/core.c |  19 ++---
 include/linux/blk-mq.h   |  10 +--
 include/linux/blkdev.h   |   4 +
 8 files changed, 146 insertions(+), 83 deletions(-)
Cc Hannes Reinecke <[email protected]>
Cc Sagi Grimberg <[email protected]>
Cc Bart Van Assche <[email protected]>
Cc Johannes Thumshirn <[email protected]>
Cc Chao Leng <[email protected]>
Signature · 1 line
2.25.2
author
#1 lore raw

PATCH v4 1/4 block: use test_and_{clear|test}_bit to set/clear QUEUE_FLAG_QUIESCED

Prepare for replacing srcu with percpu-refcount for implementing queue quiesce.

The following patch needs to avoid duplicated quiesce action for BLK_MQ_F_BLOCKING, so use test_and_{clear|test}_bit to set/clear QUEUE_FLAG_QUIESCED.

Cc Sagi Grimberg <[email protected]>
Cc Bart Van Assche <[email protected]>
Cc Johannes Thumshirn <[email protected]>
Cc Chao Leng <[email protected]>
---
 block/blk-core.c | 13 +++++++++++++
 block/blk-mq.c   | 11 ++++++++---
 block/blk.h      |  2 ++
 3 files changed, 23 insertions(+), 3 deletions(-)
block/blk-core.c +13 -0
index 093649bd252e..de440733609c 100644
@@ -107,6 +107,19 @@ bool blk_queue_flag_test_and_set(unsigned int flag, struct request_queue *q)
 }
 EXPORT_SYMBOL_GPL(blk_queue_flag_test_and_set);
 
+/**
+ * blk_queue_flag_test_and_clear - atomically test and clear a queue flag
+ * @flag: flag to be clear
+ * @q: request queue
+ *
+ * Returns the previous value of @flag - 0 if the flag was not set and 1 if
+ * the flag was set.
+ */
+bool blk_queue_flag_test_and_clear(unsigned int flag, struct request_queue *q)
+{
+	return test_and_clear_bit(flag, &q->queue_flags);
+}
+
 void blk_rq_init(struct request_queue *q, struct request *rq)
 {
 	memset(rq, 0, sizeof(*rq));
block/blk-mq.c +8 -3
index 4abb71459f94..efd17a80fcdf 100644
@@ -199,13 +199,18 @@ void blk_mq_unfreeze_queue(struct request_queue *q)
 }
 EXPORT_SYMBOL_GPL(blk_mq_unfreeze_queue);
 
+static bool __blk_mq_quiesce_queue_nowait(struct request_queue *q)
+{
+	return blk_queue_flag_test_and_set(QUEUE_FLAG_QUIESCED, q);
+}
+
 /*
  * FIXME: replace the scsi_internal_device_*block_nowait() calls in the
  * mpt3sas driver such that this function can be removed.
  */
 void blk_mq_quiesce_queue_nowait(struct request_queue *q)
 {
-	blk_queue_flag_set(QUEUE_FLAG_QUIESCED, q);
+	__blk_mq_quiesce_queue_nowait(q);
 }
 EXPORT_SYMBOL_GPL(blk_mq_quiesce_queue_nowait);
 
@@ -224,7 +229,7 @@ void blk_mq_quiesce_queue(struct request_queue *q)
 	unsigned int i;
 	bool rcu = false;
 
-	blk_mq_quiesce_queue_nowait(q);
+	__blk_mq_quiesce_queue_nowait(q);
 
 	queue_for_each_hw_ctx(q, hctx, i) {
 		if (hctx->flags & BLK_MQ_F_BLOCKING)
@@ -246,7 +251,7 @@ EXPORT_SYMBOL_GPL(blk_mq_quiesce_queue);
  */
 void blk_mq_unquiesce_queue(struct request_queue *q)
 {
-	blk_queue_flag_clear(QUEUE_FLAG_QUIESCED, q);
+	blk_queue_flag_test_and_clear(QUEUE_FLAG_QUIESCED, q);
 
 	/* dispatch requests which are inserted during quiescing */
 	blk_mq_run_hw_queues(q, true);
block/blk.h +2 -0
index c08762e10b04..312a060ea2a2 100644
@@ -448,4 +448,6 @@ int bio_add_hw_page(struct request_queue *q, struct bio *bio,
 		struct page *page, unsigned int len, unsigned int offset,
 		unsigned int max_sectors, bool *same_page);
 
+bool blk_queue_flag_test_and_clear(unsigned int flag, struct request_queue *q);
+
 #endif /* BLK_INTERNAL_H */
Signature · 1 line
2.25.2
author
#2 lore raw

PATCH v4 2/4 blk-mq: implement queue quiesce via percpu_ref for BLK_MQ_F_BLOCKING

In reply to Ming Lei

In case of BLK_MQ_F_BLOCKING, blk-mq uses SRCU to mark read critical section during dispatching request, then request queue quiesce is based on SRCU. What we want to get is low cost added in fast path.

With percpu-ref, it is cleaner and simpler & enough for implementing queue quiesce. The main requirement is to make sure all read sections to observe QUEUE_FLAG_QUIESCED once blk_mq_quiesce_queue() returns.

Also it becomes much easier to add interface of async queue quiesce.

Meantime memory footprint can be reduced with per-request-queue percpu-ref.

From implementation viewpoint, in fast path, not see percpu_ref is slower than SRCU, and srcu tree(default option in most distributions) could be slower since memory barrier is required in both lock & unlock, and rcu_read_lock()/rcu_read_unlock() should be much cheap than smp_mb().

1) percpu_ref just hold the rcu_read_lock, then run a check & increase/decrease on the percpu variable:

rcu_read_lock()
if (__ref_is_percpu(ref, &percpu_count))

	this_cpu_inc(*percpu_count);

rcu_read_unlock()

2) srcu tree:

        idx = READ_ONCE(ssp->srcu_idx) & 0x1;
        this_cpu_inc(ssp->sda->srcu_lock_count[idx]);
        smp_mb(); /* B */  /* Avoid leaking the critical section. */

Also from my test on null_blk(blocking), not observe percpu-ref performs worse than srcu, see the following test:

1) test steps:

rmmod null_blk > /dev/null 2>&1
modprobe null_blk nr_devices=1 submit_queues=1 blocking=1

fio --bs=4k --size=512G  --rw=randread --norandommap --direct=1 --ioengine=libaio \
	--iodepth=64 --runtime=60 --group_reporting=1  --name=nullb0 \
	--filename=/dev/nullb0 --numjobs=32

test machine: HP DL380, 16 cpu cores, 2 threads per core, dual sockets/numa, Intel(R) Xeon(R) Silver 4110 CPU @ 2.10GHz

2) test result:
- srcu quiesce: 6063K IOPS
- percpu-ref quiesce: 6113K IOPS

Cc Sagi Grimberg <[email protected]>
Cc Bart Van Assche <[email protected]>
Cc Johannes Thumshirn <[email protected]>
Cc Chao Leng <[email protected]>
---
 block/blk-mq-sysfs.c   |   2 -
 block/blk-mq.c         | 121 +++++++++++++++++++++--------------------
 block/blk-sysfs.c      |   6 +-
 include/linux/blk-mq.h |   8 ---
 include/linux/blkdev.h |   4 ++
 5 files changed, 72 insertions(+), 69 deletions(-)
block/blk-mq-sysfs.c +0 -2
index 062229395a50..799db7937105 100644
@@ -38,8 +38,6 @@ static void blk_mq_hw_sysfs_release(struct kobject *kobj)
 
 	cancel_delayed_work_sync(&hctx->run_work);
 
-	if (hctx->flags & BLK_MQ_F_BLOCKING)
-		cleanup_srcu_struct(hctx->srcu);
 	blk_free_flush_queue(hctx->fq);
 	sbitmap_free(&hctx->ctx_map);
 	free_cpumask_var(hctx->cpumask);
block/blk-mq.c +63 -58

244 lines not on this page · raw message

block/blk-sysfs.c +5 -1
index 7dda709f3ccb..56b6c045e30c 100644
@@ -941,9 +941,13 @@ static void blk_release_queue(struct kobject *kobj)
 
 	blk_queue_free_zone_bitmaps(q);
 
-	if (queue_is_mq(q))
+	if (queue_is_mq(q)) {
 		blk_mq_release(q);
 
+		if (q->tag_set->flags & BLK_MQ_F_BLOCKING)
+			percpu_ref_exit(&q->dispatch_counter);
+	}
+
 	blk_trace_shutdown(q);
 	mutex_lock(&q->debugfs_mutex);
 	debugfs_remove_recursive(q->debugfs_dir);
include/linux/blk-mq.h +0 -8
index b23eeca4d677..df642055f02c 100644
@@ -4,7 +4,6 @@
 
 #include <linux/blkdev.h>
 #include <linux/sbitmap.h>
-#include <linux/srcu.h>
 
 struct blk_mq_tags;
 struct blk_flush_queue;
@@ -173,13 +172,6 @@ struct blk_mq_hw_ctx {
 	 * q->unused_hctx_list.
 	 */
 	struct list_head	hctx_list;
-
-	/**
-	 * @srcu: Sleepable RCU. Use as lock when type of the hardware queue is
-	 * blocking (BLK_MQ_F_BLOCKING). Must be the last member - see also
-	 * blk_mq_hw_ctx_size().
-	 */
-	struct srcu_struct	srcu[];
 };
 
 /**
include/linux/blkdev.h +4 -0
index 7b1e53084799..de8b51bd402b 100644
@@ -572,6 +572,10 @@ struct request_queue {
 	 */
 	struct mutex		mq_freeze_lock;
 
+	/* only used for BLK_MQ_F_BLOCKING */
+	struct percpu_ref	dispatch_counter;
+	wait_queue_head_t	mq_quiesce_wq;
+
 	struct blk_mq_tag_set	*tag_set;
 	struct list_head	tag_set_list;
 	struct bio_set		bio_split;
Signature · 1 line
2.25.2
maintainer
#3 lore raw
Quoted 15 lines · Ming Lei

void blk_mq_quiesce_queue(struct request_queue *q) {

block/blk-mq.c +4 -7
-	struct blk_mq_hw_ctx *hctx;
-	unsigned int i;
-	bool rcu = false;
+	bool blocking = !!(q->tag_set->flags & BLK_MQ_F_BLOCKING);
+	bool was_quiesced =__blk_mq_quiesce_queue_nowait(q);
 
-	__blk_mq_quiesce_queue_nowait(q);
+	if (!was_quiesced && blocking)
+		percpu_ref_kill(&q->dispatch_counter);
 
-	queue_for_each_hw_ctx(q, hctx, i) {
-		if (hctx->flags & BLK_MQ_F_BLOCKING)
-			synchronize_srcu(hctx->srcu);

block/blk-mq.c EXPORT_SYMBOL_GPL(blk_mq_quiesce_queue_nowait); · line 239

Quoted 10 lines
block/blk-mq.c +4 -4
-		else
-			rcu = true;
-	}
-	if (rcu)
+	if (blocking)
+		wait_event(q->mq_quiesce_wq,
+				percpu_ref_is_zero(&q->dispatch_counter));
+	else
 		synchronize_rcu();
 }

In the previous version, you had ensured no thread can unquiesce a queue while another is waiting for quiescence. Now that the locking is gone, a thread could unquiesce the queue before percpu_ref reaches zero, so the wait_event() may never complete on the resurrected percpu_ref.

I don't think any drivers do such a thing today, but it just looks like the implementation leaves open the possibility.

_______________________________________________
Linux-nvme mailing list
[email protected]
http://lists.infradead.org/mailman/listinfo/linux-nvme

maintainer
#4 lore raw
Quoted 23 lines
Quoted 23 lines
  void blk_mq_quiesce_queue(struct request_queue *q)
  {
block/blk-mq.c +7 -11
-	struct blk_mq_hw_ctx *hctx;
-	unsigned int i;
-	bool rcu = false;
+	bool blocking = !!(q->tag_set->flags & BLK_MQ_F_BLOCKING);
+	bool was_quiesced =__blk_mq_quiesce_queue_nowait(q);
 
-	__blk_mq_quiesce_queue_nowait(q);
+	if (!was_quiesced && blocking)
+		percpu_ref_kill(&q->dispatch_counter);
 
-	queue_for_each_hw_ctx(q, hctx, i) {
-		if (hctx->flags & BLK_MQ_F_BLOCKING)
-			synchronize_srcu(hctx->srcu);
-		else
-			rcu = true;
-	}
-	if (rcu)
+	if (blocking)
+		wait_event(q->mq_quiesce_wq,
+				percpu_ref_is_zero(&q->dispatch_counter));
Quoted 8 lines

block/blk-mq.c EXPORT_SYMBOL_GPL(blk_mq_quiesce_queue_nowait); · line 239

Quoted 3 lines
block/blk-mq.c +1 -0
+	else
  		synchronize_rcu();
  }

In the previous version, you had ensured no thread can unquiesce a queue while another is waiting for quiescence. Now that the locking is gone, a thread could unquiesce the queue before percpu_ref reaches zero, so the wait_event() may never complete on the resurrected percpu_ref.

Yea, where did that go?

author
#5 lore raw
Quoted 30 lines · Sagi Grimberg
Quoted 21 lines
Quoted 21 lines
  void blk_mq_quiesce_queue(struct request_queue *q)
  {
block/blk-mq.c +7 -11
-	struct blk_mq_hw_ctx *hctx;
-	unsigned int i;
-	bool rcu = false;
+	bool blocking = !!(q->tag_set->flags & BLK_MQ_F_BLOCKING);
+	bool was_quiesced =__blk_mq_quiesce_queue_nowait(q);
-	__blk_mq_quiesce_queue_nowait(q);
+	if (!was_quiesced && blocking)
+		percpu_ref_kill(&q->dispatch_counter);
-	queue_for_each_hw_ctx(q, hctx, i) {
-		if (hctx->flags & BLK_MQ_F_BLOCKING)
-			synchronize_srcu(hctx->srcu);
-		else
-			rcu = true;
-	}
-	if (rcu)
+	if (blocking)
+		wait_event(q->mq_quiesce_wq,
+				percpu_ref_is_zero(&q->dispatch_counter));
Quoted 8 lines

block/blk-mq.c EXPORT_SYMBOL_GPL(blk_mq_quiesce_queue_nowait); · line 239

Quoted 3 lines
block/blk-mq.c +1 -0
+	else
  		synchronize_rcu();
  }

In the previous version, you had ensured no thread can unquiesce a queue while another is waiting for quiescence. Now that the locking is gone, a thread could unquiesce the queue before percpu_ref reaches zero, so the wait_event() may never complete on the resurrected percpu_ref.

Quoted 1 line

Yea, where did that go?

The mutex is removed because:

1) As Bart mentioned, blk_mq_quiesce_queue() may be called in context which doesn't allow sleep.

2) Both percpu_ref_kill() and percpu_ref_resurrect() have been protected by one global spinlock, so both two can be run concurrently.

3) warning may be triggered when percpu_ref_kill() is run on one DEAD percpu-refcount, or when percpu_ref_resurrect() is run on one live percpu-refcount. We can avoid the warning with test_and_{clear|test}_bit exactly by running the actual quiesce/unquiesce action only once.

Thanks,
Ming

author
#6 lore raw
In reply to Keith Busch
Quoted 32 lines · Keith Busch
Quoted 15 lines · Ming Lei

void blk_mq_quiesce_queue(struct request_queue *q) {

(quoted diff) +4 -7
-	struct blk_mq_hw_ctx *hctx;
-	unsigned int i;
-	bool rcu = false;
+	bool blocking = !!(q->tag_set->flags & BLK_MQ_F_BLOCKING);
+	bool was_quiesced =__blk_mq_quiesce_queue_nowait(q);
 
-	__blk_mq_quiesce_queue_nowait(q);
+	if (!was_quiesced && blocking)
+		percpu_ref_kill(&q->dispatch_counter);
 
-	queue_for_each_hw_ctx(q, hctx, i) {
-		if (hctx->flags & BLK_MQ_F_BLOCKING)
-			synchronize_srcu(hctx->srcu);

Review

Quoted 10 lines
(quoted diff) +4 -4
-		else
-			rcu = true;
-	}
-	if (rcu)
+	if (blocking)
+		wait_event(q->mq_quiesce_wq,
+				percpu_ref_is_zero(&q->dispatch_counter));
+	else
 		synchronize_rcu();
 }

In the previous version, you had ensured no thread can unquiesce a queue while another is waiting for quiescence. Now that the locking is gone, a thread could unquiesce the queue before percpu_ref reaches zero, so the wait_event() may never complete on the resurrected percpu_ref.

Quoted 2 lines

I don't think any drivers do such a thing today, but it just looks like the implementation leaves open the possibility.

This driver can cause bigger trouble if it unquiesces its queue which is being quiesced and still not done.

However, we can avoid that by the following delta change:

block/blk-mq.c +8 -1
index 7669fe815cf9..5632727d71fa 100644
@@ -225,9 +225,16 @@ static void __blk_mq_quiesce_queue(struct request_queue *q, bool wait)
 	if (!wait)
 		return;
 
+	/*
+	 * In case of F_BLOCKING, if driver unquiesces its queue which is being
+	 * quiesced and still not done, it can cause bigger trouble, and we simply
+	 * return & warn once for avoiding hang here.
+	 */
 	if (blocking)
 		wait_event(q->mq_quiesce_wq,
-				percpu_ref_is_zero(&q->dispatch_counter));
+				percpu_ref_is_zero(&q->dispatch_counter) ||
+				WARN_ON_ONCE(!percpu_ref_is_dying(
+						&q->dispatch_counter)));
 	else
 		synchronize_rcu();
 }

Thanks,
Ming

_______________________________________________
Linux-nvme mailing list
[email protected]
http://lists.infradead.org/mailman/listinfo/linux-nvme

maintainer
#7 lore raw
Quoted 53 lines · Ming Lei
Quoted 32 lines · Keith Busch
Quoted 15 lines · Ming Lei

void blk_mq_quiesce_queue(struct request_queue *q) {

(quoted diff) +4 -7
-	struct blk_mq_hw_ctx *hctx;
-	unsigned int i;
-	bool rcu = false;
+	bool blocking = !!(q->tag_set->flags & BLK_MQ_F_BLOCKING);
+	bool was_quiesced =__blk_mq_quiesce_queue_nowait(q);
 
-	__blk_mq_quiesce_queue_nowait(q);
+	if (!was_quiesced && blocking)
+		percpu_ref_kill(&q->dispatch_counter);
 
-	queue_for_each_hw_ctx(q, hctx, i) {
-		if (hctx->flags & BLK_MQ_F_BLOCKING)
-			synchronize_srcu(hctx->srcu);

Review

Quoted 10 lines
(quoted diff) +4 -4
-		else
-			rcu = true;
-	}
-	if (rcu)
+	if (blocking)
+		wait_event(q->mq_quiesce_wq,
+				percpu_ref_is_zero(&q->dispatch_counter));
+	else
 		synchronize_rcu();
 }

In the previous version, you had ensured no thread can unquiesce a queue while another is waiting for quiescence. Now that the locking is gone, a thread could unquiesce the queue before percpu_ref reaches zero, so the wait_event() may never complete on the resurrected percpu_ref.

Quoted 2 lines

I don't think any drivers do such a thing today, but it just looks like the implementation leaves open the possibility.

This driver can cause bigger trouble if it unquiesces its queue which is being quiesced and still not done.

However, we can avoid that by the following delta change:

block/blk-mq.c +4 -0
index 7669fe815cf9..5632727d71fa 100644
@@ -225,9 +225,16 @@ static void __blk_mq_quiesce_queue(struct request_queue *q, bool wait)
 	if (!wait)
 		return;
 
+	/*
+	 * In case of F_BLOCKING, if driver unquiesces its queue which is being
+	 * quiesced and still not done, it can cause bigger trouble, and we simply
+	 * return & warn once for avoiding hang here.

block/blk-mq.c static void __blk_mq_quiesce_queue(struct request_queue *q, bool wait) · line 240

Quoted 10 lines
block/blk-mq.c +4 -1
+	 */
 	if (blocking)
 		wait_event(q->mq_quiesce_wq,
-				percpu_ref_is_zero(&q->dispatch_counter));
+				percpu_ref_is_zero(&q->dispatch_counter) ||
+				WARN_ON_ONCE(!percpu_ref_is_dying(
+						&q->dispatch_counter)));
 	else
 		synchronize_rcu();
 }

Yeah, I'm okay with this. A warning and return should be good to indicate driver sequence errors. So if you just want to fold the above into this patch, then I don't think I have any remaining concerns.

The other race condition is if unquiesce resurrects the ref before quiesce kills it, and there's already a warning there too.

_______________________________________________
Linux-nvme mailing list
[email protected]
http://lists.infradead.org/mailman/listinfo/linux-nvme

author
#8 lore raw

PATCH v4 3/4 blk-mq: add tagset quiesce interface

In reply to Ming Lei

drivers that have shared tagsets may need to quiesce potentially a lot of request queues that all share a single tagset (e.g. nvme). Add an interface to quiesce all the queues on a given tagset. This interface is useful because it can speedup the quiesce by doing it in parallel.

For tagsets that have BLK_MQ_F_BLOCKING set, we kill request queue's dispatch percpu-refcount such that all of them wait for the counter becoming zero. For tagsets that don't have BLK_MQ_F_BLOCKING set, we simply call a single synchronize_rcu as this is sufficient.

This patch is against Sagi's original post.

Cc Sagi Grimberg <[email protected]>
Cc Bart Van Assche <[email protected]>
Cc Johannes Thumshirn <[email protected]>
Cc Chao Leng <[email protected]>
---
 block/blk-mq.c         | 59 +++++++++++++++++++++++++++++++++++-------
 include/linux/blk-mq.h |  2 ++
 2 files changed, 51 insertions(+), 10 deletions(-)
block/blk-mq.c +49 -10
index 37263ed48a8a..7669fe815cf9 100644
@@ -214,16 +214,7 @@ void blk_mq_quiesce_queue_nowait(struct request_queue *q)
 }
 EXPORT_SYMBOL_GPL(blk_mq_quiesce_queue_nowait);
 
-/**
- * blk_mq_quiesce_queue() - wait until all ongoing dispatches have finished
- * @q: request queue.
- *
- * Note: this function does not prevent that the struct request end_io()
- * callback function is invoked. Once this function is returned, we make
- * sure no dispatch can happen until the queue is unquiesced via
- * blk_mq_unquiesce_queue().
- */
-void blk_mq_quiesce_queue(struct request_queue *q)
+static void __blk_mq_quiesce_queue(struct request_queue *q, bool wait)
 {
 	bool blocking = !!(q->tag_set->flags & BLK_MQ_F_BLOCKING);
 	bool was_quiesced =__blk_mq_quiesce_queue_nowait(q);
@@ -231,12 +222,29 @@ void blk_mq_quiesce_queue(struct request_queue *q)
 	if (!was_quiesced && blocking)
 		percpu_ref_kill(&q->dispatch_counter);
 
+	if (!wait)
+		return;
+
 	if (blocking)
 		wait_event(q->mq_quiesce_wq,
 				percpu_ref_is_zero(&q->dispatch_counter));
 	else
 		synchronize_rcu();
 }
+
+/**
+ * blk_mq_quiesce_queue() - wait until all ongoing dispatches have finished
+ * @q: request queue.
+ *
+ * Note: this function does not prevent that the struct request end_io()
+ * callback function is invoked. Once this function is returned, we make
+ * sure no dispatch can happen until the queue is unquiesced via
+ * blk_mq_unquiesce_queue().
+ */
+void blk_mq_quiesce_queue(struct request_queue *q)
+{
+	__blk_mq_quiesce_queue(q, true);
+}
 EXPORT_SYMBOL_GPL(blk_mq_quiesce_queue);
 
 /*
@@ -258,6 +266,37 @@ void blk_mq_unquiesce_queue(struct request_queue *q)
 }
 EXPORT_SYMBOL_GPL(blk_mq_unquiesce_queue);
 
+void blk_mq_quiesce_tagset(struct blk_mq_tag_set *set)
+{
+	struct request_queue *q;
+
+	mutex_lock(&set->tag_list_lock);
+	list_for_each_entry(q, &set->tag_list, tag_set_list)
+		__blk_mq_quiesce_queue(q, false);
+
+	/* wait until all queues' quiesce is done */
+	if (set->flags & BLK_MQ_F_BLOCKING) {
+		list_for_each_entry(q, &set->tag_list, tag_set_list)
+			wait_event(q->mq_quiesce_wq,
+				   percpu_ref_is_zero(&q->dispatch_counter));
+	} else {
+		synchronize_rcu();
+	}
+	mutex_unlock(&set->tag_list_lock);
+}
+EXPORT_SYMBOL_GPL(blk_mq_quiesce_tagset);
+
+void blk_mq_unquiesce_tagset(struct blk_mq_tag_set *set)
+{
+	struct request_queue *q;
+
+	mutex_lock(&set->tag_list_lock);
+	list_for_each_entry(q, &set->tag_list, tag_set_list)
+		blk_mq_unquiesce_queue(q);
+	mutex_unlock(&set->tag_list_lock);
+}
+EXPORT_SYMBOL_GPL(blk_mq_unquiesce_tagset);
+
 void blk_mq_wake_waiters(struct request_queue *q)
 {
 	struct blk_mq_hw_ctx *hctx;
include/linux/blk-mq.h +2 -0
index df642055f02c..90da3582b91d 100644
@@ -519,6 +519,8 @@ int blk_mq_map_queues(struct blk_mq_queue_map *qmap);
 void blk_mq_update_nr_hw_queues(struct blk_mq_tag_set *set, int nr_hw_queues);
 
 void blk_mq_quiesce_queue_nowait(struct request_queue *q);
+void blk_mq_quiesce_tagset(struct blk_mq_tag_set *set);
+void blk_mq_unquiesce_tagset(struct blk_mq_tag_set *set);
 
 unsigned int blk_mq_rq_cpu(struct request *rq);
 
Signature · 1 line
2.25.2
author
#9 lore raw

PATCH v4 4/4 nvme: use blk_mq_[un]quiesce_tagset

In reply to Ming Lei

From: Sagi Grimberg <[email protected]>

All controller namespaces share the same tagset, so we can use this interface which does the optimal operation for parallel quiesce based on the tagset type (e.g. blocking tagsets and non-blocking tagsets).

Cc Sagi Grimberg <[email protected]>
Cc Bart Van Assche <[email protected]>
Cc Johannes Thumshirn <[email protected]>
Cc Chao Leng <[email protected]>

Add code to unquiesce ctrl->connect_q in nvme_stop_queues(), meantime avoid to call blk_mq_quiesce_tagset()/blk_mq_unquiesce_tagset() if this tagset isn't initialized.

---
 drivers/nvme/host/core.c | 19 +++++++++----------
 1 file changed, 9 insertions(+), 10 deletions(-)
drivers/nvme/host/core.c +9 -10
index ea1fa41fbba8..a6af8978a3ba 100644
@@ -4623,23 +4623,22 @@ EXPORT_SYMBOL_GPL(nvme_start_freeze);
 
 void nvme_stop_queues(struct nvme_ctrl *ctrl)
 {
-	struct nvme_ns *ns;
+	if (list_empty_careful(&ctrl->namespaces))
+		return;
 
-	down_read(&ctrl->namespaces_rwsem);
-	list_for_each_entry(ns, &ctrl->namespaces, list)
-		blk_mq_quiesce_queue(ns->queue);
-	up_read(&ctrl->namespaces_rwsem);
+	blk_mq_quiesce_tagset(ctrl->tagset);
+
+	if (ctrl->connect_q)
+		blk_mq_unquiesce_queue(ctrl->connect_q);
 }
 EXPORT_SYMBOL_GPL(nvme_stop_queues);
 
 void nvme_start_queues(struct nvme_ctrl *ctrl)
 {
-	struct nvme_ns *ns;
+	if (list_empty_careful(&ctrl->namespaces))
+		return;
 
-	down_read(&ctrl->namespaces_rwsem);
-	list_for_each_entry(ns, &ctrl->namespaces, list)
-		blk_mq_unquiesce_queue(ns->queue);
-	up_read(&ctrl->namespaces_rwsem);
+	blk_mq_unquiesce_tagset(ctrl->tagset);
 }
 EXPORT_SYMBOL_GPL(nvme_start_queues);
 
Signature · 1 line
2.25.2