Skip to contentSkip to conversation

block: Improve IOPS by removing the fairness code

PATCH Superseded Bart Van Assche 2 participants 4 messages Mar 25, 2024 – Mar 28, 2024 Block layer

v2 of 7 Superseded Latest is v7 →

lore
Share & tools
lore
#0 lore raw

PATCH v2 1/1 block: Improve IOPS by removing the fairness code

There is an algorithm in the block layer for maintaining fairness across queues that share a tag set. The sbitmap implementation has improved so much that we don't need the block layer fairness algorithm anymore and that we can rely on the sbitmap implementation to guarantee fairness.

This patch removes the following code and structure members:
- The function hctx_may_queue().
- blk_mq_hw_ctx.nr_active and request_queue.nr_active_requests_shared_tags and also all the code that modifies these two member variables.

On my test setup (x86 VM with 72 CPU cores) this patch results in 2.9% more IOPS. IOPS have been measured as follows:

$ modprobe null_blk nr_devices=1 completion_nsec=0
$ fio --bs=4096 --disable_clat=1 --disable_slat=1 --group_reporting=1 \
      --gtod_reduce=1 --invalidate=1 --ioengine=psync --ioscheduler=none \
      --norandommap --runtime=60 --rw=randread --thread --time_based=1 \
      --buffered=0 --numjobs=64 --name=/dev/nullb0 --filename=/dev/nullb0
Cc Christoph Hellwig <[email protected]>
Cc Hannes Reinecke <[email protected]>
Cc Ming Lei <[email protected]>
Cc Yu Kuai <[email protected]>

---

Changes compared with v1: changed a blk_mq_tagset_busy_iter() call into a
blk_mq_all_tag_iter() call in the debugfs code to preserve the current semantics of the 'active' attribute.

 block/blk-core.c       |   2 -
 block/blk-mq-debugfs.c |  22 ++++++++-
 block/blk-mq-tag.c     |   4 --
 block/blk-mq.c         |  17 +------
 block/blk-mq.h         | 100 -----------------------------------------
 include/linux/blk-mq.h |   6 ---
 include/linux/blkdev.h |   2 -
 7 files changed, 22 insertions(+), 131 deletions(-)
block/blk-core.c +0 -2
index a16b5abdbbf5..57dfa4612b43 100644
@@ -425,8 +425,6 @@ struct request_queue *blk_alloc_queue(struct queue_limits *lim, int node_id)
 
 	q->node = node_id;
 
-	atomic_set(&q->nr_active_requests_shared_tags, 0);
-
 	timer_setup(&q->timeout, blk_rq_timed_out_timer, 0);
 	INIT_WORK(&q->timeout_work, blk_timeout_work);
 	INIT_LIST_HEAD(&q->icq_list);
block/blk-mq-debugfs.c +21 -1
index 94668e72ab09..8e0c7774d6ec 100644
@@ -479,11 +479,31 @@ static int hctx_sched_tags_bitmap_show(void *data, struct seq_file *m)
 	return res;
 }
 
+struct count_active_params {
+	struct blk_mq_hw_ctx	*hctx;
+	int			*active;
+};
+
+static bool hctx_count_active(struct request *rq, void *data)
+{
+	const struct count_active_params *params = data;
+
+	if (rq->mq_hctx == params->hctx)
+		(*params->active)++;
+
+	return true;
+}
+
 static int hctx_active_show(void *data, struct seq_file *m)
 {
 	struct blk_mq_hw_ctx *hctx = data;
+	int active = 0;
+	struct count_active_params params = { .hctx = hctx, .active = &active };
+
+	blk_mq_all_tag_iter(hctx->sched_tags ?: hctx->tags, hctx_count_active,
+			    &params);
 
-	seq_printf(m, "%d\n", __blk_mq_active_requests(hctx));
+	seq_printf(m, "%d\n", active);
 	return 0;
 }
 
block/blk-mq-tag.c +0 -4
index cc57e2dd9a0b..25334bfcabf8 100644
@@ -105,10 +105,6 @@ void __blk_mq_tag_idle(struct blk_mq_hw_ctx *hctx)
 static int __blk_mq_get_tag(struct blk_mq_alloc_data *data,
 			    struct sbitmap_queue *bt)
 {
-	if (!data->q->elevator && !(data->flags & BLK_MQ_REQ_RESERVED) &&
-			!hctx_may_queue(data->hctx, bt))
-		return BLK_MQ_NO_TAG;
-
 	if (data->shallow_depth)
 		return sbitmap_queue_get_shallow(bt, data->shallow_depth);
 	else
block/blk-mq.c +1 -16
index 555ada922cf0..46c1252a38da 100644
@@ -424,8 +424,6 @@ __blk_mq_alloc_requests_batch(struct blk_mq_alloc_data *data)
 		rq_list_add(data->cached_rq, rq);
 		nr++;
 	}
-	if (!(data->rq_flags & RQF_SCHED_TAGS))
-		blk_mq_add_active_requests(data->hctx, nr);
 	/* caller already holds a reference, add for remainder */
 	percpu_ref_get_many(&data->q->q_usage_counter, nr - 1);
 	data->nr_tags -= nr;
@@ -510,8 +508,6 @@ static struct request *__blk_mq_alloc_requests(struct blk_mq_alloc_data *data)
 		goto retry;
 	}
 
-	if (!(data->rq_flags & RQF_SCHED_TAGS))
-		blk_mq_inc_active_requests(data->hctx);
 	rq = blk_mq_rq_ctx_init(data, blk_mq_tags_from_data(data), tag);
 	blk_mq_rq_time_init(rq, alloc_time_ns);
 	return rq;
@@ -671,8 +667,6 @@ struct request *blk_mq_alloc_request_hctx(struct request_queue *q,
 	tag = blk_mq_get_tag(&data);
 	if (tag == BLK_MQ_NO_TAG)
 		goto out_queue_exit;
-	if (!(data.rq_flags & RQF_SCHED_TAGS))
-		blk_mq_inc_active_requests(data.hctx);
 	rq = blk_mq_rq_ctx_init(&data, blk_mq_tags_from_data(&data), tag);
 	blk_mq_rq_time_init(rq, alloc_time_ns);
 	rq->__data_len = 0;
@@ -712,10 +706,8 @@ static void __blk_mq_free_request(struct request *rq)
 	blk_pm_mark_last_busy(rq);
 	rq->mq_hctx = NULL;
 
-	if (rq->tag != BLK_MQ_NO_TAG) {
-		blk_mq_dec_active_requests(hctx);
+	if (rq->tag != BLK_MQ_NO_TAG)
 		blk_mq_put_tag(hctx->tags, ctx, rq->tag);
-	}
 	if (sched_tag != BLK_MQ_NO_TAG)
 		blk_mq_put_tag(hctx->sched_tags, ctx, sched_tag);
 	blk_mq_sched_restart(hctx);
@@ -1069,8 +1061,6 @@ static inline void blk_mq_flush_tag_batch(struct blk_mq_hw_ctx *hctx,
 {
 	struct request_queue *q = hctx->queue;
 
-	blk_mq_sub_active_requests(hctx, nr_tags);
-
 	blk_mq_put_tags(hctx->tags, tag_array, nr_tags);
 	percpu_ref_put_many(&q->q_usage_counter, nr_tags);
 }
@@ -1765,9 +1755,6 @@ bool __blk_mq_alloc_driver_tag(struct request *rq)
 	if (blk_mq_tag_is_reserved(rq->mq_hctx->sched_tags, rq->internal_tag)) {
 		bt = &rq->mq_hctx->tags->breserved_tags;
 		tag_offset = 0;
-	} else {
-		if (!hctx_may_queue(rq->mq_hctx, bt))
-			return false;
 	}
 
 	tag = __sbitmap_queue_get(bt);
@@ -1775,7 +1762,6 @@ bool __blk_mq_alloc_driver_tag(struct request *rq)
 		return false;
 
 	rq->tag = tag + tag_offset;
-	blk_mq_inc_active_requests(rq->mq_hctx);
 	return true;
 }
 
@@ -3708,7 +3694,6 @@ blk_mq_alloc_hctx(struct request_queue *q, struct blk_mq_tag_set *set,
 	if (!zalloc_cpumask_var_node(&hctx->cpumask, gfp, node))
 		goto free_hctx;
 
-	atomic_set(&hctx->nr_active, 0);
 	if (node == NUMA_NO_NODE)
 		node = set->numa_node;
 	hctx->numa_node = node;
block/blk-mq.h +0 -100

115 lines not on this page · raw message

include/linux/blk-mq.h +0 -6
index d3d8fd8e229b..a066ea77148f 100644
@@ -398,12 +398,6 @@ struct blk_mq_hw_ctx {
 	/** @queue_num: Index of this hardware queue. */
 	unsigned int		queue_num;
 
-	/**
-	 * @nr_active: Number of active requests. Only used when a tag set is
-	 * shared across request queues.
-	 */
-	atomic_t		nr_active;
-
 	/** @cpuhp_online: List to store request if CPU is going to die */
 	struct hlist_node	cpuhp_online;
 	/** @cpuhp_dead: List to store request if some CPU die. */
include/linux/blkdev.h +0 -2
index f9b87c39cab0..9b1d2eb3eb6b 100644
@@ -445,8 +445,6 @@ struct request_queue {
 	struct timer_list	timeout;
 	struct work_struct	timeout_work;
 
-	atomic_t		nr_active_requests_shared_tags;
-
 	unsigned int		required_elevator_features;
 
 	struct blk_mq_tags	*sched_shared_tags;
#1 lore raw
Quoted 8 lines · Bart Van Assche

On my test setup (x86 VM with 72 CPU cores) this patch results in 2.9% more IOPS. IOPS have been measured as follows:

$ modprobe null_blk nr_devices=1 completion_nsec=0
$ fio --bs=4096 --disable_clat=1 --disable_slat=1 --group_reporting=1 \
      --gtod_reduce=1 --invalidate=1 --ioengine=psync --ioscheduler=none \
      --norandommap --runtime=60 --rw=randread --thread --time_based=1 \
      --buffered=0 --numjobs=64 --name=/dev/nullb0 --filename=/dev/nullb0

And how does it behave when you have multiple request_queues sharing a tag_set with very unevent use that could cause one to starve all the tags? Because that is what this code exists for.

#2 lore raw
Quoted 10 lines · Christoph Hellwig
Quoted 8 lines · Bart Van Assche

On my test setup (x86 VM with 72 CPU cores) this patch results in 2.9% more IOPS. IOPS have been measured as follows:

$ modprobe null_blk nr_devices=1 completion_nsec=0
$ fio --bs=4096 --disable_clat=1 --disable_slat=1 --group_reporting=1 \
       --gtod_reduce=1 --invalidate=1 --ioengine=psync --ioscheduler=none \
       --norandommap --runtime=60 --rw=randread --thread --time_based=1 \
       --buffered=0 --numjobs=64 --name=/dev/nullb0 --filename=/dev/nullb0
Quoted 3 lines

And how does it behave when you have multiple request_queues sharing a tag_set with very unevent use that could cause one to starve all the tags? Because that is what this code exists for.

Hi Christoph,

The proposed test case block/035 includes the following test:
* Create a first request queue with completion time 1 ms and queue depth 64.
* Create a second request queue with completion time 100 ms and that shares the tag set of the first request queue.
* Submit I/O to both request queues.

If I run that test I see a queue depth of about 60 for the second request queue and a queue depth of about 4 for the first request queue. This shows that both request queues make sufficient progress. This is because of the fairness algorithms in the sbitmap code.

Given the performance impact of the request queue fairness algorithm, I propose that users who need better fairness than what can be realized with the sbitmap algorithms enable a cgroup policy of their preference.

See also https://github.com/osandov/blktests/compare/master...bvanassche:blktests:master.

Thanks,

Bart.

#3 lore raw
Quoted 10 lines · Bart Van Assche

* Create a first request queue with completion time 1 ms and queue depth 64.
* Create a second request queue with completion time 100 ms and that shares the tag set of the first request queue.
* Submit I/O to both request queues.

If I run that test I see a queue depth of about 60 for the second request queue and a queue depth of about 4 for the first request queue. This shows that both request queues make sufficient progress. This is because of the fairness algorithms in the sbitmap code.

This information needs to go into the commit log.