Skip to contentSkip to conversation

block: don't merge across cgroup boundaries if iocost or iolatency is active

PATCH Superseded Tejun Heo 2 participants 6 messages Mar 14, 2022 – Mar 15, 2022

Subsystems (2)
lore
Share & tools
lore

Tejun Heo

author maintainer
#0 lore raw

PATCH 1/1 block: don't merge across cgroup boundaries if iocost or iolatency is active

blk-iocost and iolatency are cgroup aware rq-qos policies but, unlike elevators, they didn't have a way to disable merges across different cgroups. This obviously can lead to accounting and control errors but more importantly to priority inversions - e.g. an IO which belongs to a higher priority cgroup or IO class may end up getting throttled incorrectly because it gets merged to an IO issued from a low priority cgroup.

Fix it by adding blk_cgroup_mergeable() which is called from merge paths to test whether merges are acceptable from cgroup POV. When iocost or iolatency is active, this rejects cross-cgroup and cross-issue_as_root merges.

While at it,

* Add WARN_ON_ONCE() on blkg mismatch in ioc_rqos_merge() so that we can easily notice similar failures in the future.

* Make sure iocost enable/disable transitions only happen when iocost is actually enabled / disabled.

Fixes d70675121546 ("block: introduce blk-iolatency io controller")
Cc [email protected] # v4.19+
Cc Josef Bacik <[email protected]>
---
 block/blk-cgroup.c         |    2 +
 block/blk-iocost.c         |   20 ++++++++++++-------
 block/blk-iolatency.c      |    9 +++++---
 block/blk-merge.c          |   11 ++++++++++
 include/linux/blk-cgroup.h |   46 +++++++++++++++++++++++++++++++++++++++++++++
 include/linux/blkdev.h     |    4 ++-
 6 files changed, 81 insertions(+), 11 deletions(-)
block/blk-cgroup.c +2 -0
@@ -1176,6 +1176,8 @@ int blkcg_init_queue(struct request_queu
 	bool preloaded;
 	int ret;
 
+	atomic_set(&q->cgroup_no_cross_merges, 0);
+
 	new_blkg = blkg_alloc(&blkcg_root, q, GFP_KERNEL);
 	if (!new_blkg)
 		return -ENOMEM;
block/blk-iocost.c +13 -7
@@ -2703,6 +2703,8 @@ static void ioc_rqos_merge(struct rq_qos
 	if (!ioc->enabled || !iocg || !iocg->level)
 		return;
 
+	WARN_ON_ONCE(rq->bio->bi_blkg != bio->bi_blkg);
+
 	abs_cost = calc_vtime_cost(bio, iocg, true);
 	if (!abs_cost)
 		return;
@@ -3253,13 +3255,17 @@ static ssize_t ioc_qos_write(struct kern
 
 	spin_lock_irq(&ioc->lock);
 
-	if (enable) {
-		blk_stat_enable_accounting(ioc->rqos.q);
-		blk_queue_flag_set(QUEUE_FLAG_RQ_ALLOC_TIME, ioc->rqos.q);
-		ioc->enabled = true;
-	} else {
-		blk_queue_flag_clear(QUEUE_FLAG_RQ_ALLOC_TIME, ioc->rqos.q);
-		ioc->enabled = false;
+	if (enable != ioc->enabled) {
+		if (enable) {
+			blk_stat_enable_accounting(ioc->rqos.q);
+			blk_queue_flag_set(QUEUE_FLAG_RQ_ALLOC_TIME, ioc->rqos.q);
+			blk_cgroup_disable_cross_merges(ioc->rqos.q);
+			ioc->enabled = true;
+		} else {
+			blk_queue_flag_clear(QUEUE_FLAG_RQ_ALLOC_TIME, ioc->rqos.q);
+			blk_cgroup_enable_cross_merges(ioc->rqos.q);
+			ioc->enabled = false;
+		}
 	}
 
 	if (user) {
block/blk-iolatency.c +6 -3
@@ -855,12 +855,15 @@ out:
 
 		blk_mq_freeze_queue(blkg->q);
 
-		if (enable == 1)
+		if (enable == 1) {
 			atomic_inc(&blkiolat->enabled);
-		else if (enable == -1)
+			blk_cgroup_disable_cross_merges(blkg->q);
+		} else if (enable == -1) {
 			atomic_dec(&blkiolat->enabled);
-		else
+			blk_cgroup_enable_cross_merges(blkg->q);
+		} else {
 			WARN_ON_ONCE(1);
+		}
 
 		blk_mq_unfreeze_queue(blkg->q);
 
block/blk-merge.c +11 -0
@@ -9,6 +9,7 @@
 #include <linux/blk-integrity.h>
 #include <linux/scatterlist.h>
 #include <linux/part_stat.h>
+#include <linux/blk-cgroup.h>
 
 #include <trace/events/block.h>
 
@@ -600,6 +601,9 @@ static inline unsigned int blk_rq_get_ma
 static inline int ll_new_hw_segment(struct request *req, struct bio *bio,
 		unsigned int nr_phys_segs)
 {
+	if (!blk_cgroup_mergeable(req, bio))
+		goto no_merge;
+
 	if (blk_integrity_merge_bio(req->q, req, bio) == false)
 		goto no_merge;
 
@@ -696,6 +700,9 @@ static int ll_merge_requests_fn(struct r
 	if (total_phys_segments > blk_rq_get_max_segments(req))
 		return 0;
 
+	if (!blk_cgroup_mergeable(req, next->bio))
+		return 0;
+
 	if (blk_integrity_merge_rq(q, req, next) == false)
 		return 0;
 
@@ -904,6 +911,10 @@ bool blk_rq_merge_ok(struct request *rq,
 	if (bio_data_dir(bio) != rq_data_dir(rq))
 		return false;
 
+	/* don't merge across cgroup boundaries */
+	if (!blk_cgroup_mergeable(rq, bio))
+		return false;
+
 	/* only merge integrity protected bio into ditto rq */
 	if (blk_integrity_merge_bio(rq->q, rq, bio) == false)
 		return false;
include/linux/blk-cgroup.h +46 -0

64 lines not on this page · raw message

include/linux/blkdev.h +3 -1
@@ -274,7 +274,9 @@ struct request_queue {
 	struct work_struct	timeout_work;
 
 	atomic_t		nr_active_requests_shared_tags;
-
+#ifdef CONFIG_BLK_CGROUP
+	atomic_t		cgroup_no_cross_merges;
+#endif
 	struct blk_mq_tags	*sched_shared_tags;
 
 	struct list_head	icq_list;
maintainer
#1 lore raw
Quoted 11 lines · Tejun Heo

blk-iocost and iolatency are cgroup aware rq-qos policies but, unlike elevators, they didn't have a way to disable merges across different cgroups. This obviously can lead to accounting and control errors but more importantly to priority inversions - e.g. an IO which belongs to a higher priority cgroup or IO class may end up getting throttled incorrectly because it gets merged to an IO issued from a low priority cgroup.

Fix it by adding blk_cgroup_mergeable() which is called from merge paths to test whether merges are acceptable from cgroup POV. When iocost or iolatency is active, this rejects cross-cgroup and cross-issue_as_root merges.

Quoted 7 lines

While at it,

* Add WARN_ON_ONCE() on blkg mismatch in ioc_rqos_merge() so that we can easily notice similar failures in the future.

* Make sure iocost enable/disable transitions only happen when iocost is actually enabled / disabled.

Is there really no better way to do this than add a lot of expensive checks to the fast path?

Even just inverting the checks so that

if (req->bio->bi_blkg != bio->bi_blkg)

	...

is checked first would seem a lot saner.

In any case, since this isn't a new regression, I'd feel a lot better deferring it to 5.18.

Signature · 1 line
Jens Axboe
author maintainer
#2 lore raw
Quoted 11 lines · Jens Axboe
Quoted 5 lines

* Add WARN_ON_ONCE() on blkg mismatch in ioc_rqos_merge() so that we can easily notice similar failures in the future.

* Make sure iocost enable/disable transitions only happen when iocost is actually enabled / disabled.

Is there really no better way to do this than add a lot of expensive checks to the fast path?

Even just inverting the checks so that

Quoted 7 lines

if (req->bio->bi_blkg != bio->bi_blkg)

	...

is checked first would seem a lot saner.

In any case, since this isn't a new regression, I'd feel a lot better deferring it to 5.18.

As discussed, the only risk of removing the disable/enable switch and always testing is ruining some merges for cases where blkcg is enabled but no control is applied, which shouldn't be a big problem. I'll redo the patch.

Thanks.

Signature · 1 line
tejun
author maintainer
#3 lore raw

PATCH v2 1/1 block: don't merge across cgroup boundaries if blkcg is enabled

In reply to Tejun Heo

blk-iocost and iolatency are cgroup aware rq-qos policies but they didn't disable merges across different cgroups. This obviously can lead to accounting and control errors but more importantly to priority inversions - e.g. an IO which belongs to a higher priority cgroup or IO class may end up getting throttled incorrectly because it gets merged to an IO issued from a low priority cgroup.

Fix it by adding blk_cgroup_mergeable() which is called from merge paths and rejects cross-cgroup and cross-issue_as_root merges.

v2: Dropped conditional enabling. Always disallow cross-blkcg merges for
simpilcity. While this may spuriously prevent some merges for cases where blkcg is enabled but no control is applied, that is a small cross section.

Fixes d70675121546 ("block: introduce blk-iolatency io controller")
Cc [email protected] # v4.19+
Cc Josef Bacik <[email protected]>
---
 block/blk-merge.c          |   11 +++++++++++
 include/linux/blk-cgroup.h |   17 +++++++++++++++++
 2 files changed, 28 insertions(+)
block/blk-merge.c +11 -0
@@ -9,6 +9,7 @@
 #include <linux/blk-integrity.h>
 #include <linux/scatterlist.h>
 #include <linux/part_stat.h>
+#include <linux/blk-cgroup.h>
 
 #include <trace/events/block.h>
 
@@ -600,6 +601,9 @@ static inline unsigned int blk_rq_get_ma
 static inline int ll_new_hw_segment(struct request *req, struct bio *bio,
 		unsigned int nr_phys_segs)
 {
+	if (!blk_cgroup_mergeable(req, bio))
+		goto no_merge;
+
 	if (blk_integrity_merge_bio(req->q, req, bio) == false)
 		goto no_merge;
 
@@ -696,6 +700,9 @@ static int ll_merge_requests_fn(struct r
 	if (total_phys_segments > blk_rq_get_max_segments(req))
 		return 0;
 
+	if (!blk_cgroup_mergeable(req, next->bio))
+		return 0;
+
 	if (blk_integrity_merge_rq(q, req, next) == false)
 		return 0;
 
@@ -904,6 +911,10 @@ bool blk_rq_merge_ok(struct request *rq,
 	if (bio_data_dir(bio) != rq_data_dir(rq))
 		return false;
 
+	/* don't merge across cgroup boundaries */
+	if (!blk_cgroup_mergeable(rq, bio))
+		return false;
+
 	/* only merge integrity protected bio into ditto rq */
 	if (blk_integrity_merge_bio(rq->q, rq, bio) == false)
 		return false;
include/linux/blk-cgroup.h +17 -0
@@ -21,6 +21,7 @@
 #include <linux/seq_file.h>
 #include <linux/radix-tree.h>
 #include <linux/blkdev.h>
+#include <linux/blk-mq.h>
 #include <linux/atomic.h>
 #include <linux/kthread.h>
 #include <linux/fs.h>
@@ -604,6 +605,21 @@ static inline void blkcg_clear_delay(str
 		atomic_dec(&blkg->blkcg->css.cgroup->congestion_count);
 }
 
+/**
+ * blk_cgroup_mergeable - Determine whether to allow or disallow merges
+ * @rq: request to merge into
+ * @bio: bio to merge
+ *
+ * @bio and @rq should belong to the same cgroup and their issue_as_root should
+ * match. The latter is necessary as we don't want to throttle e.g. a metadata
+ * update because it happens to be next to a regular IO.
+ */
+static inline bool blk_cgroup_mergeable(struct request *rq, struct bio *bio)
+{
+	return rq->bio->bi_blkg == bio->bi_blkg &&
+		bio_issue_as_root_blkg(rq->bio) == bio_issue_as_root_blkg(bio);
+}
+
 void blk_cgroup_bio_start(struct bio *bio);
 void blkcg_add_delay(struct blkcg_gq *blkg, u64 now, u64 delta);
 void blkcg_schedule_throttle(struct request_queue *q, bool use_memdelay);
@@ -659,6 +675,7 @@ static inline void blkg_put(struct blkcg
 static inline bool blkcg_punt_bio_submit(struct bio *bio) { return false; }
 static inline void blkcg_bio_issue_init(struct bio *bio) { }
 static inline void blk_cgroup_bio_start(struct bio *bio) { }
+static inline bool blk_cgroup_mergeable(struct request *rq, struct bio *bio) { return true; }
 
 #define blk_queue_for_each_rl(rl, q)	\
 	for ((rl) = &(q)->root_rl; (rl); (rl) = NULL)
author maintainer
#4 lore raw

PATCH v3 1/1 block: don't merge across cgroup boundaries if blkcg is enabled

blk-iocost and iolatency are cgroup aware rq-qos policies but they didn't disable merges across different cgroups. This obviously can lead to accounting and control errors but more importantly to priority inversions - e.g. an IO which belongs to a higher priority cgroup or IO class may end up getting throttled incorrectly because it gets merged to an IO issued from a low priority cgroup.

Fix it by adding blk_cgroup_mergeable() which is called from merge paths and rejects cross-cgroup and cross-issue_as_root merges.

Fixes d70675121546 ("block: introduce blk-iolatency io controller")
Cc [email protected] # v4.19+
Cc Josef Bacik <[email protected]>

---
v2 was tagged for-5.18/block but was still based on block-for-5.17. Refresh on top of for-5.18/block. My apologies about all the messiness.

v3: Actually rebased on top of block/for-5.18.

v2: Dropped conditional enabling. Always disallow cross-blkcg merges for
simpilcity. While this may spuriously prevent some merges for cases where blkcg is enabled but no control is applied, that is a small cross section.

 block/blk-cgroup.h | 17 +++++++++++++++++
 block/blk-merge.c  | 11 +++++++++++
 2 files changed, 28 insertions(+)
block/blk-cgroup.h +17 -0
index 3e91803c4a555..47e1e38390c96 100644
@@ -15,6 +15,7 @@
  */
 
 #include <linux/blk-cgroup.h>
+#include <linux/blk-mq.h>
 
 /* percpu_counter batch for blkg_[rw]stats, per-cpu drift doesn't matter */
 #define BLKG_STAT_CPU_BATCH	(INT_MAX / 2)
@@ -428,6 +429,21 @@ static inline void blkcg_clear_delay(struct blkcg_gq *blkg)
 		atomic_dec(&blkg->blkcg->css.cgroup->congestion_count);
 }
 
+/**
+ * blk_cgroup_mergeable - Determine whether to allow or disallow merges
+ * @rq: request to merge into
+ * @bio: bio to merge
+ *
+ * @bio and @rq should belong to the same cgroup and their issue_as_root should
+ * match. The latter is necessary as we don't want to throttle e.g. a metadata
+ * update because it happens to be next to a regular IO.
+ */
+static inline bool blk_cgroup_mergeable(struct request *rq, struct bio *bio)
+{
+	return rq->bio->bi_blkg == bio->bi_blkg &&
+		bio_issue_as_root_blkg(rq->bio) == bio_issue_as_root_blkg(bio);
+}
+
 void blk_cgroup_bio_start(struct bio *bio);
 void blkcg_add_delay(struct blkcg_gq *blkg, u64 now, u64 delta);
 #else	/* CONFIG_BLK_CGROUP */
@@ -467,6 +483,7 @@ static inline void blkg_put(struct blkcg_gq *blkg) { }
 static inline bool blkcg_punt_bio_submit(struct bio *bio) { return false; }
 static inline void blkcg_bio_issue_init(struct bio *bio) { }
 static inline void blk_cgroup_bio_start(struct bio *bio) { }
+static inline bool blk_cgroup_mergeable(struct request *rq, struct bio *bio) { return true; }
 
 #define blk_queue_for_each_rl(rl, q)	\
 	for ((rl) = &(q)->root_rl; (rl); (rl) = NULL)
block/blk-merge.c +11 -0
index 8d8177f71ebdb..ea6968313b4a8 100644
@@ -9,6 +9,7 @@
 #include <linux/blk-integrity.h>
 #include <linux/scatterlist.h>
 #include <linux/part_stat.h>
+#include <linux/blk-cgroup.h>
 
 #include <trace/events/block.h>
 
@@ -598,6 +599,9 @@ static inline unsigned int blk_rq_get_max_sectors(struct request *rq,
 static inline int ll_new_hw_segment(struct request *req, struct bio *bio,
 		unsigned int nr_phys_segs)
 {
+	if (!blk_cgroup_mergeable(req, bio))
+		goto no_merge;
+
 	if (blk_integrity_merge_bio(req->q, req, bio) == false)
 		goto no_merge;
 
@@ -694,6 +698,9 @@ static int ll_merge_requests_fn(struct request_queue *q, struct request *req,
 	if (total_phys_segments > blk_rq_get_max_segments(req))
 		return 0;
 
+	if (!blk_cgroup_mergeable(req, next->bio))
+		return 0;
+
 	if (blk_integrity_merge_rq(q, req, next) == false)
 		return 0;
 
@@ -902,6 +909,10 @@ bool blk_rq_merge_ok(struct request *rq, struct bio *bio)
 	if (bio_data_dir(bio) != rq_data_dir(rq))
 		return false;
 
+	/* don't merge across cgroup boundaries */
+	if (!blk_cgroup_mergeable(rq, bio))
+		return false;
+
 	/* only merge integrity protected bio into ditto rq */
 	if (blk_integrity_merge_bio(rq->q, rq, bio) == false)
 		return false;
Signature · 1 line
2.35.1
maintainer Applied
#5 lore raw

Re: [PATCH v3 for-5.18/block] block: don't merge across cgroup boundaries if blkcg is enabled

Quoted 8 lines · Tejun Heo

blk-iocost and iolatency are cgroup aware rq-qos policies but they didn't disable merges across different cgroups. This obviously can lead to accounting and control errors but more importantly to priority inversions - e.g. an IO which belongs to a higher priority cgroup or IO class may end up getting throttled incorrectly because it gets merged to an IO issued from a low priority cgroup.

[...]

Applied, thanks!

[1/1] block: don't merge across cgroup boundaries if blkcg is enabled

      commit: 6b2b04590b51aa4cf395fcd185ce439cab5961dc

Best regards,

Signature · 1 line
Jens Axboe