block: don't merge across cgroup boundaries if iocost or iolatency is active
Subsystems (2)
Tejun Heo
maintainerPATCH 1/1 block: don't merge across cgroup boundaries if iocost or iolatency is active
blk-iocost and iolatency are cgroup aware rq-qos policies but, unlike elevators, they didn't have a way to disable merges across different cgroups. This obviously can lead to accounting and control errors but more importantly to priority inversions - e.g. an IO which belongs to a higher priority cgroup or IO class may end up getting throttled incorrectly because it gets merged to an IO issued from a low priority cgroup.
Fix it by adding blk_cgroup_mergeable() which is called from merge paths to test whether merges are acceptable from cgroup POV. When iocost or iolatency is active, this rejects cross-cgroup and cross-issue_as_root merges.
While at it,
* Add WARN_ON_ONCE() on blkg mismatch in ioc_rqos_merge() so that we can easily notice similar failures in the future.
* Make sure iocost enable/disable transitions only happen when iocost is actually enabled / disabled.
--- block/blk-cgroup.c | 2 + block/blk-iocost.c | 20 ++++++++++++------- block/blk-iolatency.c | 9 +++++--- block/blk-merge.c | 11 ++++++++++ include/linux/blk-cgroup.h | 46 +++++++++++++++++++++++++++++++++++++++++++++ include/linux/blkdev.h | 4 ++- 6 files changed, 81 insertions(+), 11 deletions(-)
block/blk-cgroup.c +2 -0
@@ -1176,6 +1176,8 @@ int blkcg_init_queue(struct request_queubool preloaded;int ret; +atomic_set (&q->cgroup_no_cross_merges,0 ); + new_blkg =blkg_alloc (&blkcg_root, q,GFP_KERNEL );if (!new_blkg)return -ENOMEM ;
block/blk-iocost.c +13 -7
@@ -2703,6 +2703,8 @@ static void ioc_rqos_merge(struct rq_qosif (!ioc->enabled || !iocg || !iocg->level)return ; +WARN_ON_ONCE (rq->bio->bi_blkg != bio->bi_blkg); + abs_cost =calc_vtime_cost (bio, iocg, true);if (!abs_cost)return ;
@@ -3253,13 +3255,17 @@ static ssize_t ioc_qos_write(struct kernspin_lock_irq (&ioc->lock); -if (enable) { -blk_stat_enable_accounting (ioc->rqos.q); -blk_queue_flag_set (QUEUE_FLAG_RQ_ALLOC_TIME , ioc->rqos.q); - ioc->enabled = true; - }else { -blk_queue_flag_clear (QUEUE_FLAG_RQ_ALLOC_TIME , ioc->rqos.q); - ioc->enabled = false; +if (enable != ioc->enabled) { +if (enable) { +blk_stat_enable_accounting (ioc->rqos.q); +blk_queue_flag_set (QUEUE_FLAG_RQ_ALLOC_TIME , ioc->rqos.q); +blk_cgroup_disable_cross_merges (ioc->rqos.q); + ioc->enabled = true; + }else { +blk_queue_flag_clear (QUEUE_FLAG_RQ_ALLOC_TIME , ioc->rqos.q); +blk_cgroup_enable_cross_merges (ioc->rqos.q); + ioc->enabled = false; + } }if (user) {
block/blk-iolatency.c +6 -3
@@ -855,12 +855,15 @@ out:blk_mq_freeze_queue (blkg->q); -if (enable ==1 ) +if (enable ==1 ) {atomic_inc (&blkiolat->enabled); -else if (enable ==-1 ) +blk_cgroup_disable_cross_merges (blkg->q); + }else if (enable ==-1 ) {atomic_dec (&blkiolat->enabled); -else +blk_cgroup_enable_cross_merges (blkg->q); + }else {WARN_ON_ONCE (1 ); + }blk_mq_unfreeze_queue (blkg->q);
block/blk-merge.c +11 -0
@@ -9,6 +9,7 @@#include <linux/blk-integrity.h> #include <linux/scatterlist.h> #include <linux/part_stat.h> +#include <linux/blk-cgroup.h> #include <trace/events/block.h>
@@ -600,6 +601,9 @@ static inline unsigned int blk_rq_get_mastatic inline int ll_new_hw_segment (struct request *req,struct bio *bio,unsigned int nr_phys_segs) { +if (!blk_cgroup_mergeable (req, bio)) + goto no_merge; +if (blk_integrity_merge_bio (req->q, req, bio) == false) goto no_merge;
@@ -696,6 +700,9 @@ static int ll_merge_requests_fn(struct rif (total_phys_segments >blk_rq_get_max_segments (req))return 0 ; +if (!blk_cgroup_mergeable (req, next->bio)) +return 0 ; +if (blk_integrity_merge_rq (q, req, next) == false)return 0 ;
@@ -904,6 +911,10 @@ bool blk_rq_merge_ok(struct request *rq,if (bio_data_dir (bio) !=rq_data_dir (rq))return false; +/* don't merge across cgroup boundaries */ +if (!blk_cgroup_mergeable (rq, bio)) +return false; +/* only merge integrity protected bio into ditto rq */ if (blk_integrity_merge_bio (rq->q, rq, bio) == false)return false;
include/linux/blk-cgroup.h +46 -0
64 lines not on this page · raw message
include/linux/blkdev.h +3 -1
@@ -274,7 +274,9 @@ struct request_queue {struct work_struct timeout_work;atomic_t nr_active_requests_shared_tags; - +#ifdef CONFIG_BLK_CGROUP +atomic_t cgroup_no_cross_merges; +#endif struct blk_mq_tags *sched_shared_tags;struct list_head icq_list;
Jens Axboe
maintainerQuoted 11 lines · Tejun Heo
blk-iocost and iolatency are cgroup aware rq-qos policies but, unlike elevators, they didn't have a way to disable merges across different cgroups. This obviously can lead to accounting and control errors but more importantly to priority inversions - e.g. an IO which belongs to a higher priority cgroup or IO class may end up getting throttled incorrectly because it gets merged to an IO issued from a low priority cgroup.
Fix it by adding
blk_cgroup_mergeable()which is called from merge paths to test whether merges are acceptable from cgroup POV. When iocost or iolatency is active, this rejects cross-cgroup and cross-issue_as_root merges.
Quoted 7 lines
While at it,
* Add
WARN_ON_ONCE()on blkg mismatch inioc_rqos_merge()so that we can easily notice similar failures in the future.* Make sure iocost enable/disable transitions only happen when iocost is actually enabled / disabled.
Is there really no better way to do this than add a lot of expensive checks to the fast path?
Even just inverting the checks so that
if (req->bio->bi_blkg != bio->bi_blkg)
...
is checked first would seem a lot saner.
In any case, since this isn't a new regression, I'd feel a lot better deferring it to 5.18.
Signature · 1 line
Jens Axboe
Tejun Heo
maintainerQuoted 11 lines · Jens Axboe
Quoted 5 lines
* Add
WARN_ON_ONCE()on blkg mismatch inioc_rqos_merge()so that we can easily notice similar failures in the future.* Make sure iocost enable/disable transitions only happen when iocost is actually enabled / disabled.
Is there really no better way to do this than add a lot of expensive checks to the fast path?
Even just inverting the checks so that
Quoted 7 lines
if (
req->bio->bi_blkg!=bio->bi_blkg)...is checked first would seem a lot saner.
In any case, since this isn't a new regression, I'd feel a lot better deferring it to 5.18.
As discussed, the only risk of removing the disable/enable switch and always testing is ruining some merges for cases where blkcg is enabled but no control is applied, which shouldn't be a big problem. I'll redo the patch.
Thanks.
Signature · 1 line
tejun
Tejun Heo
maintainerPATCH v2 1/1 block: don't merge across cgroup boundaries if blkcg is enabled
In reply to Tejun Heoblk-iocost and iolatency are cgroup aware rq-qos policies but they didn't disable merges across different cgroups. This obviously can lead to accounting and control errors but more importantly to priority inversions - e.g. an IO which belongs to a higher priority cgroup or IO class may end up getting throttled incorrectly because it gets merged to an IO issued from a low priority cgroup.
Fix it by adding blk_cgroup_mergeable() which is called from merge paths and rejects cross-cgroup and cross-issue_as_root merges.
v2: Dropped conditional enabling. Always disallow cross-blkcg merges for
simpilcity. While this may spuriously prevent some merges for cases where blkcg is enabled but no control is applied, that is a small cross section.
--- block/blk-merge.c | 11 +++++++++++ include/linux/blk-cgroup.h | 17 +++++++++++++++++ 2 files changed, 28 insertions(+)
block/blk-merge.c +11 -0
@@ -9,6 +9,7 @@#include <linux/blk-integrity.h> #include <linux/scatterlist.h> #include <linux/part_stat.h> +#include <linux/blk-cgroup.h> #include <trace/events/block.h>
@@ -600,6 +601,9 @@ static inline unsigned int blk_rq_get_mastatic inline int ll_new_hw_segment (struct request *req,struct bio *bio,unsigned int nr_phys_segs) { +if (!blk_cgroup_mergeable (req, bio)) + goto no_merge; +if (blk_integrity_merge_bio (req->q, req, bio) == false) goto no_merge;
@@ -696,6 +700,9 @@ static int ll_merge_requests_fn(struct rif (total_phys_segments >blk_rq_get_max_segments (req))return 0 ; +if (!blk_cgroup_mergeable (req, next->bio)) +return 0 ; +if (blk_integrity_merge_rq (q, req, next) == false)return 0 ;
@@ -904,6 +911,10 @@ bool blk_rq_merge_ok(struct request *rq,if (bio_data_dir (bio) !=rq_data_dir (rq))return false; +/* don't merge across cgroup boundaries */ +if (!blk_cgroup_mergeable (rq, bio)) +return false; +/* only merge integrity protected bio into ditto rq */ if (blk_integrity_merge_bio (rq->q, rq, bio) == false)return false;
include/linux/blk-cgroup.h +17 -0
@@ -21,6 +21,7 @@#include <linux/seq_file.h> #include <linux/radix-tree.h> #include <linux/blkdev.h> +#include <linux/blk-mq.h> #include <linux/atomic.h> #include <linux/kthread.h> #include <linux/fs.h>
@@ -604,6 +605,21 @@ static inline void blkcg_clear_delay(stratomic_dec (&blkg->blkcg->css.cgroup->congestion_count); } +/** +* blk_cgroup_mergeable - Determine whether to allow or disallow merges +* @rq: request to merge into +* @bio: bio to merge +* +* @bio and @rq should belong to the same cgroup and their issue_as_root should +* match. The latter is necessary as we don't want to throttle e.g. a metadata +* update because it happens to be next to a regular IO. +*/ +static inline bool blk_cgroup_mergeable (struct request *rq,struct bio *bio) +{ +return rq->bio->bi_blkg == bio->bi_blkg && +bio_issue_as_root_blkg (rq->bio) ==bio_issue_as_root_blkg (bio); +} +void blk_cgroup_bio_start (struct bio *bio);void blkcg_add_delay (struct blkcg_gq *blkg,u64 now,u64 delta);void blkcg_schedule_throttle (struct request_queue *q,bool use_memdelay);
@@ -659,6 +675,7 @@ static inline void blkg_put(struct blkcgstatic inline bool blkcg_punt_bio_submit (struct bio *bio) {return false; }static inline void blkcg_bio_issue_init (struct bio *bio) { }static inline void blk_cgroup_bio_start (struct bio *bio) { } +static inline bool blk_cgroup_mergeable (struct request *rq,struct bio *bio) {return true; }#define blk_queue_for_each_rl (rl, q) \ for ((rl) = &(q)->root_rl; (rl); (rl) = NULL)
Tejun Heo
maintainerPATCH v3 1/1 block: don't merge across cgroup boundaries if blkcg is enabled
blk-iocost and iolatency are cgroup aware rq-qos policies but they didn't disable merges across different cgroups. This obviously can lead to accounting and control errors but more importantly to priority inversions - e.g. an IO which belongs to a higher priority cgroup or IO class may end up getting throttled incorrectly because it gets merged to an IO issued from a low priority cgroup.
Fix it by adding blk_cgroup_mergeable() which is called from merge paths and rejects cross-cgroup and cross-issue_as_root merges.
---
v2 was tagged for-5.18/block but was still based on block-for-5.17. Refresh on top of for-5.18/block. My apologies about all the messiness.
v3: Actually rebased on top of block/for-5.18.
v2: Dropped conditional enabling. Always disallow cross-blkcg merges for
simpilcity. While this may spuriously prevent some merges for cases where blkcg is enabled but no control is applied, that is a small cross section.
block/blk-cgroup.h | 17 +++++++++++++++++ block/blk-merge.c | 11 +++++++++++ 2 files changed, 28 insertions(+)
block/blk-cgroup.h +17 -0
@@ -15,6 +15,7 @@*/ #include <linux/blk-cgroup.h> +#include <linux/blk-mq.h> /* percpu_counter batch for blkg_[rw]stats, per-cpu drift doesn't matter */ #define BLKG_STAT_CPU_BATCH (INT_MAX / 2)
@@ -428,6 +429,21 @@ static inline void blkcg_clear_delay(struct blkcg_gq *blkg) atomic_dec(&blkg->blkcg->css.cgroup->congestion_count); } +/** +* blk_cgroup_mergeable - Determine whether to allow or disallow merges +* @rq: request to merge into +* @bio: bio to merge +* +* @bio and @rq should belong to the same cgroup and their issue_as_root should +* match. The latter is necessary as we don't want to throttle e.g. a metadata +* update because it happens to be next to a regular IO. +*/ +static inline bool blk_cgroup_mergeable (struct request *rq,struct bio *bio) +{ +return rq->bio->bi_blkg == bio->bi_blkg && +bio_issue_as_root_blkg (rq->bio) ==bio_issue_as_root_blkg (bio); +} +void blk_cgroup_bio_start (struct bio *bio);void blkcg_add_delay (struct blkcg_gq *blkg,u64 now,u64 delta);#else /* CONFIG_BLK_CGROUP */
@@ -467,6 +483,7 @@ static inline void blkg_put(struct blkcg_gq *blkg) { }static inline bool blkcg_punt_bio_submit (struct bio *bio) {return false; }static inline void blkcg_bio_issue_init (struct bio *bio) { }static inline void blk_cgroup_bio_start (struct bio *bio) { } +static inline bool blk_cgroup_mergeable (struct request *rq,struct bio *bio) {return true; }#define blk_queue_for_each_rl (rl, q) \ for ((rl) = &(q)->root_rl; (rl); (rl) = NULL)
block/blk-merge.c +11 -0
@@ -9,6 +9,7 @@#include <linux/blk-integrity.h> #include <linux/scatterlist.h> #include <linux/part_stat.h> +#include <linux/blk-cgroup.h> #include <trace/events/block.h>
@@ -598,6 +599,9 @@ static inline unsigned int blk_rq_get_max_sectors(struct request *rq,static inline int ll_new_hw_segment (struct request *req,struct bio *bio,unsigned int nr_phys_segs) { +if (!blk_cgroup_mergeable (req, bio)) + goto no_merge; +if (blk_integrity_merge_bio (req->q, req, bio) == false) goto no_merge;
@@ -694,6 +698,9 @@ static int ll_merge_requests_fn(struct request_queue *q, struct request *req,if (total_phys_segments >blk_rq_get_max_segments (req))return 0 ; +if (!blk_cgroup_mergeable (req, next->bio)) +return 0 ; +if (blk_integrity_merge_rq (q, req, next) == false)return 0 ;
@@ -902,6 +909,10 @@ bool blk_rq_merge_ok(struct request *rq, struct bio *bio)if (bio_data_dir (bio) !=rq_data_dir (rq))return false; +/* don't merge across cgroup boundaries */ +if (!blk_cgroup_mergeable (rq, bio)) +return false; +/* only merge integrity protected bio into ditto rq */ if (blk_integrity_merge_bio (rq->q, rq, bio) == false)return false;
Signature · 1 line
2.35.1
Re: [PATCH v3 for-5.18/block] block: don't merge across cgroup boundaries if blkcg is enabled
Quoted 8 lines · Tejun Heo
blk-iocost and iolatency are cgroup aware rq-qos policies but they didn't disable merges across different cgroups. This obviously can lead to accounting and control errors but more importantly to priority inversions - e.g. an IO which belongs to a higher priority cgroup or IO class may end up getting throttled incorrectly because it gets merged to an IO issued from a low priority cgroup.
[...]
Applied, thanks!
[1/1] block: don't merge across cgroup boundaries if blkcg is enabled
commit: 6b2b04590b51aa4cf395fcd185ce439cab5961dc
Best regards,
Signature · 1 line
Jens Axboe