blk-cgroup: use cgroup rstat for IO stats
Subsystems (4)
Tejun Heo
maintainerPATCH 1/1 blk-cgroup: use cgroup rstat for IO stats
Hello,
blk-cgroup IO stats currently use blkg_rwstat which unforutnately requires walking all descendants recursively on read. On systems with a large number of cgroups (dead or alive), this can make each stat read a substantially expensive operation.
This patch updates blk-cgroup to use cgroup rstat which makes stat reading O(# descendants which have been active since last reading) instead of O(# descendants).
0001-bfq-iosched-stop-using-blkg-stat_bytes-and-stat_ios.patch 0002-blk-throtl-stop-using-blkg-stat_bytes-and-stat_ios.patch 0003-blk-cgroup-remove-now-unused-blkg_print_stat_-bytes-.patch 0004-blk-cgroup-reimplement-basic-IO-stats-using-cgroup-r.patch 0005-blk-cgroup-separate-out-blkg_rwstat-under-CONFIG_BLK.patch
0001-0002 make bfq-iosched and blk-throtl use their own blkg_rwstat to track basic IO stats on cgroup1 instead of sharing blk-cgroup core's. 0003 is a follow-up cleanup.
0004 switches blk-cgroup to cgroup rstat.
0005 moves blkg_rwstat to its own files and gate it under a config option as it's now only used by blk-throtl and bfq-iosched.
The patchset is on top of
for-next/block 257855f86aa2 ("Merge branch 'for-5.4/block' into for-next")
+ [1] blkcg: make blkcg_print_stat() print stats only for online blkgs
and also available in the following git branch.
git://git.kernel.org/pub/scm/linux/kernel/git/tj/cgroup.git review-blkcg-rstat
Thanks.
block/Kconfig | 4 block/Kconfig.iosched | 1 block/Makefile | 1 block/bfq-cgroup.c | 37 +++-- block/bfq-iosched.c | 4 block/bfq-iosched.h | 6 block/blk-cgroup-rwstat.c | 129 +++++++++++++++++++ block/blk-cgroup-rwstat.h | 149 ++++++++++++++++++++++ block/blk-cgroup.c | 304 ++++++++++++++------------------------------- block/blk-throttle.c | 71 +++++++++- include/linux/blk-cgroup.h | 198 +++++------------------------ 11 files changed, 517 insertions(+), 387 deletions(-)
--
tejun
Tejun Heo
maintainerPATCH 1/5 bfq-iosched: stop using blkg->stat_bytes and ->stat_ios
When used on cgroup1, bfq uses the blkg->stat_bytes and ->stat_ios from blk-cgroup core to populate six stat knobs. blk-cgroup core is moving away from blkg_rwstat to improve scalability and won't be able to support this usage.
It isn't like the sharing gains all that much. Let's break it out to dedicated rwstat counters which are updated when on cgroup1.
--- block/bfq-cgroup.c | 37 ++++++++++++++++++++++++++----------- block/bfq-iosched.c | 4 ++++ block/bfq-iosched.h | 4 ++++ 3 files changed, 34 insertions(+), 11 deletions(-)
block/bfq-cgroup.c +26 -11
@@ -347,6 +347,14 @@ void bfqg_and_blkg_put(struct bfq_group *bfqg)bfqg_put (bfqg); } +void bfqg_stats_update_legacy_io (struct request_queue *q,struct request *rq) +{ +struct bfq_group *bfqg =blkg_to_bfqg (rq->bio->bi_blkg); + +blkg_rwstat_add (&bfqg->stats.bytes, rq->cmd_flags,blk_rq_bytes (rq)); +blkg_rwstat_add (&bfqg->stats.ios, rq->cmd_flags,1 ); +} +/* @stats = 0 */ static void bfqg_stats_reset (struct bfqg_stats *stats) {
@@ -431,6 +439,8 @@ void bfq_init_entity(struct bfq_entity *entity, struct bfq_group *bfqg)static void bfqg_stats_exit (struct bfqg_stats *stats) { +blkg_rwstat_exit (&stats->bytes); +blkg_rwstat_exit (&stats->ios);#ifdef CONFIG_BFQ_CGROUP_DEBUG blkg_rwstat_exit (&stats->merged);blkg_rwstat_exit (&stats->service_time);
@@ -448,6 +458,10 @@ static void bfqg_stats_exit(struct bfqg_stats *stats)static int bfqg_stats_init (struct bfqg_stats *stats,gfp_t gfp) { +if (blkg_rwstat_init (&stats->bytes, gfp) || +blkg_rwstat_init (&stats->ios, gfp)) +return -ENOMEM ; +#ifdef CONFIG_BFQ_CGROUP_DEBUG if (blkg_rwstat_init(&stats->merged, gfp) || blkg_rwstat_init(&stats->service_time, gfp) ||
@@ -1125,7 +1139,8 @@ static int bfqg_print_rwstat_recursive(struct seq_file *sf, void *v)static u64 bfqg_prfill_sectors (struct seq_file *sf,struct blkg_policy_data *pd,int off) { -u64 sum =blkg_rwstat_total (&pd->blkg->stat_bytes); +struct bfq_group *bfqg =blkg_to_bfqg (pd->blkg); +u64 sum =blkg_rwstat_total (&bfqg->stats.bytes);return __blkg_prfill_u64 (sf, pd, sum >>9 ); }
@@ -1142,8 +1157,8 @@ static u64 bfqg_prfill_sectors_recursive(struct seq_file *sf, {struct blkg_rwstat_sample tmp; -blkg_rwstat_recursive_sum (pd->blkg,NULL , - offsetof(struct blkcg_gq , stat_bytes), &tmp); +blkg_rwstat_recursive_sum (pd->blkg, &blkcg_policy_bfq, + offsetof(struct bfq_group, stats.bytes), &tmp);return __blkg_prfill_u64 (sf, pd, (tmp.cnt[BLKG_RWSTAT_READ ] + tmp.cnt[BLKG_RWSTAT_WRITE ]) >>9 );
@@ -1226,13 +1241,13 @@ struct cftype bfq_blkcg_legacy_files[] = {/* statistics, covers only the tasks in the bfqg */ { .name = "bfq.io_service_bytes", - .private = (unsigned long)&blkcg_policy_bfq, - .seq_show = blkg_print_stat_bytes, + .private = offsetof(struct bfq_group, stats.bytes), + .seq_show = bfqg_print_rwstat, }, { .name = "bfq.io_serviced", - .private = (unsigned long)&blkcg_policy_bfq, - .seq_show = blkg_print_stat_ios, + .private = offsetof(struct bfq_group, stats.ios), + .seq_show = bfqg_print_rwstat, }, #ifdef CONFIG_BFQ_CGROUP_DEBUG {
@@ -1269,13 +1284,13 @@ struct cftype bfq_blkcg_legacy_files[] = {/* the same statistics which cover the bfqg and its descendants */ { .name = "bfq.io_service_bytes_recursive", - .private = (unsigned long)&blkcg_policy_bfq, - .seq_show = blkg_print_stat_bytes_recursive, + .private = offsetof(struct bfq_group, stats.bytes), + .seq_show = bfqg_print_rwstat_recursive, }, { .name = "bfq.io_serviced_recursive", - .private = (unsigned long)&blkcg_policy_bfq, - .seq_show = blkg_print_stat_ios_recursive, + .private = offsetof(struct bfq_group, stats.ios), + .seq_show = bfqg_print_rwstat_recursive, }, #ifdef CONFIG_BFQ_CGROUP_DEBUG {
block/bfq-iosched.c +4 -0
@@ -5450,6 +5450,10 @@ static void bfq_insert_request(struct blk_mq_hw_ctx *hctx, struct request *rq,bool idle_timer_disabled = false;unsigned int cmd_flags; +#ifdef CONFIG_BFQ_GROUP_IOSCHED +if (!cgroup_subsys_on_dfl (io_cgrp_subsys) && rq->bio) +bfqg_stats_update_legacy_io (q, rq); +#endif spin_lock_irq (&bfqd->lock);if (blk_mq_sched_try_insert_merge (q, rq)) {spin_unlock_irq (&bfqd->lock);
block/bfq-iosched.h +4 -0
@@ -809,6 +809,9 @@ struct bfq_stat { };struct bfqg_stats { +/* basic stats */ +struct blkg_rwstat bytes; +struct blkg_rwstat ios;#ifdef CONFIG_BFQ_CGROUP_DEBUG /* number of ios merged */ struct blkg_rwstat merged;
@@ -956,6 +959,7 @@ void bfq_put_async_queues(struct bfq_data *bfqd, struct bfq_group *bfqg);/* ---------------- cgroups-support interface ---------------- */ +void bfqg_stats_update_legacy_io (struct request_queue *q,struct request *rq);void bfqg_stats_update_io_add (struct bfq_group *bfqg,struct bfq_queue *bfqq,unsigned int op);void bfqg_stats_update_io_remove (struct bfq_group *bfqg,unsigned int op);
Signature · 1 line
2.17.1
Tejun Heo
maintainerPATCH v2 1/5 bfq-iosched: stop using blkg->stat_bytes and ->stat_ios
From 4f711dd784f8666ec41fa5319f16838235557a30 Mon Sep 17 00:00:00 2001
From: Tejun Heo <[email protected]>
Date: Thu, 7 Nov 2019 11:01:19 -0800
When used on cgroup1, bfq uses the blkg->stat_bytes and ->stat_ios from blk-cgroup core to populate six stat knobs. blk-cgroup core is moving away from blkg_rwstat to improve scalability and won't be able to support this usage.
It isn't like the sharing gains all that much. Let's break it out to dedicated rwstat counters which are updated when on cgroup1. This makes use of bfqg_*rwstat*() helpers outside of CONFIG_BFQ_CGROUP_DEBUG. Move them out.
v2: Compile fix when !CONFIG_BFQ_CGROUP_DEBUG.
--- block/bfq-cgroup.c | 39 +++++++++++++++++++++++++++------------ block/bfq-iosched.c | 4 ++++ block/bfq-iosched.h | 4 ++++ 3 files changed, 35 insertions(+), 12 deletions(-)
block/bfq-cgroup.c +27 -12
@@ -347,6 +347,14 @@ void bfqg_and_blkg_put(struct bfq_group *bfqg)bfqg_put (bfqg); } +void bfqg_stats_update_legacy_io (struct request_queue *q,struct request *rq) +{ +struct bfq_group *bfqg =blkg_to_bfqg (rq->bio->bi_blkg); + +blkg_rwstat_add (&bfqg->stats.bytes, rq->cmd_flags,blk_rq_bytes (rq)); +blkg_rwstat_add (&bfqg->stats.ios, rq->cmd_flags,1 ); +} +/* @stats = 0 */ static void bfqg_stats_reset (struct bfqg_stats *stats) {
@@ -431,6 +439,8 @@ void bfq_init_entity(struct bfq_entity *entity, struct bfq_group *bfqg)static void bfqg_stats_exit (struct bfqg_stats *stats) { +blkg_rwstat_exit (&stats->bytes); +blkg_rwstat_exit (&stats->ios);#ifdef CONFIG_BFQ_CGROUP_DEBUG blkg_rwstat_exit (&stats->merged);blkg_rwstat_exit (&stats->service_time);
@@ -448,6 +458,10 @@ static void bfqg_stats_exit(struct bfqg_stats *stats)static int bfqg_stats_init (struct bfqg_stats *stats,gfp_t gfp) { +if (blkg_rwstat_init (&stats->bytes, gfp) || +blkg_rwstat_init (&stats->ios, gfp)) +return -ENOMEM ; +#ifdef CONFIG_BFQ_CGROUP_DEBUG if (blkg_rwstat_init(&stats->merged, gfp) || blkg_rwstat_init(&stats->service_time, gfp) ||
@@ -1057,7 +1071,6 @@ static ssize_t bfq_io_set_weight(struct kernfs_open_file *of,return bfq_io_set_device_weight (of, buf, nbytes, off); } -#ifdef CONFIG_BFQ_CGROUP_DEBUG static int bfqg_print_rwstat (struct seq_file *sf,void *v) { blkcg_print_blkgs(sf, css_to_blkcg(seq_css(sf)), blkg_prfill_rwstat,
@@ -1082,6 +1095,7 @@ static int bfqg_print_rwstat_recursive(struct seq_file *sf, void *v)return 0 ; } +#ifdef CONFIG_BFQ_CGROUP_DEBUG static int bfqg_print_stat (struct seq_file *sf,void *v) { blkcg_print_blkgs(sf, css_to_blkcg(seq_css(sf)), blkg_prfill_stat,
@@ -1125,7 +1139,8 @@ static int bfqg_print_stat_recursive(struct seq_file *sf, void *v)static u64 bfqg_prfill_sectors (struct seq_file *sf,struct blkg_policy_data *pd,int off) { -u64 sum =blkg_rwstat_total (&pd->blkg->stat_bytes); +struct bfq_group *bfqg =blkg_to_bfqg (pd->blkg); +u64 sum =blkg_rwstat_total (&bfqg->stats.bytes);return __blkg_prfill_u64 (sf, pd, sum >>9 ); }
@@ -1142,8 +1157,8 @@ static u64 bfqg_prfill_sectors_recursive(struct seq_file *sf, {struct blkg_rwstat_sample tmp; -blkg_rwstat_recursive_sum (pd->blkg,NULL , - offsetof(struct blkcg_gq , stat_bytes), &tmp); +blkg_rwstat_recursive_sum (pd->blkg, &blkcg_policy_bfq, + offsetof(struct bfq_group, stats.bytes), &tmp);return __blkg_prfill_u64 (sf, pd, (tmp.cnt[BLKG_RWSTAT_READ ] + tmp.cnt[BLKG_RWSTAT_WRITE ]) >>9 );
@@ -1226,13 +1241,13 @@ struct cftype bfq_blkcg_legacy_files[] = {/* statistics, covers only the tasks in the bfqg */ { .name = "bfq.io_service_bytes", - .private = (unsigned long)&blkcg_policy_bfq, - .seq_show = blkg_print_stat_bytes, + .private = offsetof(struct bfq_group, stats.bytes), + .seq_show = bfqg_print_rwstat, }, { .name = "bfq.io_serviced", - .private = (unsigned long)&blkcg_policy_bfq, - .seq_show = blkg_print_stat_ios, + .private = offsetof(struct bfq_group, stats.ios), + .seq_show = bfqg_print_rwstat, }, #ifdef CONFIG_BFQ_CGROUP_DEBUG {
@@ -1269,13 +1284,13 @@ struct cftype bfq_blkcg_legacy_files[] = {/* the same statistics which cover the bfqg and its descendants */ { .name = "bfq.io_service_bytes_recursive", - .private = (unsigned long)&blkcg_policy_bfq, - .seq_show = blkg_print_stat_bytes_recursive, + .private = offsetof(struct bfq_group, stats.bytes), + .seq_show = bfqg_print_rwstat_recursive, }, { .name = "bfq.io_serviced_recursive", - .private = (unsigned long)&blkcg_policy_bfq, - .seq_show = blkg_print_stat_ios_recursive, + .private = offsetof(struct bfq_group, stats.ios), + .seq_show = bfqg_print_rwstat_recursive, }, #ifdef CONFIG_BFQ_CGROUP_DEBUG {
block/bfq-iosched.c +4 -0
@@ -5450,6 +5450,10 @@ static void bfq_insert_request(struct blk_mq_hw_ctx *hctx, struct request *rq,bool idle_timer_disabled = false;unsigned int cmd_flags; +#ifdef CONFIG_BFQ_GROUP_IOSCHED +if (!cgroup_subsys_on_dfl (io_cgrp_subsys) && rq->bio) +bfqg_stats_update_legacy_io (q, rq); +#endif spin_lock_irq (&bfqd->lock);if (blk_mq_sched_try_insert_merge (q, rq)) {spin_unlock_irq (&bfqd->lock);
block/bfq-iosched.h +4 -0
@@ -809,6 +809,9 @@ struct bfq_stat { };struct bfqg_stats { +/* basic stats */ +struct blkg_rwstat bytes; +struct blkg_rwstat ios;#ifdef CONFIG_BFQ_CGROUP_DEBUG /* number of ios merged */ struct blkg_rwstat merged;
@@ -956,6 +959,7 @@ void bfq_put_async_queues(struct bfq_data *bfqd, struct bfq_group *bfqg);/* ---------------- cgroups-support interface ---------------- */ +void bfqg_stats_update_legacy_io (struct request_queue *q,struct request *rq);void bfqg_stats_update_io_add (struct bfq_group *bfqg,struct bfq_queue *bfqq,unsigned int op);void bfqg_stats_update_io_remove (struct bfq_group *bfqg,unsigned int op);
Signature · 1 line
2.17.1
Tejun Heo
maintainerPATCH 2/5 blk-throtl: stop using blkg->stat_bytes and ->stat_ios
In reply to Tejun HeoWhen used on cgroup1, blk-throtl uses the blkg->stat_bytes and ->stat_ios from blk-cgroup core to populate four stat knobs. blk-cgroup core is moving away from blkg_rwstat to improve scalability and won't be able to support this usage.
It isn't like the sharing gains all that much. Let's break them out to dedicated rwstat counters which are updated when on cgroup1.
--- block/blk-throttle.c | 70 ++++++++++++++++++++++++++++++++++++++------ 1 file changed, 61 insertions(+), 9 deletions(-)
block/blk-throttle.c +61 -9
@@ -176,6 +176,9 @@ struct throtl_grp {unsigned int bio_cnt;/* total bios */ unsigned int bad_bio_cnt;/* bios exceeding latency threshold */ unsigned long bio_cnt_reset_time; + +struct blkg_rwstat stat_bytes; +struct blkg_rwstat stat_ios; };/* We measure latency for request size from <= 4k to >= 1M */
@@ -489,6 +492,12 @@ static struct blkg_policy_data *throtl_pd_alloc(gfp_t gfp,if (!tg)return NULL ; +if (blkg_rwstat_init (&tg->stat_bytes, gfp)) + goto err_free_tg; + +if (blkg_rwstat_init (&tg->stat_ios, gfp)) + goto err_exit_stat_bytes; +throtl_service_queue_init (&tg->service_queue);for (rw =READ ; rw <=WRITE ; rw++) {
@@ -513,6 +522,12 @@ static struct blkg_policy_data *throtl_pd_alloc(gfp_t gfp, tg->idletime_threshold_conf =DFL_IDLE_THRESHOLD ;return &tg->pd; + +err_exit_stat_bytes: +blkg_rwstat_exit (&tg->stat_bytes); +err_free_tg: +kfree (tg); +return NULL ; }static void throtl_pd_init (struct blkg_policy_data *pd)
@@ -611,6 +626,8 @@ static void throtl_pd_free(struct blkg_policy_data *pd)struct throtl_grp *tg =pd_to_tg (pd);del_timer_sync (&tg->service_queue.pending_timer); +blkg_rwstat_exit (&tg->stat_bytes); +blkg_rwstat_exit (&tg->stat_ios);kfree (tg); }
@@ -1464,6 +1481,32 @@ static ssize_t tg_set_conf_uint(struct kernfs_open_file *of,return tg_set_conf (of, buf, nbytes, off, false); } +static int tg_print_rwstat (struct seq_file *sf,void *v) +{ +blkcg_print_blkgs (sf,css_to_blkcg (seq_css (sf)), + blkg_prfill_rwstat, &blkcg_policy_throtl, +seq_cft (sf)->private, true); +return 0 ; +} + +static u64 tg_prfill_rwstat_recursive (struct seq_file *sf, +struct blkg_policy_data *pd,int off) +{ +struct blkg_rwstat_sample sum; + +blkg_rwstat_recursive_sum (pd_to_blkg (pd), &blkcg_policy_throtl, off, + &sum); +return __blkg_prfill_rwstat (sf, pd, &sum); +} + +static int tg_print_rwstat_recursive (struct seq_file *sf,void *v) +{ +blkcg_print_blkgs (sf,css_to_blkcg (seq_css (sf)), + tg_prfill_rwstat_recursive, &blkcg_policy_throtl, +seq_cft (sf)->private, true); +return 0 ; +} +static struct cftype throtl_legacy_files[] = { { .name ="throttle.read_bps_device" ,
@@ -1491,23 +1534,23 @@ static struct cftype throtl_legacy_files[] = { }, { .name = "throttle.io_service_bytes", - .private = (unsigned long)&blkcg_policy_throtl, - .seq_show = blkg_print_stat_bytes, + .private = offsetof(struct throtl_grp, stat_bytes), + .seq_show = tg_print_rwstat, }, { .name = "throttle.io_service_bytes_recursive", - .private = (unsigned long)&blkcg_policy_throtl, - .seq_show = blkg_print_stat_bytes_recursive, + .private = offsetof(struct throtl_grp, stat_bytes), + .seq_show = tg_print_rwstat_recursive, }, { .name = "throttle.io_serviced", - .private = (unsigned long)&blkcg_policy_throtl, - .seq_show = blkg_print_stat_ios, + .private = offsetof(struct throtl_grp, stat_ios), + .seq_show = tg_print_rwstat, }, { .name = "throttle.io_serviced_recursive", - .private = (unsigned long)&blkcg_policy_throtl, - .seq_show = blkg_print_stat_ios_recursive, + .private = offsetof(struct throtl_grp, stat_ios), + .seq_show = tg_print_rwstat_recursive, }, { } /* terminate */ };
@@ -2127,7 +2170,16 @@ bool blk_throtl_bio(struct request_queue *q, struct blkcg_gq *blkg,WARN_ON_ONCE (!rcu_read_lock_held ());/* see throtl_charge_bio() */ -if (bio_flagged (bio,BIO_THROTTLED ) || !tg->has_rules[rw]) +if (bio_flagged (bio,BIO_THROTTLED )) + goto out; + +if (!cgroup_subsys_on_dfl (io_cgrp_subsys)) { +blkg_rwstat_add (&tg->stat_bytes, bio->bi_opf, + bio->bi_iter.bi_size); +blkg_rwstat_add (&tg->stat_ios, bio->bi_opf,1 ); + } + +if (!tg->has_rules[rw]) goto out;spin_lock_irq (&q->queue_lock);
Signature · 1 line
2.17.1
Tejun Heo
maintainerPATCH 3/5 blk-cgroup: remove now unused blkg_print_stat_{bytes|ios}_recursive()
In reply to Tejun HeoThese don't have users anymore. Remove them.
--- block/blk-cgroup.c | 83 -------------------------------------- include/linux/blk-cgroup.h | 5 --- 2 files changed, 88 deletions(-)
block/blk-cgroup.c +0 -83
@@ -615,89 +615,6 @@ u64 blkg_prfill_rwstat(struct seq_file *sf, struct blkg_policy_data *pd, }EXPORT_SYMBOL_GPL (blkg_prfill_rwstat); -static u64 blkg_prfill_rwstat_field (struct seq_file *sf, -struct blkg_policy_data *pd,int off) -{ -struct blkg_rwstat_sample rwstat = { }; - -blkg_rwstat_read ((void *)pd->blkg + off, &rwstat); -return __blkg_prfill_rwstat (sf, pd, &rwstat); -} - -/** -* blkg_print_stat_bytes - seq_show callback for blkg->stat_bytes -* @sf: seq_file to print to -* @v: unused -* -* To be used as cftype->seq_show to print blkg->stat_bytes. -* cftype->private must be set to the blkcg_policy. -*/ -int blkg_print_stat_bytes (struct seq_file *sf,void *v) -{ -blkcg_print_blkgs (sf,css_to_blkcg (seq_css (sf)), - blkg_prfill_rwstat_field, (void *)seq_cft (sf)->private, - offsetof(struct blkcg_gq , stat_bytes), true); -return 0 ; -} -EXPORT_SYMBOL_GPL (blkg_print_stat_bytes); - -/** -* blkg_print_stat_bytes - seq_show callback for blkg->stat_ios -* @sf: seq_file to print to -* @v: unused -* -* To be used as cftype->seq_show to print blkg->stat_ios. cftype->private -* must be set to the blkcg_policy. -*/ -int blkg_print_stat_ios (struct seq_file *sf,void *v) -{ -blkcg_print_blkgs (sf,css_to_blkcg (seq_css (sf)), - blkg_prfill_rwstat_field, (void *)seq_cft (sf)->private, - offsetof(struct blkcg_gq , stat_ios), true); -return 0 ; -} -EXPORT_SYMBOL_GPL (blkg_print_stat_ios); - -static u64 blkg_prfill_rwstat_field_recursive (struct seq_file *sf, -struct blkg_policy_data *pd, -int off) -{ -struct blkg_rwstat_sample rwstat; - -blkg_rwstat_recursive_sum (pd->blkg,NULL , off, &rwstat); -return __blkg_prfill_rwstat (sf, pd, &rwstat); -} - -/** -* blkg_print_stat_bytes_recursive - recursive version of blkg_print_stat_bytes -* @sf: seq_file to print to -* @v: unused -*/ -int blkg_print_stat_bytes_recursive (struct seq_file *sf,void *v) -{ -blkcg_print_blkgs (sf,css_to_blkcg (seq_css (sf)), - blkg_prfill_rwstat_field_recursive, - (void *)seq_cft (sf)->private, - offsetof(struct blkcg_gq , stat_bytes), true); -return 0 ; -} -EXPORT_SYMBOL_GPL (blkg_print_stat_bytes_recursive); - -/** -* blkg_print_stat_ios_recursive - recursive version of blkg_print_stat_ios -* @sf: seq_file to print to -* @v: unused -*/ -int blkg_print_stat_ios_recursive (struct seq_file *sf,void *v) -{ -blkcg_print_blkgs (sf,css_to_blkcg (seq_css (sf)), - blkg_prfill_rwstat_field_recursive, - (void *)seq_cft (sf)->private, - offsetof(struct blkcg_gq , stat_ios), true); -return 0 ; -} -EXPORT_SYMBOL_GPL (blkg_print_stat_ios_recursive); -/** * blkg_rwstat_recursive_sum - collect hierarchical blkg_rwstat * @blkg: blkg of interest
include/linux/blk-cgroup.h +0 -5
@@ -220,11 +220,6 @@ u64 __blkg_prfill_rwstat(struct seq_file *sf, struct blkg_policy_data *pd, const struct blkg_rwstat_sample *rwstat);u64 blkg_prfill_rwstat (struct seq_file *sf,struct blkg_policy_data *pd,int off); -int blkg_print_stat_bytes (struct seq_file *sf,void *v); -int blkg_print_stat_ios (struct seq_file *sf,void *v); -int blkg_print_stat_bytes_recursive (struct seq_file *sf,void *v); -int blkg_print_stat_ios_recursive (struct seq_file *sf,void *v); -void blkg_rwstat_recursive_sum (struct blkcg_gq *blkg,struct blkcg_policy *pol,int off,struct blkg_rwstat_sample *sum);
Signature · 1 line
2.17.1
Tejun Heo
maintainerPATCH 4/5 blk-cgroup: reimplement basic IO stats using cgroup rstat
In reply to Tejun Heoblk-cgroup has been using blkg_rwstat to track basic IO stats. Unfortunately, reading recursive stats scales badly as itinvolves walking all descendants. On systems with a huge number of cgroups (dead or alive), this can lead to substantial CPU cost when reading IO stats.
This patch reimplements basic IO stats using cgroup rstat which uses more memory but makes recursive stat reading O(# descendants which have been active since last reading) instead of O(# descendants).
* blk-cgroup core no longer uses sync/async stats. Introduce new stat enums - BLKG_IOSTAT_{READ|WRITE|DISCARD}.
* Add blkg_iostat[_set] which encapsulates byte and io stats, last values for propagation delta calculation and u64_stats_sync for correctness on 32bit archs.
* Update the new percpu stat counters directly and implement blkcg_rstat_flush() to implement propagation.
* blkg_print_stat() can now bring the stats up to date by calling cgroup_rstat_flush() and print them instead of directly summing up all descendants.
* It now allocates 96 bytes per cpu. It used to be 40 bytes.
--- block/blk-cgroup.c | 124 +++++++++++++++++++++++++++++-------- include/linux/blk-cgroup.h | 48 ++++++++++++-- 2 files changed, 142 insertions(+), 30 deletions(-)
block/blk-cgroup.c +99 -25
@@ -80,8 +80,7 @@ static void blkg_free(struct blkcg_gq *blkg)if (blkg->pd[i]) blkcg_policy[i]->pd_free_fn (blkg->pd[i]); -blkg_rwstat_exit (&blkg->stat_ios); -blkg_rwstat_exit (&blkg->stat_bytes); +free_percpu (blkg->iostat_cpu);percpu_ref_exit (&blkg->refcnt);kfree (blkg); }
@@ -146,7 +145,7 @@ static struct blkcg_gq *blkg_alloc(struct blkcg *blkcg, struct request_queue *q, gfp_t gfp_mask) {struct blkcg_gq *blkg; -int i; +int i, cpu;/* alloc and init base part */ blkg =kzalloc_node (sizeof (*blkg), gfp_mask, q->node);
@@ -156,8 +155,8 @@ static struct blkcg_gq *blkg_alloc(struct blkcg *blkcg, struct request_queue *q,if (percpu_ref_init (&blkg->refcnt, blkg_release,0 , gfp_mask)) goto err_free; -if (blkg_rwstat_init (&blkg->stat_bytes, gfp_mask) || -blkg_rwstat_init (&blkg->stat_ios, gfp_mask)) + blkg->iostat_cpu = alloc_percpu_gfp(struct blkg_iostat_set, gfp_mask); +if (!blkg->iostat_cpu) goto err_free; blkg->q = q;
@@ -167,6 +166,10 @@ static struct blkcg_gq *blkg_alloc(struct blkcg *blkcg, struct request_queue *q,INIT_WORK (&blkg->async_bio_work, blkg_async_bio_workfn); blkg->blkcg = blkcg; +u64_stats_init (&blkg->iostat.sync); +for_each_possible_cpu (cpu) +u64_stats_init (&per_cpu_ptr (blkg->iostat_cpu, cpu)->sync); +for (i =0 ; i <BLKCG_MAX_POLS ; i++) {struct blkcg_policy *pol = blkcg_policy[i];struct blkg_policy_data *pd;
@@ -393,7 +396,6 @@ struct blkcg_gq *blkg_lookup_create(struct blkcg *blkcg,static void blkg_destroy (struct blkcg_gq *blkg) {struct blkcg *blkcg = blkg->blkcg; -struct blkcg_gq *parent = blkg->parent;int i;lockdep_assert_held (&blkg->q->queue_lock);
@@ -410,11 +412,6 @@ static void blkg_destroy(struct blkcg_gq *blkg) pol->pd_offline_fn (blkg->pd[i]); } -if (parent) { -blkg_rwstat_add_aux (&parent->stat_bytes, &blkg->stat_bytes); -blkg_rwstat_add_aux (&parent->stat_ios, &blkg->stat_ios); - } - blkg->online = false;radix_tree_delete (&blkcg->blkg_tree, blkg->q->id);
@@ -464,7 +461,7 @@ static int blkcg_reset_stats(struct cgroup_subsys_state *css, {struct blkcg *blkcg =css_to_blkcg (css);struct blkcg_gq *blkg; -int i; +int i, cpu;mutex_lock (&blkcg_pol_mutex);spin_lock_irq (&blkcg->lock);
@@ -475,8 +472,12 @@ static int blkcg_reset_stats(struct cgroup_subsys_state *css,* anyway. If you get hit by a race, retry. */ hlist_for_each_entry (blkg, &blkcg->blkg_list, blkcg_node) { -blkg_rwstat_reset (&blkg->stat_bytes); -blkg_rwstat_reset (&blkg->stat_ios); +for_each_possible_cpu (cpu) { +struct blkg_iostat_set *bis = +per_cpu_ptr (blkg->iostat_cpu, cpu); +memset (bis,0 ,sizeof (*bis)); + } +memset (&blkg->iostat,0 ,sizeof (blkg->iostat));for (i =0 ; i <BLKCG_MAX_POLS ; i++) {struct blkcg_policy *pol = blkcg_policy[i];
@@ -838,16 +839,18 @@ static int blkcg_print_stat(struct seq_file *sf, void *v)struct blkcg *blkcg =css_to_blkcg (seq_css (sf));struct blkcg_gq *blkg; +cgroup_rstat_flush (blkcg->css.cgroup);rcu_read_lock ();hlist_for_each_entry_rcu (blkg, &blkcg->blkg_list, blkcg_node) { +struct blkg_iostat_set *bis = &blkg->iostat;const char *dname;char *buf; -struct blkg_rwstat_sample rwstat;u64 rbytes, wbytes, rios, wios, dbytes, dios;size_t size =seq_get_buf (sf, &buf), off =0 ;int i;bool has_stats = false; +unsigned seq;spin_lock_irq (&blkg->q->queue_lock);
@@ -866,17 +869,16 @@ static int blkcg_print_stat(struct seq_file *sf, void *v)*/ off +=scnprintf (buf+off, size-off,"%s " , dname); -blkg_rwstat_recursive_sum (blkg,NULL , - offsetof(struct blkcg_gq , stat_bytes), &rwstat); - rbytes = rwstat.cnt[BLKG_RWSTAT_READ ]; - wbytes = rwstat.cnt[BLKG_RWSTAT_WRITE ]; - dbytes = rwstat.cnt[BLKG_RWSTAT_DISCARD ]; +do { + seq =u64_stats_fetch_begin (&bis->sync); -blkg_rwstat_recursive_sum (blkg,NULL , - offsetof(struct blkcg_gq , stat_ios), &rwstat); - rios = rwstat.cnt[BLKG_RWSTAT_READ ]; - wios = rwstat.cnt[BLKG_RWSTAT_WRITE ]; - dios = rwstat.cnt[BLKG_RWSTAT_DISCARD ]; + rbytes = bis->cur.bytes[BLKG_IOSTAT_READ ]; + wbytes = bis->cur.bytes[BLKG_IOSTAT_WRITE ]; + dbytes = bis->cur.bytes[BLKG_IOSTAT_DISCARD ]; + rios = bis->cur.ios[BLKG_IOSTAT_READ ]; + wios = bis->cur.ios[BLKG_IOSTAT_WRITE ]; + dios = bis->cur.ios[BLKG_IOSTAT_DISCARD ]; + }while (u64_stats_fetch_retry (&bis->sync, seq));if (rbytes || wbytes || rios || wios) { has_stats = true;
@@ -1212,6 +1214,77 @@ static int blkcg_can_attach(struct cgroup_taskset *tset)return ret; } +static void blkg_iostat_set (struct blkg_iostat *dst,struct blkg_iostat *src) +{ +int i; + +for (i =0 ; i <BLKG_IOSTAT_NR ; i++) { + dst->bytes[i] = src->bytes[i]; + dst->ios[i] = src->ios[i]; + } +} + +static void blkg_iostat_add (struct blkg_iostat *dst,struct blkg_iostat *src) +{ +int i; + +for (i =0 ; i <BLKG_IOSTAT_NR ; i++) { + dst->bytes[i] += src->bytes[i]; + dst->ios[i] += src->ios[i]; + } +} + +static void blkg_iostat_sub (struct blkg_iostat *dst,struct blkg_iostat *src) +{ +int i; + +for (i =0 ; i <BLKG_IOSTAT_NR ; i++) { + dst->bytes[i] -= src->bytes[i]; + dst->ios[i] -= src->ios[i]; + } +} + +static void blkcg_rstat_flush (struct cgroup_subsys_state *css,int cpu) +{ +struct blkcg *blkcg =css_to_blkcg (css); +struct blkcg_gq *blkg; + +rcu_read_lock (); + +hlist_for_each_entry_rcu (blkg, &blkcg->blkg_list, blkcg_node) { +struct blkcg_gq *parent = blkg->parent; +struct blkg_iostat_set *bisc =per_cpu_ptr (blkg->iostat_cpu, cpu); +struct blkg_iostat cur, delta; +unsigned seq; + +/* fetch the current per-cpu values */ +do { + seq =u64_stats_fetch_begin (&bisc->sync); +blkg_iostat_set (&cur, &bisc->cur); + }while (u64_stats_fetch_retry (&bisc->sync, seq)); + +/* propagate percpu delta to global */ +u64_stats_update_begin (&blkg->iostat.sync); +blkg_iostat_set (&delta, &cur); +blkg_iostat_sub (&delta, &bisc->last); +blkg_iostat_add (&blkg->iostat.cur, &delta); +blkg_iostat_add (&bisc->last, &delta); +u64_stats_update_end (&blkg->iostat.sync); + +/* propagate global delta to parent */ +if (parent) { +u64_stats_update_begin (&parent->iostat.sync); +blkg_iostat_set (&delta, &blkg->iostat.cur); +blkg_iostat_sub (&delta, &blkg->iostat.last); +blkg_iostat_add (&parent->iostat.cur, &delta); +blkg_iostat_add (&blkg->iostat.last, &delta); +u64_stats_update_end (&parent->iostat.sync); + } + } + +rcu_read_unlock (); +} +static void blkcg_bind (struct cgroup_subsys_state *root_css) {int i;
@@ -1244,6 +1317,7 @@ struct cgroup_subsys io_cgrp_subsys = { .css_offline = blkcg_css_offline, .css_free = blkcg_css_free, .can_attach = blkcg_can_attach, + .css_rstat_flush = blkcg_rstat_flush, .bind = blkcg_bind, .dfl_cftypes = blkcg_files, .legacy_cftypes = blkcg_legacy_files,
include/linux/blk-cgroup.h +43 -5
85 lines not on this page · raw message
Signature · 1 line
2.17.1
Tejun Heo
maintainerPATCH 5/5 blk-cgroup: separate out blkg_rwstat under CONFIG_BLK_CGROUP_RWSTAT
In reply to Tejun Heoblkg_rwstat is now only used by bfq-iosched and blk-throtl when on cgroup1. Let's move it into its own files and gate it behind a config option.
--- block/Kconfig | 4 + block/Kconfig.iosched | 1 + block/Makefile | 1 + block/bfq-iosched.h | 2 + block/blk-cgroup-rwstat.c | 129 ++++++++++++++++++++++++++++++ block/blk-cgroup-rwstat.h | 149 ++++++++++++++++++++++++++++++++++ block/blk-cgroup.c | 97 ---------------------- block/blk-throttle.c | 1 + include/linux/blk-cgroup.h | 159 ------------------------------------- 9 files changed, 287 insertions(+), 256 deletions(-) create mode 100644 block/blk-cgroup-rwstat.c create mode 100644 block/blk-cgroup-rwstat.h
block/Kconfig +4 -0
@@ -32,6 +32,9 @@ config BLK_RQ_ALLOC_TIME config BLK_SCSI_REQUEST bool +config BLK_CGROUP_RWSTAT + bool + config BLK_DEV_BSG bool "Block layer SG support v4" default y
@@ -86,6 +89,7 @@ config BLK_DEV_ZONED config BLK_DEV_THROTTLING bool "Block layer bio throttling support" depends on BLK_CGROUP=y + select BLK_CGROUP_RWSTAT ---help--- Block layer bio throttling support. It can be used to limit the IO rate to a device. IO rate policies are per cgroup and
block/Kconfig.iosched +1 -0
@@ -31,6 +31,7 @@ config IOSCHED_BFQ config BFQ_GROUP_IOSCHED bool "BFQ hierarchical scheduling support" depends on IOSCHED_BFQ && BLK_CGROUP + select BLK_CGROUP_RWSTAT ---help--- Enable hierarchical scheduling in BFQ, using the blkio
block/Makefile +1 -0
@@ -16,6 +16,7 @@ obj-$(CONFIG_BLK_SCSI_REQUEST) += scsi_ioctl.o obj-$(CONFIG_BLK_DEV_BSG) += bsg.o obj-$(CONFIG_BLK_DEV_BSGLIB) += bsg-lib.o obj-$(CONFIG_BLK_CGROUP) += blk-cgroup.o +obj-$(CONFIG_BLK_CGROUP_RWSTAT) += blk-cgroup-rwstat.o obj-$(CONFIG_BLK_DEV_THROTTLING) += blk-throttle.o obj-$(CONFIG_BLK_CGROUP_IOLATENCY) += blk-iolatency.o obj-$(CONFIG_BLK_CGROUP_IOCOST) += blk-iocost.o
block/bfq-iosched.h +2 -0
@@ -10,6 +10,8 @@#include <linux/hrtimer.h> #include <linux/blk-cgroup.h> +#include "blk-cgroup-rwstat.h" +#define BFQ_IOPRIO_CLASSES 3#define BFQ_CL_IDLE_TIMEOUT (HZ/5)
block/blk-cgroup-rwstat.c new file +129 -0
129 lines not on this page · raw message
block/blk-cgroup-rwstat.h new file +149 -0
149 lines not on this page · raw message
block/blk-cgroup.c +0 -97
103 lines not on this page · raw message
block/blk-throttle.c +1 -0
@@ -12,6 +12,7 @@#include <linux/blktrace_api.h> #include <linux/blk-cgroup.h> #include "blk.h" +#include "blk-cgroup-rwstat.h" /* Max dispatch from a group in 1 round */ static int throtl_grp_quantum =8 ;
include/linux/blk-cgroup.h +0 -159
189 lines not on this page · raw message
Signature · 1 line
2.17.1
Tejun Heo
maintainerPATCH 1/1 bfq-iosched: relocate bfqg_*rwstat*() helpers
In reply to Tejun HeoFrom a7b9100d72d75a1472bed130b6edd271b734776f Mon Sep 17 00:00:00 2001
From: Tejun Heo <[email protected]>
Date: Thu, 7 Nov 2019 11:01:19 -0800
Collect them right under #ifdef CONFIG_BFQ_CGROUP_DEBUG. The next patch will use them from !DEBUG path and this makes it easy to move them out of the ifdef block.
This is pure code reorganization.
---
Jens, this is an extra patch to help fixing the build bug when !CONFIG_BFQ_CGROUP_DEBUG. The git tree has been updated accordingly. Please let me know if you want the whole series reposted.
Thanks.
block/bfq-cgroup.c | 46 +++++++++++++++++++++++----------------------- 1 file changed, 23 insertions(+), 23 deletions(-)
block/bfq-cgroup.c +23 -23
@@ -1058,17 +1058,34 @@ static ssize_t bfq_io_set_weight(struct kernfs_open_file *of, }#ifdef CONFIG_BFQ_CGROUP_DEBUG -static int bfqg_print_stat (struct seq_file *sf,void *v) +static int bfqg_print_rwstat (struct seq_file *sf,void *v) { -blkcg_print_blkgs (sf,css_to_blkcg (seq_css (sf)), blkg_prfill_stat, - &blkcg_policy_bfq,seq_cft (sf)->private, false); +blkcg_print_blkgs (sf,css_to_blkcg (seq_css (sf)), blkg_prfill_rwstat, + &blkcg_policy_bfq,seq_cft (sf)->private, true);return 0 ; } -static int bfqg_print_rwstat (struct seq_file *sf,void *v) +static u64 bfqg_prfill_rwstat_recursive (struct seq_file *sf, +struct blkg_policy_data *pd,int off) { -blkcg_print_blkgs (sf,css_to_blkcg (seq_css (sf)), blkg_prfill_rwstat, - &blkcg_policy_bfq,seq_cft (sf)->private, true); +struct blkg_rwstat_sample sum; + +blkg_rwstat_recursive_sum (pd_to_blkg (pd), &blkcg_policy_bfq, off, &sum); +return __blkg_prfill_rwstat (sf, pd, &sum); +} + +static int bfqg_print_rwstat_recursive (struct seq_file *sf,void *v) +{ +blkcg_print_blkgs (sf,css_to_blkcg (seq_css (sf)), + bfqg_prfill_rwstat_recursive, &blkcg_policy_bfq, +seq_cft (sf)->private, true); +return 0 ; +} + +static int bfqg_print_stat (struct seq_file *sf,void *v) +{ +blkcg_print_blkgs (sf,css_to_blkcg (seq_css (sf)), blkg_prfill_stat, + &blkcg_policy_bfq,seq_cft (sf)->private, false);return 0 ; }
@@ -1097,15 +1114,6 @@ static u64 bfqg_prfill_stat_recursive(struct seq_file *sf,return __blkg_prfill_u64 (sf, pd, sum); } -static u64 bfqg_prfill_rwstat_recursive (struct seq_file *sf, -struct blkg_policy_data *pd,int off) -{ -struct blkg_rwstat_sample sum; - -blkg_rwstat_recursive_sum (pd_to_blkg (pd), &blkcg_policy_bfq, off, &sum); -return __blkg_prfill_rwstat (sf, pd, &sum); -} -static int bfqg_print_stat_recursive (struct seq_file *sf,void *v) { blkcg_print_blkgs(sf, css_to_blkcg(seq_css(sf)),
@@ -1114,14 +1122,6 @@ static int bfqg_print_stat_recursive(struct seq_file *sf, void *v)return 0 ; } -static int bfqg_print_rwstat_recursive (struct seq_file *sf,void *v) -{ -blkcg_print_blkgs (sf,css_to_blkcg (seq_css (sf)), - bfqg_prfill_rwstat_recursive, &blkcg_policy_bfq, -seq_cft (sf)->private, true); -return 0 ; -} -static u64 bfqg_prfill_sectors (struct seq_file *sf,struct blkg_policy_data *pd,int off) {
Signature · 1 line
2.17.1
Jens Axboe
maintainerRe: [PATCH 0.5/5] bfq-iosched: relocate bfqg_*rwstat*() helpers
Quoted 3 lines · Tejun Heo
Jens, this is an extra patch to help fixing the build bug when !CONFIG_BFQ_CGROUP_DEBUG. The git tree has been updated accordingly. Please let me know if you want the whole series reposted.
I seem to run into issues applying the series with the 0.5 added first and with the updated patch 1. So maybe a resend isn't a bad idea :-)
Signature · 1 line
Jens Axboe