blk-cgroup: use cgroup rstat for IO stats
Subsystems (4)
Tejun Heo
maintainerPATCH v2 1/1 blk-cgroup: use cgroup rstat for IO stats
Hello,
v2: Build fix when !DEBUG.
blk-cgroup IO stats currently use blkg_rwstat which unforutnately requires walking all descendants recursively on read. On systems with a large number of cgroups (dead or alive), this can make each stat read a substantially expensive operation.
This patch updates blk-cgroup to use cgroup rstat which makes stat reading O(# descendants which have been active since last reading) instead of O(# descendants).
0001-bfq-iosched-relocate-bfqg_-rwstat-helpers.patch 0002-bfq-iosched-stop-using-blkg-stat_bytes-and-stat_ios.patch 0003-blk-throtl-stop-using-blkg-stat_bytes-and-stat_ios.patch 0004-blk-cgroup-remove-now-unused-blkg_print_stat_-bytes-.patch 0005-blk-cgroup-reimplement-basic-IO-stats-using-cgroup-r.patch 0006-blk-cgroup-separate-out-blkg_rwstat-under-CONFIG_BLK.patch
0001-0003 make bfq-iosched and blk-throtl use their own blkg_rwstat to track basic IO stats on cgroup1 instead of sharing blk-cgroup core's. 0004 is a follow-up cleanup.
0005 switches blk-cgroup to cgroup rstat.
0006 moves blkg_rwstat to its own files and gate it under a config option as it's now only used by blk-throtl and bfq-iosched.
The patchset is on top of
block/for-next 40afbe18b03a ("Merge branch 'for-5.5/drivers-post' into for-next")
+ block/for-linus b0814361a25c ("blkcg: make blkcg_print_stat() print stats only for online blkgs")
and also available in the following git branch.
git://git.kernel.org/pub/scm/linux/kernel/git/tj/cgroup.git review-blkcg-rstat
Thanks.
block/Kconfig | 4 block/Kconfig.iosched | 1 block/Makefile | 1 block/bfq-cgroup.c | 37 +++-- block/bfq-iosched.c | 4 block/bfq-iosched.h | 6 block/blk-cgroup-rwstat.c | 129 +++++++++++++++++++ block/blk-cgroup-rwstat.h | 149 ++++++++++++++++++++++ block/blk-cgroup.c | 304 ++++++++++++++------------------------------- block/blk-throttle.c | 71 +++++++++- include/linux/blk-cgroup.h | 198 +++++------------------------ 11 files changed, 517 insertions(+), 387 deletions(-)
--
tejun
Tejun Heo
maintainerPATCH 1/6 bfq-iosched: relocate bfqg_*rwstat*() helpers
Collect them right under #ifdef CONFIG_BFQ_CGROUP_DEBUG. The next patch will use them from !DEBUG path and this makes it easy to move them out of the ifdef block.
This is pure code reorganization.
--- block/bfq-cgroup.c | 46 +++++++++++++++++++++++----------------------- 1 file changed, 23 insertions(+), 23 deletions(-)
block/bfq-cgroup.c +23 -23
@@ -1058,17 +1058,34 @@ static ssize_t bfq_io_set_weight(struct kernfs_open_file *of, }#ifdef CONFIG_BFQ_CGROUP_DEBUG -static int bfqg_print_stat (struct seq_file *sf,void *v) +static int bfqg_print_rwstat (struct seq_file *sf,void *v) { -blkcg_print_blkgs (sf,css_to_blkcg (seq_css (sf)), blkg_prfill_stat, - &blkcg_policy_bfq,seq_cft (sf)->private, false); +blkcg_print_blkgs (sf,css_to_blkcg (seq_css (sf)), blkg_prfill_rwstat, + &blkcg_policy_bfq,seq_cft (sf)->private, true);return 0 ; } -static int bfqg_print_rwstat (struct seq_file *sf,void *v) +static u64 bfqg_prfill_rwstat_recursive (struct seq_file *sf, +struct blkg_policy_data *pd,int off) { -blkcg_print_blkgs (sf,css_to_blkcg (seq_css (sf)), blkg_prfill_rwstat, - &blkcg_policy_bfq,seq_cft (sf)->private, true); +struct blkg_rwstat_sample sum; + +blkg_rwstat_recursive_sum (pd_to_blkg (pd), &blkcg_policy_bfq, off, &sum); +return __blkg_prfill_rwstat (sf, pd, &sum); +} + +static int bfqg_print_rwstat_recursive (struct seq_file *sf,void *v) +{ +blkcg_print_blkgs (sf,css_to_blkcg (seq_css (sf)), + bfqg_prfill_rwstat_recursive, &blkcg_policy_bfq, +seq_cft (sf)->private, true); +return 0 ; +} + +static int bfqg_print_stat (struct seq_file *sf,void *v) +{ +blkcg_print_blkgs (sf,css_to_blkcg (seq_css (sf)), blkg_prfill_stat, + &blkcg_policy_bfq,seq_cft (sf)->private, false);return 0 ; }
@@ -1097,15 +1114,6 @@ static u64 bfqg_prfill_stat_recursive(struct seq_file *sf,return __blkg_prfill_u64 (sf, pd, sum); } -static u64 bfqg_prfill_rwstat_recursive (struct seq_file *sf, -struct blkg_policy_data *pd,int off) -{ -struct blkg_rwstat_sample sum; - -blkg_rwstat_recursive_sum (pd_to_blkg (pd), &blkcg_policy_bfq, off, &sum); -return __blkg_prfill_rwstat (sf, pd, &sum); -} -static int bfqg_print_stat_recursive (struct seq_file *sf,void *v) { blkcg_print_blkgs(sf, css_to_blkcg(seq_css(sf)),
@@ -1114,14 +1122,6 @@ static int bfqg_print_stat_recursive(struct seq_file *sf, void *v)return 0 ; } -static int bfqg_print_rwstat_recursive (struct seq_file *sf,void *v) -{ -blkcg_print_blkgs (sf,css_to_blkcg (seq_css (sf)), - bfqg_prfill_rwstat_recursive, &blkcg_policy_bfq, -seq_cft (sf)->private, true); -return 0 ; -} -static u64 bfqg_prfill_sectors (struct seq_file *sf,struct blkg_policy_data *pd,int off) {
Signature · 1 line
2.17.1
Tejun Heo
maintainerPATCH 2/6 bfq-iosched: stop using blkg->stat_bytes and ->stat_ios
In reply to Tejun HeoWhen used on cgroup1, bfq uses the blkg->stat_bytes and ->stat_ios from blk-cgroup core to populate six stat knobs. blk-cgroup core is moving away from blkg_rwstat to improve scalability and won't be able to support this usage.
It isn't like the sharing gains all that much. Let's break it out to dedicated rwstat counters which are updated when on cgroup1. This makes use of bfqg_*rwstat*() helpers outside of CONFIG_BFQ_CGROUP_DEBUG. Move them out.
v2: Compile fix when !CONFIG_BFQ_CGROUP_DEBUG.
--- block/bfq-cgroup.c | 39 +++++++++++++++++++++++++++------------ block/bfq-iosched.c | 4 ++++ block/bfq-iosched.h | 4 ++++ 3 files changed, 35 insertions(+), 12 deletions(-)
block/bfq-cgroup.c +27 -12
@@ -347,6 +347,14 @@ void bfqg_and_blkg_put(struct bfq_group *bfqg)bfqg_put (bfqg); } +void bfqg_stats_update_legacy_io (struct request_queue *q,struct request *rq) +{ +struct bfq_group *bfqg =blkg_to_bfqg (rq->bio->bi_blkg); + +blkg_rwstat_add (&bfqg->stats.bytes, rq->cmd_flags,blk_rq_bytes (rq)); +blkg_rwstat_add (&bfqg->stats.ios, rq->cmd_flags,1 ); +} +/* @stats = 0 */ static void bfqg_stats_reset (struct bfqg_stats *stats) {
@@ -431,6 +439,8 @@ void bfq_init_entity(struct bfq_entity *entity, struct bfq_group *bfqg)static void bfqg_stats_exit (struct bfqg_stats *stats) { +blkg_rwstat_exit (&stats->bytes); +blkg_rwstat_exit (&stats->ios);#ifdef CONFIG_BFQ_CGROUP_DEBUG blkg_rwstat_exit (&stats->merged);blkg_rwstat_exit (&stats->service_time);
@@ -448,6 +458,10 @@ static void bfqg_stats_exit(struct bfqg_stats *stats)static int bfqg_stats_init (struct bfqg_stats *stats,gfp_t gfp) { +if (blkg_rwstat_init (&stats->bytes, gfp) || +blkg_rwstat_init (&stats->ios, gfp)) +return -ENOMEM ; +#ifdef CONFIG_BFQ_CGROUP_DEBUG if (blkg_rwstat_init(&stats->merged, gfp) || blkg_rwstat_init(&stats->service_time, gfp) ||
@@ -1057,7 +1071,6 @@ static ssize_t bfq_io_set_weight(struct kernfs_open_file *of,return bfq_io_set_device_weight (of, buf, nbytes, off); } -#ifdef CONFIG_BFQ_CGROUP_DEBUG static int bfqg_print_rwstat (struct seq_file *sf,void *v) { blkcg_print_blkgs(sf, css_to_blkcg(seq_css(sf)), blkg_prfill_rwstat,
@@ -1082,6 +1095,7 @@ static int bfqg_print_rwstat_recursive(struct seq_file *sf, void *v)return 0 ; } +#ifdef CONFIG_BFQ_CGROUP_DEBUG static int bfqg_print_stat (struct seq_file *sf,void *v) { blkcg_print_blkgs(sf, css_to_blkcg(seq_css(sf)), blkg_prfill_stat,
@@ -1125,7 +1139,8 @@ static int bfqg_print_stat_recursive(struct seq_file *sf, void *v)static u64 bfqg_prfill_sectors (struct seq_file *sf,struct blkg_policy_data *pd,int off) { -u64 sum =blkg_rwstat_total (&pd->blkg->stat_bytes); +struct bfq_group *bfqg =blkg_to_bfqg (pd->blkg); +u64 sum =blkg_rwstat_total (&bfqg->stats.bytes);return __blkg_prfill_u64 (sf, pd, sum >>9 ); }
@@ -1142,8 +1157,8 @@ static u64 bfqg_prfill_sectors_recursive(struct seq_file *sf, {struct blkg_rwstat_sample tmp; -blkg_rwstat_recursive_sum (pd->blkg,NULL , - offsetof(struct blkcg_gq , stat_bytes), &tmp); +blkg_rwstat_recursive_sum (pd->blkg, &blkcg_policy_bfq, + offsetof(struct bfq_group, stats.bytes), &tmp);return __blkg_prfill_u64 (sf, pd, (tmp.cnt[BLKG_RWSTAT_READ ] + tmp.cnt[BLKG_RWSTAT_WRITE ]) >>9 );
@@ -1226,13 +1241,13 @@ struct cftype bfq_blkcg_legacy_files[] = {/* statistics, covers only the tasks in the bfqg */ { .name = "bfq.io_service_bytes", - .private = (unsigned long)&blkcg_policy_bfq, - .seq_show = blkg_print_stat_bytes, + .private = offsetof(struct bfq_group, stats.bytes), + .seq_show = bfqg_print_rwstat, }, { .name = "bfq.io_serviced", - .private = (unsigned long)&blkcg_policy_bfq, - .seq_show = blkg_print_stat_ios, + .private = offsetof(struct bfq_group, stats.ios), + .seq_show = bfqg_print_rwstat, }, #ifdef CONFIG_BFQ_CGROUP_DEBUG {
@@ -1269,13 +1284,13 @@ struct cftype bfq_blkcg_legacy_files[] = {/* the same statistics which cover the bfqg and its descendants */ { .name = "bfq.io_service_bytes_recursive", - .private = (unsigned long)&blkcg_policy_bfq, - .seq_show = blkg_print_stat_bytes_recursive, + .private = offsetof(struct bfq_group, stats.bytes), + .seq_show = bfqg_print_rwstat_recursive, }, { .name = "bfq.io_serviced_recursive", - .private = (unsigned long)&blkcg_policy_bfq, - .seq_show = blkg_print_stat_ios_recursive, + .private = offsetof(struct bfq_group, stats.ios), + .seq_show = bfqg_print_rwstat_recursive, }, #ifdef CONFIG_BFQ_CGROUP_DEBUG {
block/bfq-iosched.c +4 -0
@@ -5464,6 +5464,10 @@ static void bfq_insert_request(struct blk_mq_hw_ctx *hctx, struct request *rq,bool idle_timer_disabled = false;unsigned int cmd_flags; +#ifdef CONFIG_BFQ_GROUP_IOSCHED +if (!cgroup_subsys_on_dfl (io_cgrp_subsys) && rq->bio) +bfqg_stats_update_legacy_io (q, rq); +#endif spin_lock_irq (&bfqd->lock);if (blk_mq_sched_try_insert_merge (q, rq)) {spin_unlock_irq (&bfqd->lock);
block/bfq-iosched.h +4 -0
@@ -809,6 +809,9 @@ struct bfq_stat { };struct bfqg_stats { +/* basic stats */ +struct blkg_rwstat bytes; +struct blkg_rwstat ios;#ifdef CONFIG_BFQ_CGROUP_DEBUG /* number of ios merged */ struct blkg_rwstat merged;
@@ -956,6 +959,7 @@ void bfq_put_async_queues(struct bfq_data *bfqd, struct bfq_group *bfqg);/* ---------------- cgroups-support interface ---------------- */ +void bfqg_stats_update_legacy_io (struct request_queue *q,struct request *rq);void bfqg_stats_update_io_add (struct bfq_group *bfqg,struct bfq_queue *bfqq,unsigned int op);void bfqg_stats_update_io_remove (struct bfq_group *bfqg,unsigned int op);
Signature · 1 line
2.17.1
Tejun Heo
maintainerPATCH 3/6 blk-throtl: stop using blkg->stat_bytes and ->stat_ios
In reply to Tejun HeoWhen used on cgroup1, blk-throtl uses the blkg->stat_bytes and ->stat_ios from blk-cgroup core to populate four stat knobs. blk-cgroup core is moving away from blkg_rwstat to improve scalability and won't be able to support this usage.
It isn't like the sharing gains all that much. Let's break them out to dedicated rwstat counters which are updated when on cgroup1.
--- block/blk-throttle.c | 70 ++++++++++++++++++++++++++++++++++++++------ 1 file changed, 61 insertions(+), 9 deletions(-)
block/blk-throttle.c +61 -9
@@ -176,6 +176,9 @@ struct throtl_grp {unsigned int bio_cnt;/* total bios */ unsigned int bad_bio_cnt;/* bios exceeding latency threshold */ unsigned long bio_cnt_reset_time; + +struct blkg_rwstat stat_bytes; +struct blkg_rwstat stat_ios; };/* We measure latency for request size from <= 4k to >= 1M */
@@ -489,6 +492,12 @@ static struct blkg_policy_data *throtl_pd_alloc(gfp_t gfp,if (!tg)return NULL ; +if (blkg_rwstat_init (&tg->stat_bytes, gfp)) + goto err_free_tg; + +if (blkg_rwstat_init (&tg->stat_ios, gfp)) + goto err_exit_stat_bytes; +throtl_service_queue_init (&tg->service_queue);for (rw =READ ; rw <=WRITE ; rw++) {
@@ -513,6 +522,12 @@ static struct blkg_policy_data *throtl_pd_alloc(gfp_t gfp, tg->idletime_threshold_conf =DFL_IDLE_THRESHOLD ;return &tg->pd; + +err_exit_stat_bytes: +blkg_rwstat_exit (&tg->stat_bytes); +err_free_tg: +kfree (tg); +return NULL ; }static void throtl_pd_init (struct blkg_policy_data *pd)
@@ -611,6 +626,8 @@ static void throtl_pd_free(struct blkg_policy_data *pd)struct throtl_grp *tg =pd_to_tg (pd);del_timer_sync (&tg->service_queue.pending_timer); +blkg_rwstat_exit (&tg->stat_bytes); +blkg_rwstat_exit (&tg->stat_ios);kfree (tg); }
@@ -1464,6 +1481,32 @@ static ssize_t tg_set_conf_uint(struct kernfs_open_file *of,return tg_set_conf (of, buf, nbytes, off, false); } +static int tg_print_rwstat (struct seq_file *sf,void *v) +{ +blkcg_print_blkgs (sf,css_to_blkcg (seq_css (sf)), + blkg_prfill_rwstat, &blkcg_policy_throtl, +seq_cft (sf)->private, true); +return 0 ; +} + +static u64 tg_prfill_rwstat_recursive (struct seq_file *sf, +struct blkg_policy_data *pd,int off) +{ +struct blkg_rwstat_sample sum; + +blkg_rwstat_recursive_sum (pd_to_blkg (pd), &blkcg_policy_throtl, off, + &sum); +return __blkg_prfill_rwstat (sf, pd, &sum); +} + +static int tg_print_rwstat_recursive (struct seq_file *sf,void *v) +{ +blkcg_print_blkgs (sf,css_to_blkcg (seq_css (sf)), + tg_prfill_rwstat_recursive, &blkcg_policy_throtl, +seq_cft (sf)->private, true); +return 0 ; +} +static struct cftype throtl_legacy_files[] = { { .name ="throttle.read_bps_device" ,
@@ -1491,23 +1534,23 @@ static struct cftype throtl_legacy_files[] = { }, { .name = "throttle.io_service_bytes", - .private = (unsigned long)&blkcg_policy_throtl, - .seq_show = blkg_print_stat_bytes, + .private = offsetof(struct throtl_grp, stat_bytes), + .seq_show = tg_print_rwstat, }, { .name = "throttle.io_service_bytes_recursive", - .private = (unsigned long)&blkcg_policy_throtl, - .seq_show = blkg_print_stat_bytes_recursive, + .private = offsetof(struct throtl_grp, stat_bytes), + .seq_show = tg_print_rwstat_recursive, }, { .name = "throttle.io_serviced", - .private = (unsigned long)&blkcg_policy_throtl, - .seq_show = blkg_print_stat_ios, + .private = offsetof(struct throtl_grp, stat_ios), + .seq_show = tg_print_rwstat, }, { .name = "throttle.io_serviced_recursive", - .private = (unsigned long)&blkcg_policy_throtl, - .seq_show = blkg_print_stat_ios_recursive, + .private = offsetof(struct throtl_grp, stat_ios), + .seq_show = tg_print_rwstat_recursive, }, { } /* terminate */ };
@@ -2127,7 +2170,16 @@ bool blk_throtl_bio(struct request_queue *q, struct blkcg_gq *blkg,WARN_ON_ONCE (!rcu_read_lock_held ());/* see throtl_charge_bio() */ -if (bio_flagged (bio,BIO_THROTTLED ) || !tg->has_rules[rw]) +if (bio_flagged (bio,BIO_THROTTLED )) + goto out; + +if (!cgroup_subsys_on_dfl (io_cgrp_subsys)) { +blkg_rwstat_add (&tg->stat_bytes, bio->bi_opf, + bio->bi_iter.bi_size); +blkg_rwstat_add (&tg->stat_ios, bio->bi_opf,1 ); + } + +if (!tg->has_rules[rw]) goto out;spin_lock_irq (&q->queue_lock);
Signature · 1 line
2.17.1
Tejun Heo
maintainerPATCH 4/6 blk-cgroup: remove now unused blkg_print_stat_{bytes|ios}_recursive()
In reply to Tejun HeoThese don't have users anymore. Remove them.
--- block/blk-cgroup.c | 83 -------------------------------------- include/linux/blk-cgroup.h | 5 --- 2 files changed, 88 deletions(-)
block/blk-cgroup.c +0 -83
@@ -615,89 +615,6 @@ u64 blkg_prfill_rwstat(struct seq_file *sf, struct blkg_policy_data *pd, }EXPORT_SYMBOL_GPL (blkg_prfill_rwstat); -static u64 blkg_prfill_rwstat_field (struct seq_file *sf, -struct blkg_policy_data *pd,int off) -{ -struct blkg_rwstat_sample rwstat = { }; - -blkg_rwstat_read ((void *)pd->blkg + off, &rwstat); -return __blkg_prfill_rwstat (sf, pd, &rwstat); -} - -/** -* blkg_print_stat_bytes - seq_show callback for blkg->stat_bytes -* @sf: seq_file to print to -* @v: unused -* -* To be used as cftype->seq_show to print blkg->stat_bytes. -* cftype->private must be set to the blkcg_policy. -*/ -int blkg_print_stat_bytes (struct seq_file *sf,void *v) -{ -blkcg_print_blkgs (sf,css_to_blkcg (seq_css (sf)), - blkg_prfill_rwstat_field, (void *)seq_cft (sf)->private, - offsetof(struct blkcg_gq , stat_bytes), true); -return 0 ; -} -EXPORT_SYMBOL_GPL (blkg_print_stat_bytes); - -/** -* blkg_print_stat_bytes - seq_show callback for blkg->stat_ios -* @sf: seq_file to print to -* @v: unused -* -* To be used as cftype->seq_show to print blkg->stat_ios. cftype->private -* must be set to the blkcg_policy. -*/ -int blkg_print_stat_ios (struct seq_file *sf,void *v) -{ -blkcg_print_blkgs (sf,css_to_blkcg (seq_css (sf)), - blkg_prfill_rwstat_field, (void *)seq_cft (sf)->private, - offsetof(struct blkcg_gq , stat_ios), true); -return 0 ; -} -EXPORT_SYMBOL_GPL (blkg_print_stat_ios); - -static u64 blkg_prfill_rwstat_field_recursive (struct seq_file *sf, -struct blkg_policy_data *pd, -int off) -{ -struct blkg_rwstat_sample rwstat; - -blkg_rwstat_recursive_sum (pd->blkg,NULL , off, &rwstat); -return __blkg_prfill_rwstat (sf, pd, &rwstat); -} - -/** -* blkg_print_stat_bytes_recursive - recursive version of blkg_print_stat_bytes -* @sf: seq_file to print to -* @v: unused -*/ -int blkg_print_stat_bytes_recursive (struct seq_file *sf,void *v) -{ -blkcg_print_blkgs (sf,css_to_blkcg (seq_css (sf)), - blkg_prfill_rwstat_field_recursive, - (void *)seq_cft (sf)->private, - offsetof(struct blkcg_gq , stat_bytes), true); -return 0 ; -} -EXPORT_SYMBOL_GPL (blkg_print_stat_bytes_recursive); - -/** -* blkg_print_stat_ios_recursive - recursive version of blkg_print_stat_ios -* @sf: seq_file to print to -* @v: unused -*/ -int blkg_print_stat_ios_recursive (struct seq_file *sf,void *v) -{ -blkcg_print_blkgs (sf,css_to_blkcg (seq_css (sf)), - blkg_prfill_rwstat_field_recursive, - (void *)seq_cft (sf)->private, - offsetof(struct blkcg_gq , stat_ios), true); -return 0 ; -} -EXPORT_SYMBOL_GPL (blkg_print_stat_ios_recursive); -/** * blkg_rwstat_recursive_sum - collect hierarchical blkg_rwstat * @blkg: blkg of interest
include/linux/blk-cgroup.h +0 -5
@@ -220,11 +220,6 @@ u64 __blkg_prfill_rwstat(struct seq_file *sf, struct blkg_policy_data *pd, const struct blkg_rwstat_sample *rwstat);u64 blkg_prfill_rwstat (struct seq_file *sf,struct blkg_policy_data *pd,int off); -int blkg_print_stat_bytes (struct seq_file *sf,void *v); -int blkg_print_stat_ios (struct seq_file *sf,void *v); -int blkg_print_stat_bytes_recursive (struct seq_file *sf,void *v); -int blkg_print_stat_ios_recursive (struct seq_file *sf,void *v); -void blkg_rwstat_recursive_sum (struct blkcg_gq *blkg,struct blkcg_policy *pol,int off,struct blkg_rwstat_sample *sum);
Signature · 1 line
2.17.1
Tejun Heo
maintainerPATCH 5/6 blk-cgroup: reimplement basic IO stats using cgroup rstat
In reply to Tejun Heoblk-cgroup has been using blkg_rwstat to track basic IO stats. Unfortunately, reading recursive stats scales badly as itinvolves walking all descendants. On systems with a huge number of cgroups (dead or alive), this can lead to substantial CPU cost when reading IO stats.
This patch reimplements basic IO stats using cgroup rstat which uses more memory but makes recursive stat reading O(# descendants which have been active since last reading) instead of O(# descendants).
* blk-cgroup core no longer uses sync/async stats. Introduce new stat enums - BLKG_IOSTAT_{READ|WRITE|DISCARD}.
* Add blkg_iostat[_set] which encapsulates byte and io stats, last values for propagation delta calculation and u64_stats_sync for correctness on 32bit archs.
* Update the new percpu stat counters directly and implement blkcg_rstat_flush() to implement propagation.
* blkg_print_stat() can now bring the stats up to date by calling cgroup_rstat_flush() and print them instead of directly summing up all descendants.
* It now allocates 96 bytes per cpu. It used to be 40 bytes.
--- block/blk-cgroup.c | 124 +++++++++++++++++++++++++++++-------- include/linux/blk-cgroup.h | 48 ++++++++++++-- 2 files changed, 142 insertions(+), 30 deletions(-)
block/blk-cgroup.c +99 -25
@@ -80,8 +80,7 @@ static void blkg_free(struct blkcg_gq *blkg)if (blkg->pd[i]) blkcg_policy[i]->pd_free_fn (blkg->pd[i]); -blkg_rwstat_exit (&blkg->stat_ios); -blkg_rwstat_exit (&blkg->stat_bytes); +free_percpu (blkg->iostat_cpu);percpu_ref_exit (&blkg->refcnt);kfree (blkg); }
@@ -146,7 +145,7 @@ static struct blkcg_gq *blkg_alloc(struct blkcg *blkcg, struct request_queue *q, gfp_t gfp_mask) {struct blkcg_gq *blkg; -int i; +int i, cpu;/* alloc and init base part */ blkg =kzalloc_node (sizeof (*blkg), gfp_mask, q->node);
@@ -156,8 +155,8 @@ static struct blkcg_gq *blkg_alloc(struct blkcg *blkcg, struct request_queue *q,if (percpu_ref_init (&blkg->refcnt, blkg_release,0 , gfp_mask)) goto err_free; -if (blkg_rwstat_init (&blkg->stat_bytes, gfp_mask) || -blkg_rwstat_init (&blkg->stat_ios, gfp_mask)) + blkg->iostat_cpu = alloc_percpu_gfp(struct blkg_iostat_set, gfp_mask); +if (!blkg->iostat_cpu) goto err_free; blkg->q = q;
@@ -167,6 +166,10 @@ static struct blkcg_gq *blkg_alloc(struct blkcg *blkcg, struct request_queue *q,INIT_WORK (&blkg->async_bio_work, blkg_async_bio_workfn); blkg->blkcg = blkcg; +u64_stats_init (&blkg->iostat.sync); +for_each_possible_cpu (cpu) +u64_stats_init (&per_cpu_ptr (blkg->iostat_cpu, cpu)->sync); +for (i =0 ; i <BLKCG_MAX_POLS ; i++) {struct blkcg_policy *pol = blkcg_policy[i];struct blkg_policy_data *pd;
@@ -393,7 +396,6 @@ struct blkcg_gq *blkg_lookup_create(struct blkcg *blkcg,static void blkg_destroy (struct blkcg_gq *blkg) {struct blkcg *blkcg = blkg->blkcg; -struct blkcg_gq *parent = blkg->parent;int i;lockdep_assert_held (&blkg->q->queue_lock);
@@ -410,11 +412,6 @@ static void blkg_destroy(struct blkcg_gq *blkg) pol->pd_offline_fn (blkg->pd[i]); } -if (parent) { -blkg_rwstat_add_aux (&parent->stat_bytes, &blkg->stat_bytes); -blkg_rwstat_add_aux (&parent->stat_ios, &blkg->stat_ios); - } - blkg->online = false;radix_tree_delete (&blkcg->blkg_tree, blkg->q->id);
@@ -464,7 +461,7 @@ static int blkcg_reset_stats(struct cgroup_subsys_state *css, {struct blkcg *blkcg =css_to_blkcg (css);struct blkcg_gq *blkg; -int i; +int i, cpu;mutex_lock (&blkcg_pol_mutex);spin_lock_irq (&blkcg->lock);
@@ -475,8 +472,12 @@ static int blkcg_reset_stats(struct cgroup_subsys_state *css,* anyway. If you get hit by a race, retry. */ hlist_for_each_entry (blkg, &blkcg->blkg_list, blkcg_node) { -blkg_rwstat_reset (&blkg->stat_bytes); -blkg_rwstat_reset (&blkg->stat_ios); +for_each_possible_cpu (cpu) { +struct blkg_iostat_set *bis = +per_cpu_ptr (blkg->iostat_cpu, cpu); +memset (bis,0 ,sizeof (*bis)); + } +memset (&blkg->iostat,0 ,sizeof (blkg->iostat));for (i =0 ; i <BLKCG_MAX_POLS ; i++) {struct blkcg_policy *pol = blkcg_policy[i];
@@ -840,16 +841,18 @@ static int blkcg_print_stat(struct seq_file *sf, void *v)struct blkcg *blkcg =css_to_blkcg (seq_css (sf));struct blkcg_gq *blkg; +cgroup_rstat_flush (blkcg->css.cgroup);rcu_read_lock ();hlist_for_each_entry_rcu (blkg, &blkcg->blkg_list, blkcg_node) { +struct blkg_iostat_set *bis = &blkg->iostat;const char *dname;char *buf; -struct blkg_rwstat_sample rwstat;u64 rbytes, wbytes, rios, wios, dbytes, dios;size_t size =seq_get_buf (sf, &buf), off =0 ;int i;bool has_stats = false; +unsigned seq;spin_lock_irq (&blkg->q->queue_lock);
@@ -868,17 +871,16 @@ static int blkcg_print_stat(struct seq_file *sf, void *v)*/ off +=scnprintf (buf+off, size-off,"%s " , dname); -blkg_rwstat_recursive_sum (blkg,NULL , - offsetof(struct blkcg_gq , stat_bytes), &rwstat); - rbytes = rwstat.cnt[BLKG_RWSTAT_READ ]; - wbytes = rwstat.cnt[BLKG_RWSTAT_WRITE ]; - dbytes = rwstat.cnt[BLKG_RWSTAT_DISCARD ]; +do { + seq =u64_stats_fetch_begin (&bis->sync); -blkg_rwstat_recursive_sum (blkg,NULL , - offsetof(struct blkcg_gq , stat_ios), &rwstat); - rios = rwstat.cnt[BLKG_RWSTAT_READ ]; - wios = rwstat.cnt[BLKG_RWSTAT_WRITE ]; - dios = rwstat.cnt[BLKG_RWSTAT_DISCARD ]; + rbytes = bis->cur.bytes[BLKG_IOSTAT_READ ]; + wbytes = bis->cur.bytes[BLKG_IOSTAT_WRITE ]; + dbytes = bis->cur.bytes[BLKG_IOSTAT_DISCARD ]; + rios = bis->cur.ios[BLKG_IOSTAT_READ ]; + wios = bis->cur.ios[BLKG_IOSTAT_WRITE ]; + dios = bis->cur.ios[BLKG_IOSTAT_DISCARD ]; + }while (u64_stats_fetch_retry (&bis->sync, seq));if (rbytes || wbytes || rios || wios) { has_stats = true;
@@ -1214,6 +1216,77 @@ static int blkcg_can_attach(struct cgroup_taskset *tset)return ret; } +static void blkg_iostat_set (struct blkg_iostat *dst,struct blkg_iostat *src) +{ +int i; + +for (i =0 ; i <BLKG_IOSTAT_NR ; i++) { + dst->bytes[i] = src->bytes[i]; + dst->ios[i] = src->ios[i]; + } +} + +static void blkg_iostat_add (struct blkg_iostat *dst,struct blkg_iostat *src) +{ +int i; + +for (i =0 ; i <BLKG_IOSTAT_NR ; i++) { + dst->bytes[i] += src->bytes[i]; + dst->ios[i] += src->ios[i]; + } +} + +static void blkg_iostat_sub (struct blkg_iostat *dst,struct blkg_iostat *src) +{ +int i; + +for (i =0 ; i <BLKG_IOSTAT_NR ; i++) { + dst->bytes[i] -= src->bytes[i]; + dst->ios[i] -= src->ios[i]; + } +} + +static void blkcg_rstat_flush (struct cgroup_subsys_state *css,int cpu) +{ +struct blkcg *blkcg =css_to_blkcg (css); +struct blkcg_gq *blkg; + +rcu_read_lock (); + +hlist_for_each_entry_rcu (blkg, &blkcg->blkg_list, blkcg_node) { +struct blkcg_gq *parent = blkg->parent; +struct blkg_iostat_set *bisc =per_cpu_ptr (blkg->iostat_cpu, cpu); +struct blkg_iostat cur, delta; +unsigned seq; + +/* fetch the current per-cpu values */ +do { + seq =u64_stats_fetch_begin (&bisc->sync); +blkg_iostat_set (&cur, &bisc->cur); + }while (u64_stats_fetch_retry (&bisc->sync, seq)); + +/* propagate percpu delta to global */ +u64_stats_update_begin (&blkg->iostat.sync); +blkg_iostat_set (&delta, &cur); +blkg_iostat_sub (&delta, &bisc->last); +blkg_iostat_add (&blkg->iostat.cur, &delta); +blkg_iostat_add (&bisc->last, &delta); +u64_stats_update_end (&blkg->iostat.sync); + +/* propagate global delta to parent */ +if (parent) { +u64_stats_update_begin (&parent->iostat.sync); +blkg_iostat_set (&delta, &blkg->iostat.cur); +blkg_iostat_sub (&delta, &blkg->iostat.last); +blkg_iostat_add (&parent->iostat.cur, &delta); +blkg_iostat_add (&blkg->iostat.last, &delta); +u64_stats_update_end (&parent->iostat.sync); + } + } + +rcu_read_unlock (); +} +static void blkcg_bind (struct cgroup_subsys_state *root_css) {int i;
@@ -1246,6 +1319,7 @@ struct cgroup_subsys io_cgrp_subsys = { .css_offline = blkcg_css_offline, .css_free = blkcg_css_free, .can_attach = blkcg_can_attach, + .css_rstat_flush = blkcg_rstat_flush, .bind = blkcg_bind, .dfl_cftypes = blkcg_files, .legacy_cftypes = blkcg_legacy_files,
include/linux/blk-cgroup.h +43 -5
85 lines not on this page · raw message
Signature · 1 line
2.17.1
Re: [PATCH 5/6] blk-cgroup: reimplement basic IO stats using cgroup rstat
Hi,
Quoted 24 lines · Tejun Heo
blk-cgroup has been using
blkg_rwstatto track basic IO stats. Unfortunately, reading recursive stats scales badly as itinvolves walking all descendants. On systems with a huge number of cgroups (dead or alive), this can lead to substantial CPU cost when reading IO stats.This patch reimplements basic IO stats using cgroup rstat which uses more memory but makes recursive stat reading O(# descendants which have been active since last reading) instead of O(# descendants).
* blk-cgroup core no longer uses sync/async stats. Introduce new stat enums - BLKG_IOSTAT_{READ|WRITE|DISCARD}.
* Add blkg_iostat[_set] which encapsulates byte and io stats, last values for propagation delta calculation and
u64_stats_syncfor correctness on 32bit archs.* Update the new percpu stat counters directly and implement
blkcg_rstat_flush()to implement propagation.*
blkg_print_stat()can now bring the stats up to date by callingcgroup_rstat_flush()and print them instead of directly summing up all descendants.
Quoted 6 lines
* It now allocates 96 bytes per cpu. It used to be 40 bytes.
Signed-off-by Tejun Heo <[email protected]>Cc Dan Schatzberg <[email protected]>Cc Daniel Xu <[email protected]>---
I bisected a Kernel OOPs issue to this patch on linux-next. Any idea why this is happening? Here is the log:
[ 32.033025] 8<--- cut here --- [ 32.036136] Unable to handle kernel paging request at virtual address 2e83803c [ 32.043637] pgd = 75330198 [ 32.046360] [2e83803c] *pgd=00000000 [ 32.050008] Internal error: Oops: 5 [#1] SMP ARM [ 32.054647] Modules linked in: [ 32.057724] CPU: 0 PID: 780 Comm: (systemd) Tainted: G W 5.4.0-rc7-next-20191113 #172 [ 32.066893] Hardware name: Generic AM33XX (Flattened Device Tree) [ 32.073026] PC is at cgroup_rstat_updated+0x30/0xe8 [ 32.077939] LR is at generic_make_request_checks+0x3d4/0x748 [ 32.083621] pc : [<c01e6f50>] lr : [<c04af820>] psr: a0040013 [ 32.089912] sp : ed9b3b78 ip : 2e838000 fp : ed826c00 [ 32.095156] r10: 00001000 r9 : 00000000 r8 : ff7ff428 [ 32.100402] r7 : c0d05148 r6 : c0d0554c r5 : c0c8b9ec r4 : edb26180 [ 32.106954] r3 : 2e838000 r2 : 2e838000 r1 : 00000000 r0 : eda32000 [ 32.113510] Flags: NzCv IRQs on FIQs on Mode SVC_32 ISA ARM Segment none [ 32.120674] Control: 10c5387d Table: adac0019 DAC: 00000051 [ 32.126444] Process (systemd) (pid: 780, stack limit = 0x5087843c) [ 32.132648] Stack: (0xed9b3b78 to 0xed9b4000) [ 32.137022] 3b60: edb26180 eee19550 [ 32.145237] 3b80: 2e838000 c0d05148 ff7ff428 c04af820 00000004 00000800 0074e7f8 00000000 [ 32.153452] 3ba0: a0040093 c08d1798 00000000 80040093 00002000 00000008 00000007 edb8168c [ 32.161667] 3bc0: 00000000 00000000 ffffe000 71b97da9 00000022 edb26180 c0d05148 00000008 [ 32.169882] 3be0: c0d05148 00000001 00000000 edb26180 00000000 c04b0ad8 00000000 00000000 [ 32.178097] 3c00: edb81a00 ed826c00 ed826cc4 71b97da9 c0de2c7c edb26180 c0d05148 00000008 [ 32.186312] 3c20: 00000001 00000001 00000000 0005fcfd 00000000 c04b0de0 c0de2c88 edb81600 [ 32.194526] 3c40: ed826800 0005fcfd 00000000 c04ce968 00001000 c0d05148 edb26180 efd29a84 [ 32.202741] 3c60: 00000000 00000000 0005fcfd 71b97da9 ed9b3c7b 00001000 00000001 00000001 [ 32.210956] 3c80: 00000001 00000001 00000000 0005fcfd 00000000 c039bdb0 20040013 00000001 [ 32.219170] 3ca0: 00000001 00000000 0005fcfd 00000000 ed9b3cc0 00000001 efd29a84 00000000 [ 32.227385] 3cc0: 00000000 ed9b3e04 edb26180 ec8421b0 00000001 ec842100 0000000c ec8422b8 [ 32.235600] 3ce0: 0005fcfd 00000000 00000fff 00000000 ee2a7b40 00080000 00000000 00112cca [ 32.243814] 3d00: ec8422bc c02983e0 0005fcfd 00000000 00000000 00000001 00000000 00000008 [ 32.252028] 3d20: 0005fcfd 00000000 00000000 eef82400 00000010 00000000 00000004 ed9b3e88 [ 32.260242] 3d40: 00000000 ed9b3d68 00000000 00000003 00000000 c0d05148 60040013 c01837f4 [ 32.268457] 3d60: 00000000 71b97da9 00000000 00000001 00000001 c03783bc ec8422b8 ed9b3e04 [ 32.276671] 3d80: ed9b3e04 00000001 ec8422bc c0378404 00000001 00000000 ec8421b0 c0255360 [ 32.284886] 3da0: eeee0000 ed9d2180 ed9b3da8 ed9b3da8 ed9b3db0 ed9b3db0 00000000 71b97da9 [ 32.293101] 3dc0: 00000000 00000001 00000001 00000000 00000003 ed9b3e04 00000000 00112cca [ 32.301316] 3de0: ec8422bc c025563c 00112cca 00000000 00000000 00000001 ec8422b8 ed9d2180 [ 32.309531] 3e00: ed9b3dfc ed9b3e04 ed9b3e04 71b97da9 ec8422b8 ed9d21e8 ed9d2180 ec8422b8 [ 32.317746] 3e20: 00000000 00000001 ffffffff 00000000 ed9d2180 c0255b8c 00000003 00000001 [ 32.325961] 3e40: ec8421b0 ed9b3f00 00000000 00000000 ec8422b8 c024b73c 00000001 beba6ca0 [ 32.334175] 3e60: c0d05148 00000000 00000000 beba6ca0 ed9b3ee8 ed9d2180 00000051 00000000 [ 32.342389] 3e80: ed9d21e8 00000001 ffffffff 00000fff 000081a4 00000001 000003e8 000003e8 [ 32.350604] 3ea0: 00000000 00000000 00000000 71b97da9 000000d2 ed9d2180 c0d05148 00000000 [ 32.358819] 3ec0: 00000000 ed9b3f78 00001000 00000000 00000000 c02bdb6c 00001000 00020000 [ 32.367033] 3ee0: 0058b9c8 00001000 00000004 00000000 00001000 ed9b3ee0 00000001 00000000 [ 32.375248] 3f00: ed9d2180 00000000 00000000 00000000 00000000 00000000 00000000 00000000 [ 32.383463] 3f20: 00000000 00000000 00000000 71b97da9 0058b9c8 00000001 00001000 ed9b3f78 [ 32.391678] 3f40: ed9d2180 00000000 00000000 c02bdc78 00000000 eda3ce1c eda3cc00 ed9d2180 [ 32.399893] 3f60: ed9d2180 c0d05148 0058b9c8 00001000 ed9b2000 c02bdf68 00000000 00000000 [ 32.408107] 3f80: 000005e8 71b97da9 005868d8 b6c02f41 000005e8 00000003 c0101204 00000003 [ 32.416322] 3fa0: 00000000 c01011e0 005868d8 b6c02f41 00000007 0058b9c8 00001000 00000000 [ 32.424537] 3fc0: 005868d8 b6c02f41 000005e8 00000003 0000000a beba6e88 00000000 00000000 [ 32.432753] 3fe0: 00000000 beba6d24 b6c037e1 b6c3e4b8 40040030 00000007 00000000 00000000 [ 32.440982] [<c01e6f50>] (cgroup_rstat_updated) from [<c04af820>] (generic_make_request_checks+0x3d4/0 x748) [ 32.450770] [<c04af820>] (generic_make_request_checks) from [<c04b0ad8>] (generic_make_request+0x1c/0x 2e4) [ 32.460468] [<c04b0ad8>] (generic_make_request) from [<c04b0de0>] (submit_bio+0x40/0x1b4) [ 32.468686] [<c04b0de0>] (submit_bio) from [<c039bdb0>] (ext4_mpage_readpages+0x704/0x904) [ 32.476995] [<c039bdb0>] (ext4_mpage_readpages) from [<c0378404>] (ext4_readpages+0x48/0x50) [ 32.485481] [<c0378404>] (ext4_readpages) from [<c0255360>] (read_pages+0x50/0x154) [ 32.493175] [<c0255360>] (read_pages) from [<c025563c>] (__do_page_cache_readahead+0x1d8/0x1f8) [ 32.501914] [<c025563c>] (__do_page_cache_readahead) from [<c0255b8c>] (page_cache_sync_readahead+0xa0 /0xf4) [ 32.511799] [<c0255b8c>] (page_cache_sync_readahead) from [<c024b73c>] (generic_file_read_iter+0x75c/0 xc40) [ 32.521594] [<c024b73c>] (generic_file_read_iter) from [<c02bdb6c>] (__vfs_read+0x138/0x1bc) [ 32.530073] [<c02bdb6c>] (__vfs_read) from [<c02bdc78>] (vfs_read+0x88/0x114) [ 32.537241] [<c02bdc78>] (vfs_read) from [<c02bdf68>] (ksys_read+0x54/0xd0) [ 32.544237] [<c02bdf68>] (ksys_read) from [<c01011e0>] (__sys_trace_return+0x0/0x20) [ 32.552010] Exception stack(0xed9b3fa8 to 0xed9b3ff0) [ 32.557085] 3fa0: 005868d8 b6c02f41 00000007 0058b9c8 00001000 00000000 [ 32.565300] 3fc0: 005868d8 b6c02f41 000005e8 00000003 0000000a beba6e88 00000000 00000000 [ 32.573512] 3fe0: 00000000 beba6d24 b6c037e1 b6c3e4b8 [ 32.578591] Code: ee073fba e7962101 e5903168 e0823003 (e593303c) [ 32.584889] ---[ end trace 08d6b7172e3ff29b ]--- [ 32.797983] 8<--- cut here --- [ 32.801090] Unable to handle kernel paging request at virtual address 2e83803c [ 32.808421] pgd = f285aa90 [ 32.811140] [2e83803c] *pgd=00000000 [ 32.814739] Internal error: Oops: 5 [#2] SMP ARM [ 32.819378] Modules linked in: [ 32.822453] CPU: 0 PID: 527 Comm: login Tainted: G D W 5.4.0-rc7-next-20191113 #172 [ 32.831273] Hardware name: Generic AM33XX (Flattened Device Tree) [ 32.837406] PC is at cgroup_rstat_updated+0x30/0xe8 [ 32.842320] LR is at generic_make_request_checks+0x3d4/0x748 [ 32.848002] pc : [<c01e6f50>] lr : [<c04af820>] psr: a0070013 [ 32.854292] sp : edbdfb78 ip : 2e838000 fp : eda49c00 [ 32.859537] r10: 00001000 r9 : 00000000 r8 : ff7fff60 [ 32.864782] r7 : c0d05148 r6 : c0d0554c r5 : c0c8b9ec r4 : edd8c6c0 [ 32.871335] r3 : 2e838000 r2 : 2e838000 r1 : 00000000 r0 : ed9dec00 [ 32.877891] Flags: NzCv IRQs on FIQs on Mode SVC_32 ISA ARM Segment none [ 32.885056] Control: 10c5387d Table: adb40019 DAC: 00000051 [ 32.890826] Process login (pid: 527, stack limit = 0x1deade48) [ 32.896681] Stack: (0xedbdfb78 to 0xedbe0000) [ 32.901056] fb60: edd8c6c0 eee19550 [ 32.909271] fb80: 2e838000 c0d05148 ff7fff60 c04af820 c0d0554c c01023dc 0074e7f8 00000000 [ 32.917487] fba0: 0000000a ffff979f 00400100 71b97da9 ee81ba00 ffffe000 00000000 c0d0554c [ 32.925702] fbc0: c0c90e7c 00000000 c0d0554c 71b97da9 00000001 edd8c6c0 c0d05148 00000008 [ 32.933916] fbe0: c0d05148 00000001 00000000 edd8c6c0 00000000 c04b0ad8 00000000 c0101aec [ 32.942130] fc00: 00000000 00000000 00001000 71b97da9 edd8c6c0 edd8c6c0 c0d05148 00000008 [ 32.950345] fc20: 00000001 00000001 00000000 0005fba9 00000000 c04b0de0 c04a8f24 c04a8340 [ 32.958560] fc40: 20070013 ffffffff 00000051 bf000000 00001000 c0d05148 edd8c6c0 efd47fac [ 32.966775] fc60: 00000000 00000000 0005fba9 71b97da9 edbdfc7b 00001000 00000001 00000001 [ 32.974990] fc80: 00000001 00000001 00000000 0005fba9 00000000 c039bdb0 20070013 00000001 [ 32.983204] fca0: 00000001 00000000 0005fba9 00000000 edbdfcc0 00000001 efd47fac 00000000 [ 32.991418] fcc0: 00000000 edbdfe04 edd8c6c0 ec85dd70 00000001 ec85dcc0 0000000c ec85de78 [ 32.999633] fce0: 0005fba9 00000000 00000fff 00000000 ee2a7b40 00080000 00000000 00112cca [ 33.007848] fd00: ec85de7c c02983e0 0005fba9 00000000 00000000 00000001 00000000 00000008 [ 33.016061] fd20: 0005fba9 00000000 00000000 eef82400 00000010 00000000 00000004 edbdfe88 [ 33.024276] fd40: 00000000 edbdfd68 00000000 00000003 00000000 c0d05148 60070013 c01837f4 [ 33.032491] fd60: 00000000 71b97da9 00000000 00000001 00000001 c03783bc ec85de78 edbdfe04 [ 33.040705] fd80: edbdfe04 00000001 ec85de7c c0378404 00000001 00000000 ec85dd70 c0255360 [ 33.048919] fda0: eeee0000 ed952d80 edbdfda8 edbdfda8 edbdfdb0 edbdfdb0 00000000 71b97da9 [ 33.057134] fdc0: 00000000 00000001 00000001 00000000 00000003 edbdfe04 00000000 00112cca [ 33.065348] fde0: ec85de7c c025563c 00112cca 00000000 00000000 00000001 ec85de78 ed952d80 [ 33.073563] fe00: edbdfdfc edbdfe04 edbdfe04 71b97da9 ec85de78 ed952de8 ed952d80 ec85de78 [ 33.081777] fe20: 00000000 00000001 ffffffff 00000000 ed952d80 c0255b8c 00000003 00000001 [ 33.089992] fe40: ec85dd70 edbdff00 00000000 00000000 ec85de78 c024b73c 00000001 00000041 [ 33.098206] fe60: ffffe000 00000000 00000000 00000000 edbdfee8 ed952d80 00000000 00000000 [ 33.106422] fe80: ed952de8 00000001 ffffffff 00000fff edbdfe8c 71b97da9 000003e8 00000004 [ 33.114637] fea0: edbdff70 c0d05148 00000001 71b97da9 edbde000 ed952d80 c0d05148 00000000 [ 33.122852] fec0: 00000000 edbdff78 000003e8 00000000 00000000 c02bdb6c 000003e8 00020000 [ 33.131066] fee0: 000365a8 000003e8 00000004 00000000 000003e8 edbdfee0 00000001 00000000 [ 33.139280] ff00: ed952d80 00000000 00000000 00000000 00000000 00000000 00000000 00000000 [ 33.147495] ff20: 00000000 00000000 00000000 71b97da9 000365a8 00000001 000003e8 edbdff78 [ 33.155709] ff40: ed952d80 00000000 00000000 c02bdc78 00000000 edd7721c edd77000 ed952d80 [ 33.163923] ff60: ed952d80 c0d05148 000365a8 000003e8 edbde000 c02bdf68 00000000 00000000 [ 33.172137] ff80: 00000000 71b97da9 000003e8 be9bb7ac 00000000 00000003 c0101204 00000003 [ 33.180353] ffa0: 00000000 c01011e0 000003e8 be9bb7ac 00000004 000365a8 000003e8 00000000 [ 33.188567] ffc0: 000003e8 be9bb7ac 00000000 00000003 00000004 000365a8 b6d74d64 00000000 [ 33.196782] ffe0: 00000000 be9bb704 b6f7516c b6ef64b8 60070030 00000004 00000000 00000000 [ 33.205010] [<c01e6f50>] (cgroup_rstat_updated) from [<c04af820>] (generic_make_request_checks+0x3d4/0 x748) [ 33.214800] [<c04af820>] (generic_make_request_checks) from [<c04b0ad8>] (generic_make_request+0x1c/0x 2e4) [ 33.224495] [<c04b0ad8>] (generic_make_request) from [<c04b0de0>] (submit_bio+0x40/0x1b4) [ 33.232714] [<c04b0de0>] (submit_bio) from [<c039bdb0>] (ext4_mpage_readpages+0x704/0x904) [ 33.241023] [<c039bdb0>] (ext4_mpage_readpages) from [<c0378404>] (ext4_readpages+0x48/0x50) [ 33.249509] [<c0378404>] (ext4_readpages) from [<c0255360>] (read_pages+0x50/0x154) [ 33.257203] [<c0255360>] (read_pages) from [<c025563c>] (__do_page_cache_readahead+0x1d8/0x1f8) [ 33.265943] [<c025563c>] (__do_page_cache_readahead) from [<c0255b8c>] (page_cache_sync_readahead+0xa0 /0xf4) [ 33.275826] [<c0255b8c>] (page_cache_sync_readahead) from [<c024b73c>] (generic_file_read_iter+0x75c/0 xc40) [ 33.285621] [<c024b73c>] (generic_file_read_iter) from [<c02bdb6c>] (__vfs_read+0x138/0x1bc) [ 33.294099] [<c02bdb6c>] (__vfs_read) from [<c02bdc78>] (vfs_read+0x88/0x114) [ 33.301268] [<c02bdc78>] (vfs_read) from [<c02bdf68>] (ksys_read+0x54/0xd0) [ 33.308264] [<c02bdf68>] (ksys_read) from [<c01011e0>] (__sys_trace_return+0x0/0x20) [ 33.316038] Exception stack(0xedbdffa8 to 0xedbdfff0) [ 33.321112] ffa0: 000003e8 be9bb7ac 00000004 000365a8 000003e8 00000000 [ 33.329327] ffc0: 000003e8 be9bb7ac 00000000 00000003 00000004 000365a8 b6d74d64 00000000 [ 33.337540] ffe0: 00000000 be9bb704 b6f7516c b6ef64b8 [ 33.342619] Code: ee073fba e7962101 e5903168 e0823003 (e593303c) [ 33.348850] ---[ end trace 08d6b7172e3ff29c ]---
Thanks,
Faiz
Tejun Heo
maintainerRe: [PATCH 5/6] blk-cgroup: reimplement basic IO stats using cgroup rstat
Hello,
Can you please see whether the following patch fixes the issue?
Thanks.
include/linux/blk-cgroup.h +2 -1
@@ -626,7 +626,8 @@ static inline bool blkcg_bio_issue_check(struct request_queue *q, bis->cur.ios[rwd]++;u64_stats_update_end (&bis->sync); -cgroup_rstat_updated (blkg->blkcg->css.cgroup, cpu); +if (cgroup_subsys_on_dfl (io_cgrp_subsys)) +cgroup_rstat_updated (blkg->blkcg->css.cgroup, cpu);put_cpu (); }
Re: [PATCH 5/6] blk-cgroup: reimplement basic IO stats using cgroup rstat
Hi Tejun,
Quoted 4 lines · Tejun Heo
Hello,
Can you please see whether the following patch fixes the issue?
This patch does fix the issue for me.
Thanks,
Ionela.
Quoted 17 lines
Thanks.
include/linux/blk-cgroup.h +2
-1@@ -626,7 +626,8 @@ static inline bool blkcg_bio_issue_check(struct request_queue *q, bis->cur.ios[rwd]++;u64_stats_update_end (&bis->sync); -cgroup_rstat_updated (blkg->blkcg->css.cgroup, cpu); +if (cgroup_subsys_on_dfl (io_cgrp_subsys)) +cgroup_rstat_updated (blkg->blkcg->css.cgroup, cpu);put_cpu (); }
Tejun Heo
maintainerPATCH 1/1 blk-cgroup: cgroup_rstat_updated() shouldn't be called on cgroup1
In reply to Tejun HeoCurrently, cgroup rstat is supported only on cgroup2 hierarchy and rstat functions shouldn't be called on cgroup1 cgroups. While converting blk-cgroup core statistics to rstat, f73316482977 ("blk-cgroup: reimplement basic IO stats using cgroup rstat") accidentally ended up calling cgroup_rstat_updated() on cgroup1 cgroups causing crashes.
Longer term, we probably should add cgroup1 support to rstat but for now let's mask the call directly.
--- include/linux/blk-cgroup.h | 3 ++- 1 file changed, 2 insertions(+), 1 deletion(-)
include/linux/blk-cgroup.h +2 -1
@@ -626,7 +626,8 @@ static inline bool blkcg_bio_issue_check(struct request_queue *q, bis->cur.ios[rwd]++;u64_stats_update_end (&bis->sync); -cgroup_rstat_updated (blkg->blkcg->css.cgroup, cpu); +if (cgroup_subsys_on_dfl (io_cgrp_subsys)) +cgroup_rstat_updated (blkg->blkcg->css.cgroup, cpu);put_cpu (); }
Re: [PATCH block/for-next] blk-cgroup: cgroup_rstat_updated() shouldn't be called on cgroup1
Hi,
Quoted 7 lines · Tejun Heo
Currently, cgroup rstat is supported only on cgroup2 hierarchy and rstat functions shouldn't be called on cgroup1 cgroups. While converting blk-cgroup core statistics to rstat,
f73316482977("blk-cgroup: reimplement basic IO stats using cgroup rstat") accidentally ended up callingcgroup_rstat_updated()on cgroup1 cgroups causing crashes.
Quoted 5 lines
Longer term, we probably should add cgroup1 support to rstat but for now let's mask the call directly.
Signed-off-by Tejun Heo <[email protected]>Fixes f73316482977 ("blk-cgroup: reimplement basic IO stats using cgroup rstat")
Quoted 10 lines
--- include/linux/blk-cgroup.h | 3 ++- 1 file changed, 2 insertions(+), 1 deletion(-)include/linux/blk-cgroup.h +0
-0
include/linux/blk-cgroup.h static inline bool blkcg_bio_issue_check(struct request_queue *q, · line 633
Quoted 9 lines
include/linux/blk-cgroup.h +2
-1@@ -626,7 +626,8 @@ static inline bool blkcg_bio_issue_check(struct request_queue *q, bis->cur.ios[rwd]++;u64_stats_update_end (&bis->sync); -cgroup_rstat_updated (blkg->blkcg->css.cgroup, cpu); +if (cgroup_subsys_on_dfl (io_cgrp_subsys)) +cgroup_rstat_updated (blkg->blkcg->css.cgroup, cpu);put_cpu (); }
Thanks,
Faiz
Re: [PATCH block/for-next] blk-cgroup: cgroup_rstat_updated() shouldn't be called on cgroup1
In reply to Tejun HeoQuoted 9 lines · Tejun Heo
Currently, cgroup rstat is supported only on cgroup2 hierarchy and rstat functions shouldn't be called on cgroup1 cgroups. While converting blk-cgroup core statistics to rstat,
f73316482977("blk-cgroup: reimplement basic IO stats using cgroup rstat") accidentally ended up callingcgroup_rstat_updated()on cgroup1 cgroups causing crashes.Longer term, we probably should add cgroup1 support to rstat but for now let's mask the call directly.
Applied, thanks.
Signature · 1 line
Jens Axboe
Tejun Heo
maintainerPATCH 6/6 blk-cgroup: separate out blkg_rwstat under CONFIG_BLK_CGROUP_RWSTAT
In reply to Tejun Heoblkg_rwstat is now only used by bfq-iosched and blk-throtl when on cgroup1. Let's move it into its own files and gate it behind a config option.
--- block/Kconfig | 4 + block/Kconfig.iosched | 1 + block/Makefile | 1 + block/bfq-iosched.h | 2 + block/blk-cgroup-rwstat.c | 129 ++++++++++++++++++++++++++++++ block/blk-cgroup-rwstat.h | 149 ++++++++++++++++++++++++++++++++++ block/blk-cgroup.c | 97 ---------------------- block/blk-throttle.c | 1 + include/linux/blk-cgroup.h | 159 ------------------------------------- 9 files changed, 287 insertions(+), 256 deletions(-) create mode 100644 block/blk-cgroup-rwstat.c create mode 100644 block/blk-cgroup-rwstat.h
block/Kconfig +4 -0
@@ -32,6 +32,9 @@ config BLK_RQ_ALLOC_TIME config BLK_SCSI_REQUEST bool +config BLK_CGROUP_RWSTAT + bool + config BLK_DEV_BSG bool "Block layer SG support v4" default y
@@ -86,6 +89,7 @@ config BLK_DEV_ZONED config BLK_DEV_THROTTLING bool "Block layer bio throttling support" depends on BLK_CGROUP=y + select BLK_CGROUP_RWSTAT ---help--- Block layer bio throttling support. It can be used to limit the IO rate to a device. IO rate policies are per cgroup and
block/Kconfig.iosched +1 -0
@@ -31,6 +31,7 @@ config IOSCHED_BFQ config BFQ_GROUP_IOSCHED bool "BFQ hierarchical scheduling support" depends on IOSCHED_BFQ && BLK_CGROUP + select BLK_CGROUP_RWSTAT ---help--- Enable hierarchical scheduling in BFQ, using the blkio
block/Makefile +1 -0
@@ -16,6 +16,7 @@ obj-$(CONFIG_BLK_SCSI_REQUEST) += scsi_ioctl.o obj-$(CONFIG_BLK_DEV_BSG) += bsg.o obj-$(CONFIG_BLK_DEV_BSGLIB) += bsg-lib.o obj-$(CONFIG_BLK_CGROUP) += blk-cgroup.o +obj-$(CONFIG_BLK_CGROUP_RWSTAT) += blk-cgroup-rwstat.o obj-$(CONFIG_BLK_DEV_THROTTLING) += blk-throttle.o obj-$(CONFIG_BLK_CGROUP_IOLATENCY) += blk-iolatency.o obj-$(CONFIG_BLK_CGROUP_IOCOST) += blk-iocost.o
block/bfq-iosched.h +2 -0
@@ -10,6 +10,8 @@#include <linux/hrtimer.h> #include <linux/blk-cgroup.h> +#include "blk-cgroup-rwstat.h" +#define BFQ_IOPRIO_CLASSES 3#define BFQ_CL_IDLE_TIMEOUT (HZ/5)
block/blk-cgroup-rwstat.c new file +129 -0
129 lines not on this page · raw message
block/blk-cgroup-rwstat.h new file +149 -0
149 lines not on this page · raw message
block/blk-cgroup.c +0 -97
103 lines not on this page · raw message
block/blk-throttle.c +1 -0
@@ -12,6 +12,7 @@#include <linux/blktrace_api.h> #include <linux/blk-cgroup.h> #include "blk.h" +#include "blk-cgroup-rwstat.h" /* Max dispatch from a group in 1 round */ static int throtl_grp_quantum =8 ;
include/linux/blk-cgroup.h +0 -159
189 lines not on this page · raw message
Signature · 1 line
2.17.1
Quoted 29 lines · Tejun Heo
Hello,
v2: Build fix when !DEBUG.
blk-cgroup IO stats currently use
blkg_rwstatwhich unforutnately requires walking all descendants recursively on read. On systems with a large number of cgroups (dead or alive), this can make each stat read a substantially expensive operation.This patch updates blk-cgroup to use cgroup rstat which makes stat reading O(# descendants which have been active since last reading) instead of O(# descendants).
0001-bfq-iosched-relocate-bfqg_-rwstat-helpers.patch 0002-bfq-iosched-stop-using-blkg-stat_bytes-and-stat_ios.patch 0003-blk-throtl-stop-using-blkg-stat_bytes-and-stat_ios.patch 0004-blk-cgroup-remove-now-unused-blkg_print_stat_-bytes-.patch 0005-blk-cgroup-reimplement-basic-IO-stats-using-cgroup-r.patch 0006-blk-cgroup-separate-out-blkg_rwstat-under-CONFIG_BLK.patch
0001-0003 make bfq-iosched and blk-throtl use their own
blkg_rwstatto track basic IO stats on cgroup1 instead of sharing blk-cgroup core's. 0004 is a follow-up cleanup.0005 switches blk-cgroup to cgroup rstat.
0006 moves
blkg_rwstatto its own files and gate it under a config option as it's now only used by blk-throtl and bfq-iosched.
Quoted 8 lines
The patchset is on top of
block/for-next 40afbe18b03a ("Merge branch 'for-5.5/drivers-post' into for-next")+
block/for-linusb0814361a25c("blkcg: makeblkcg_print_stat()print stats only for online blkgs")and also available in the following git branch.
git://git.kernel.org/pub/scm/linux/kernel/git/tj/cgroup.gitreview-blkcg-rstat
Thanks, applied!
Signature · 1 line
Jens Axboe