blk-cgroup: Replace u64 sync with spinlock for iostat update
Subsystems (4)
PATCH v3 1/1 blk-cgroup: Replace u64 sync with spinlock for iostat update
From: Boy Wu <[email protected]>
In 32bit SMP systems, if multiple CPUs call blkcg_print_stat, it may cause blkcg_fill_root_iostats to have a concurrent problem on the seqlock in u64_stats_update, which will cause a deadlock on u64_stats_fetch_begin in blkcg_print_one_stat.
Thus, replace u64 sync with spinlock to protect iostat update.
---
Change in v2:
- update commit message
- Remove u64_sync
- Replace spin_lock_irq with guard statement
- Replace blkg->q->queue_lock with blkg_stat_lock
Change in v3:
- update commit message
- Add spinlock in blkg_iostat_set structure
- Replace all u64_sync with spinlock for iostat
- Replace blkg_stat_lock with iostat.spinlock
--- block/blk-cgroup.c | 62 +++++++++++++++++++--------------------------- block/blk-cgroup.h | 1 + 2 files changed, 26 insertions(+), 37 deletions(-)
block/blk-cgroup.c +25 -37
@@ -329,7 +329,7 @@ static struct blkcg_gq *blkg_alloc(struct blkcg *blkcg, struct gendisk *disk,INIT_WORK (&blkg->async_bio_work, blkg_async_bio_workfn);#endif -u64_stats_init (&blkg->iostat.sync); +spin_lock_init (&blkg->iostat.spinlock);for_each_possible_cpu (cpu) {u64_stats_init (&per_cpu_ptr (blkg->iostat_cpu, cpu)->sync);per_cpu_ptr (blkg->iostat_cpu, cpu)->blkg = blkg;
@@ -995,15 +995,13 @@ static void blkcg_iostat_update(struct blkcg_gq *blkg, struct blkg_iostat *cur, struct blkg_iostat *last) {struct blkg_iostat delta; -unsigned long flags;/* propagate percpu delta to global */ - flags =u64_stats_update_begin_irqsave (&blkg->iostat.sync); +guard (spinlock_irqsave)(&blkg->iostat.spinlock);blkg_iostat_set (&delta, cur);blkg_iostat_sub (&delta, last);blkg_iostat_add (&blkg->iostat.cur, &delta);blkg_iostat_add (last, &delta); -u64_stats_update_end_irqrestore (&blkg->iostat.sync, flags); }static void __blkcg_rstat_flush (struct blkcg *blkcg,int cpu)
@@ -1034,7 +1032,6 @@ static void __blkcg_rstat_flush(struct blkcg *blkcg, int cpu)struct blkcg_gq *blkg = bisc->blkg;struct blkcg_gq *parent = blkg->parent;struct blkg_iostat cur; -unsigned int seq;/* * Order assignment of `next_bisc` from `bisc->lnode.next` in
@@ -1051,10 +1048,8 @@ static void __blkcg_rstat_flush(struct blkcg *blkcg, int cpu) goto propagate_up;/* propagate up to parent only */ /* fetch the current per-cpu values */ -do { - seq =u64_stats_fetch_begin (&bisc->sync); + scoped_guard(spinlock_irqsave, &bisc->spinlock)blkg_iostat_set (&cur, &bisc->cur); - }while (u64_stats_fetch_retry (&bisc->sync, seq));blkcg_iostat_update (blkg, &cur, &bisc->last);
@@ -1112,7 +1107,6 @@ static void blkcg_fill_root_iostats(void)struct blkcg_gq *blkg = bdev->bd_disk->queue->root_blkg;struct blkg_iostat tmp;int cpu; -unsigned long flags;memset (&tmp,0 ,sizeof (tmp));for_each_possible_cpu (cpu) {
@@ -1134,9 +1128,8 @@ static void blkcg_fill_root_iostats(void) cpu_dkstats->sectors[STAT_DISCARD ] <<9 ; } - flags =u64_stats_update_begin_irqsave (&blkg->iostat.sync); +guard (spinlock_irqsave)(&blkg->iostat.spinlock);blkg_iostat_set (&blkg->iostat.cur, &tmp); -u64_stats_update_end_irqrestore (&blkg->iostat.sync, flags); } }
@@ -1145,7 +1138,6 @@ static void blkcg_print_one_stat(struct blkcg_gq *blkg, struct seq_file *s)struct blkg_iostat_set *bis = &blkg->iostat;u64 rbytes, wbytes, rios, wios, dbytes, dios;const char *dname; -unsigned seq;int i;if (!blkg->online)
@@ -1157,16 +1149,14 @@ static void blkcg_print_one_stat(struct blkcg_gq *blkg, struct seq_file *s)seq_printf (s,"%s " , dname); -do { - seq =u64_stats_fetch_begin (&bis->sync); - + scoped_guard(spinlock_irqsave, &bis->spinlock) { rbytes = bis->cur.bytes[BLKG_IOSTAT_READ ]; wbytes = bis->cur.bytes[BLKG_IOSTAT_WRITE ]; dbytes = bis->cur.bytes[BLKG_IOSTAT_DISCARD ]; rios = bis->cur.ios[BLKG_IOSTAT_READ ]; wios = bis->cur.ios[BLKG_IOSTAT_WRITE ]; dios = bis->cur.ios[BLKG_IOSTAT_DISCARD ]; - }while (u64_stats_fetch_retry (&bis->sync, seq)); + }if (rbytes || wbytes || rios || wios) { seq_printf(s, "rbytes=%llu wbytes=%llu rios=%llu wios=%llu dbytes=%llu dios=%llu",
@@ -2141,7 +2131,6 @@ void blk_cgroup_bio_start(struct bio *bio)struct blkcg *blkcg = bio->bi_blkg->blkcg;int rwd =blk_cgroup_io_type (bio), cpu;struct blkg_iostat_set *bis; -unsigned long flags;if (!cgroup_subsys_on_dfl (io_cgrp_subsys))return ;
@@ -2152,30 +2141,29 @@ void blk_cgroup_bio_start(struct bio *bio) cpu =get_cpu (); bis =per_cpu_ptr (bio->bi_blkg->iostat_cpu, cpu); - flags =u64_stats_update_begin_irqsave (&bis->sync); - -/* -* If the bio is flagged with BIO_CGROUP_ACCT it means this is a split -* bio and we would have already accounted for the size of the bio. -*/ -if (!bio_flagged (bio,BIO_CGROUP_ACCT )) { -bio_set_flag (bio,BIO_CGROUP_ACCT ); - bis->cur.bytes[rwd] += bio->bi_iter.bi_size; - } - bis->cur.ios[rwd]++; + scoped_guard(spinlock_irqsave, &bis->spinlock) { +/* +* If the bio is flagged with BIO_CGROUP_ACCT it means this is a split +* bio and we would have already accounted for the size of the bio. +*/ +if (!bio_flagged (bio,BIO_CGROUP_ACCT )) { +bio_set_flag (bio,BIO_CGROUP_ACCT ); + bis->cur.bytes[rwd] += bio->bi_iter.bi_size; + } + bis->cur.ios[rwd]++; -/* -* If the iostat_cpu isn't in a lockless list, put it into the -* list to indicate that a stat update is pending. -*/ -if (!READ_ONCE (bis->lqueued)) { -struct llist_head *lhead =this_cpu_ptr (blkcg->lhead); +/* +* If the iostat_cpu isn't in a lockless list, put it into the +* list to indicate that a stat update is pending. +*/ +if (!READ_ONCE (bis->lqueued)) { +struct llist_head *lhead =this_cpu_ptr (blkcg->lhead); -llist_add (&bis->lnode, lhead); -WRITE_ONCE (bis->lqueued, true); +llist_add (&bis->lnode, lhead); +WRITE_ONCE (bis->lqueued, true); + } } -u64_stats_update_end_irqrestore (&bis->sync, flags);cgroup_rstat_updated (blkcg->css.cgroup, cpu);put_cpu (); }
block/blk-cgroup.h +1 -0
@@ -44,6 +44,7 @@ struct blkg_iostat { };struct blkg_iostat_set { +spinlock_t spinlock;struct u64_stats_sync sync;struct blkcg_gq *blkg;struct llist_node lnode;
Signature · 1 line
2.18.0
Tejun Heo
maintainerHello, Boy.
So, looking at the patch, I'm not sure per-blkg lock makes sense.
Quoted 5 lines · boy.wu
block/blk-cgroup.c +0
-1@@ -995,15 +995,13 @@ static void blkcg_iostat_update(struct blkcg_gq *blkg, struct blkg_iostat *cur, struct blkg_iostat *last) {struct blkg_iostat delta; -unsigned long flags;
block/blk-cgroup.c static void blkcg_iostat_update(struct blkcg_gq *blkg, struct blkg_iostat *cur, · line 1005
Quoted 10 lines
block/blk-cgroup.c +1
-2/* propagate percpu delta to global */ - flags =u64_stats_update_begin_irqsave (&blkg->iostat.sync); +guard (spinlock_irqsave)(&blkg->iostat.spinlock);blkg_iostat_set (&delta, cur);blkg_iostat_sub (&delta, last);blkg_iostat_add (&blkg->iostat.cur, &delta);blkg_iostat_add (last, &delta); -u64_stats_update_end_irqrestore (&blkg->iostat.sync, flags); }
This is already called with blkg_stat_lock held.
block/blk-cgroup.c static void __blkcg_rstat_flush(struct blkcg *blkcg, int cpu) · line 1057
Quoted 9 lines
block/blk-cgroup.c +1
-3@@ -1051,10 +1048,8 @@ static void __blkcg_rstat_flush(struct blkcg *blkcg, int cpu) goto propagate_up;/* propagate up to parent only */ /* fetch the current per-cpu values */ -do { - seq =u64_stats_fetch_begin (&bisc->sync); + scoped_guard(spinlock_irqsave, &bisc->spinlock)blkg_iostat_set (&cur, &bisc->cur); - }while (u64_stats_fetch_retry (&bisc->sync, seq));
This is per-cpu stat and we should keep using u64_sync for them.
Quoted 10 lines
block/blk-cgroup.c +1
-2@@ -1134,9 +1128,8 @@ static void blkcg_fill_root_iostats(void) cpu_dkstats->sectors[STAT_DISCARD ] <<9 ; } - flags =u64_stats_update_begin_irqsave (&blkg->iostat.sync); +guard (spinlock_irqsave)(&blkg->iostat.spinlock);blkg_iostat_set (&blkg->iostat.cur, &tmp); -u64_stats_update_end_irqrestore (&blkg->iostat.sync, flags); } }
...
Quoted 6 lines
block/blk-cgroup.c +0
-2@@ -1157,16 +1149,14 @@ static void blkcg_print_one_stat(struct blkcg_gq *blkg, struct seq_file *s)seq_printf (s,"%s " , dname); - do { - seq = u64_stats_fetch_begin(&bis->sync);
block/blk-cgroup.c static void blkcg_print_one_stat(struct blkcg_gq *blkg, struct seq_file *s) · line 1159
Quoted 10 lines
block/blk-cgroup.c +2
-2- + scoped_guard(spinlock_irqsave, &bis->spinlock) { rbytes = bis->cur.bytes[BLKG_IOSTAT_READ ]; wbytes = bis->cur.bytes[BLKG_IOSTAT_WRITE ]; dbytes = bis->cur.bytes[BLKG_IOSTAT_DISCARD ]; rios = bis->cur.ios[BLKG_IOSTAT_READ ]; wios = bis->cur.ios[BLKG_IOSTAT_WRITE ]; dios = bis->cur.ios[BLKG_IOSTAT_DISCARD ]; - }while (u64_stats_fetch_retry (&bis->sync, seq)); + }
The above two are the only places which can potentially benefit from per-blkg lock but these aren't hot paths. I'd just use blkg_stat_lock for the above.
Quoted 34 lines
block/blk-cgroup.c +12
-17@@ -2152,30 +2141,29 @@ void blk_cgroup_bio_start(struct bio *bio) cpu =get_cpu (); bis =per_cpu_ptr (bio->bi_blkg->iostat_cpu, cpu); - flags =u64_stats_update_begin_irqsave (&bis->sync); - -/* -* If the bio is flagged with BIO_CGROUP_ACCT it means this is a split -* bio and we would have already accounted for the size of the bio. -*/ -if (!bio_flagged (bio,BIO_CGROUP_ACCT )) { -bio_set_flag (bio,BIO_CGROUP_ACCT ); - bis->cur.bytes[rwd] += bio->bi_iter.bi_size; - } - bis->cur.ios[rwd]++; + scoped_guard(spinlock_irqsave, &bis->spinlock) { +/* +* If the bio is flagged with BIO_CGROUP_ACCT it means this is a split +* bio and we would have already accounted for the size of the bio. +*/ +if (!bio_flagged (bio,BIO_CGROUP_ACCT )) { +bio_set_flag (bio,BIO_CGROUP_ACCT ); + bis->cur.bytes[rwd] += bio->bi_iter.bi_size; + } + bis->cur.ios[rwd]++; -/* -* If the iostat_cpu isn't in a lockless list, put it into the -* list to indicate that a stat update is pending. -*/ -if (!READ_ONCE (bis->lqueued)) { -struct llist_head *lhead =this_cpu_ptr (blkcg->lhead); +/* +* If the iostat_cpu isn't in a lockless list, put it into the
block/blk-cgroup.c void blk_cgroup_bio_start(struct bio *bio) · line 2164
Quoted 10 lines
block/blk-cgroup.c +7
-2+* list to indicate that a stat update is pending. +*/ +if (!READ_ONCE (bis->lqueued)) { +struct llist_head *lhead =this_cpu_ptr (blkcg->lhead); -llist_add (&bis->lnode, lhead); -WRITE_ONCE (bis->lqueued, true); +llist_add (&bis->lnode, lhead); +WRITE_ONCE (bis->lqueued, true); + }
These are per-cpu stat updates which should keep using u64_sync. We don't want to incur locking overhead for stat updates in the hot issue path.
Thanks.
Signature · 1 line
tejun
Quoted 15 lines · Tejun Heo
External email : Please do not click links or open attachments until you have verified the sender or the content.
Hello, Boy.So, looking at the patch, I'm not sure per-blkg lock makes sense.
Review static void blkcg_iostat_update(struct
Quoted 1 line · boy.wu
(quoted diff) +0
-0@@ -995,15 +995,13 @@ static void blkcg_iostat_update(struct
blkcg_gq*blkg, structblkg_iostat*cur,Quoted 5 lines
block/blk-cgroup.c +0
-1struct blkg_iostat *last) {struct blkg_iostat delta; -unsigned long flags;
Quoted 11 lines
block/blk-cgroup.c static void blkcg_iostat_update(struct blkcg_gq *blkg, struct blkg_iostat *cur, · line 1005
Quoted 9 lines
block/blk-cgroup.c +1
-2/* propagate percpu delta to global */ -flags =u64_stats_update_begin_irqsave (&blkg->iostat.sync); +guard (spinlock_irqsave)(&blkg->iostat.spinlock);blkg_iostat_set (&delta, cur);blkg_iostat_sub (&delta, last);blkg_iostat_add (&blkg->iostat.cur, &delta);blkg_iostat_add (last, &delta); -u64_stats_update_end_irqrestore (&blkg->iostat.sync, flags); }This is already called with
blkg_stat_lockheld.
In blkcg_iostat_update, it is protected by both blkg_stat_lock and u64 sync. I agree that no changes are needed here.
Quoted 39 lines
Review static void __blkcg_rstat_flush(struct blkcg
Quoted 2 lines
(quoted diff) +0
-0@@ -1051,10 +1048,8 @@ static void __blkcg_rstat_flush(struct blkcg*blkcg, int cpu)
block/blk-cgroup.c static void __blkcg_rstat_flush(struct blkcg *blkcg, int cpu) · line 1057
Quoted 8 lines
block/blk-cgroup.c +1
-3goto propagate_up;/* propagate up to parent only */ /* fetch the current per-cpu values */ -do { -seq =u64_stats_fetch_begin (&bisc->sync); +scoped_guard(spinlock_irqsave, &bisc->spinlock)blkg_iostat_set (&cur, &bisc->cur); -}while (u64_stats_fetch_retry (&bisc->sync, seq));This is per-cpu stat and we should keep using
u64_syncfor them.Quoted 10 lines
block/blk-cgroup.c +1
-2@@ -1134,9 +1128,8 @@ static void blkcg_fill_root_iostats(void) cpu_dkstats->sectors[STAT_DISCARD ] <<9 ; } -flags =u64_stats_update_begin_irqsave (&blkg->iostat.sync); +guard (spinlock_irqsave)(&blkg->iostat.spinlock);blkg_iostat_set (&blkg->iostat.cur, &tmp); -u64_stats_update_end_irqrestore (&blkg->iostat.sync, flags); } }...
Review static void blkcg_print_one_stat(struct
Quoted 1 line
(quoted diff) +0
-0@@ -1157,16 +1149,14 @@ static void blkcg_print_one_stat(struct
blkcg_gq*blkg, structseq_file*s)Quoted 12 lines
block/blk-cgroup.c +1
-3seq_printf (s,"%s " , dname); -do { -seq = u64_stats_fetch_begin(&bis->sync); - +scoped_guard(spinlock_irqsave, &bis->spinlock) { rbytes = bis->cur.bytes[BLKG_IOSTAT_READ ]; wbytes = bis->cur.bytes[BLKG_IOSTAT_WRITE ]; dbytes = bis->cur.bytes[BLKG_IOSTAT_DISCARD ]; rios = bis->cur.ios[BLKG_IOSTAT_READ ]; wios = bis->cur.ios[BLKG_IOSTAT_WRITE ];
Quoted 8 lines
block/blk-cgroup.c static void blkcg_print_one_stat(struct blkcg_gq *blkg, struct seq_file *s) · line 1159
Quoted 3 lines
block/blk-cgroup.c +1
-1dios = bis->cur.ios[BLKG_IOSTAT_DISCARD ]; -}while (u64_stats_fetch_retry (&bis->sync, seq)); +}The above two are the only places which can potentially benefit from per-blkg lock but these aren't hot paths. I'd just use
blkg_stat_lockfor
the above.
In __blkcg_rstat_flush, u64 sync is used for fetching data, changing to spinlock will increase overhead both 64 bit and 32 bit systems. 64 bit systems do noting in u64 sync, and 32 bit systems can read data in parallel in u64 sync if no one updating data. However, it is already protected by blkg_stat_lock, so there is no parallelism now, and there is no issue here. I think that no changes are needed here.
In blkcg_fill_root_iostats, this is where the issue happens in 32 bit SMP systems, so spinlock needs to be added to protect it.
In blkcg_print_one_stat, u64 sync is used for fetching data. Changing to spinlock will increase overhead like in __blkcg_rstat_flush. However, it is already protected by spin_lock_irq(&blkg->q->queue_lock) in blkcg_print_stat, so there is no parallelism now, and there is no issue here. I think that no changes are needed here.
Quoted 44 lines
block/blk-cgroup.c void blk_cgroup_bio_start(struct bio *bio) · line 2158
Quoted 9 lines
block/blk-cgroup.c +0
-4@@ -2152,30 +2141,29 @@ void blk_cgroup_bio_start(struct bio *bio) cpu =get_cpu (); bis =per_cpu_ptr (bio->bi_blkg->iostat_cpu, cpu); -flags =u64_stats_update_begin_irqsave (&bis->sync); - -/* -* If the bio is flagged with BIO_CGROUP_ACCT it means this is a split
Review void blk_cgroup_bio_start(struct bio *bio) · line 2159
Quoted 1 line
(quoted diff) +0
-1- * bio and we would have already accounted for the size of thebio.
block/blk-cgroup.c void blk_cgroup_bio_start(struct bio *bio)
Quoted 9 lines
block/blk-cgroup.c +3
-6-*/ -if (!bio_flagged (bio,BIO_CGROUP_ACCT )) { -bio_set_flag (bio,BIO_CGROUP_ACCT ); -bis->cur.bytes[rwd] += bio->bi_iter.bi_size; -} -bis->cur.ios[rwd]++; +scoped_guard(spinlock_irqsave, &bis->spinlock) { +/* +* If the bio is flagged with BIO_CGROUP_ACCT it means this is a split
Review void blk_cgroup_bio_start(struct bio *bio) · line 2147
Quoted 1 line
(quoted diff) +1
-0+ * bio and we would have already accounted for the size of thebio.
Quoted 20 lines
block/blk-cgroup.c +12
-6+*/ +if (!bio_flagged (bio,BIO_CGROUP_ACCT )) { +bio_set_flag (bio,BIO_CGROUP_ACCT ); +bis->cur.bytes[rwd] += bio->bi_iter.bi_size; +} +bis->cur.ios[rwd]++; -/* -* If the iostat_cpu isn't in a lockless list, put it into the -* list to indicate that a stat update is pending. -*/ -if (!READ_ONCE (bis->lqueued)) { -struct llist_head *lhead =this_cpu_ptr (blkcg->lhead); +/* +* If the iostat_cpu isn't in a lockless list, put it into the +* list to indicate that a stat update is pending. +*/ +if (!READ_ONCE (bis->lqueued)) { +struct llist_head *lhead =this_cpu_ptr (blkcg->lhead);
Quoted 11 lines
block/blk-cgroup.c void blk_cgroup_bio_start(struct bio *bio) · line 2164
Quoted 5 lines
block/blk-cgroup.c +3
-2-llist_add (&bis->lnode, lhead); -WRITE_ONCE (bis->lqueued, true); +llist_add (&bis->lnode, lhead); +WRITE_ONCE (bis->lqueued, true); +}These are per-cpu stat updates which should keep using
u64_sync. We don't
want to incur locking overhead for stat updates in the hot issue path.
I agree that no changes are needed in blk_cgroup_bio_start.
Quoted 4 lines
Thanks.
Signature · 1 line
tejun
I think we can look back to the original issue, which is that on 32 bit SMP systems will have concurrent problems on u64_stats_update_begin_irqsave and u64_stats_update_end_irqrestore in blkcg_fill_root_iostats. So, adding a lock only on 32 bits systems in blkcg_fill_root_iostats is to resolve the concurrent issue and not affect 64 bit systems, which do not have the issue in the first place, because u64 sync does noting in 64 bit systems and they don't need it.
I think we can just fix it by following change.
@@ -1134,9 +1128,8 @@ static void blkcg_fill_root_iostats(void)
cpu_dkstats->sectors[STAT_DISCARD] <<
9;
}+#if BITS_PER_LONG == 32
+ guard(spinlock_irqsave)(&blkg_stat_lock);
+#endif
flags = u64_stats_update_begin_irqsave(&blkg-
Quoted 1 line
iostat.sync);
blkg_iostat_set(&blkg->iostat.cur, &tmp);
u64_stats_update_end_irqrestore(&blkg->iostat.sync,
flags);
}}
Quoted 200 lines · Boy.Wu
Quoted 17 lines · Tejun Heo
External email : Please do not click links or open attachments until
you have verified the sender or the content.
Hello, Boy.So, looking at the patch, I'm not sure per-blkg lock makes sense.
Review static void blkcg_iostat_update(struct
Quoted 1 line · boy.wu
(quoted diff) +0
-0@@ -995,15 +995,13 @@ static void blkcg_iostat_update(struct
blkcg_gq*blkg, structblkg_iostat*cur,Quoted 5 lines
(quoted diff) +0
-1struct blkg_iostat *last) { struct blkg_iostat delta; -unsigned long flags;Quoted 11 lines
Review static void blkcg_iostat_update(struct · line 1005
Quoted 9 lines
(quoted diff) +1
-2/* propagate percpu delta to global */ -flags = u64_stats_update_begin_irqsave(&blkg->iostat.sync); +guard(spinlock_irqsave)(&blkg->iostat.spinlock); blkg_iostat_set(&delta, cur); blkg_iostat_sub(&delta, last); blkg_iostat_add(&blkg->iostat.cur, &delta); blkg_iostat_add(last, &delta); -u64_stats_update_end_irqrestore(&blkg->iostat.sync, flags); }This is already called with
blkg_stat_lockheld.In
blkcg_iostat_update, it is protected by bothblkg_stat_lockand u64
sync. I agree that no changes are needed here.Quoted 45 lines
Quoted 3 lines
(quoted diff) +0
-0@@ -1051,10 +1048,8 @@ static void __blkcg_rstat_flush(structblkcg
*blkcg, int cpu)
Review
Quoted 8 lines
goto
propagate_up; /* propagate up to parent only *//* fetch the current per-cpu values */
(quoted diff) +1
-3-do { -seq = u64_stats_fetch_begin(&bisc->sync); +scoped_guard(spinlock_irqsave, &bisc->spinlock) blkg_iostat_set(&cur, &bisc->cur); -} while (u64_stats_fetch_retry(&bisc->sync, seq));This is per-cpu stat and we should keep using
u64_syncfor them.Quoted 10 lines
(quoted diff) +1
-2@@ -1134,9 +1128,8 @@ static void blkcg_fill_root_iostats(void) cpu_dkstats->sectors[STAT_DISCARD] << 9; } -flags = u64_stats_update_begin_irqsave(&blkg->iostat.sync); +guard(spinlock_irqsave)(&blkg->iostat.spinlock); blkg_iostat_set(&blkg->iostat.cur, &tmp); -u64_stats_update_end_irqrestore(&blkg->iostat.sync, flags); } }...
Review static void blkcg_print_one_stat(struct
Quoted 1 line
(quoted diff) +0
-0@@ -1157,16 +1149,14 @@ static void blkcg_print_one_stat(struct
blkcg_gq*blkg, structseq_file*s)Quoted 14 lines
(quoted diff) +1
-4seq_printf(s, "%s ", dname); -do { -seq = u64_stats_fetch_begin(&bis->sync); - +scoped_guard(spinlock_irqsave, &bis->spinlock) { rbytes = bis->cur.bytes[BLKG_IOSTAT_READ]; wbytes = bis->cur.bytes[BLKG_IOSTAT_WRITE]; dbytes = bis->cur.bytes[BLKG_IOSTAT_DISCARD]; rios = bis->cur.ios[BLKG_IOSTAT_READ]; wios = bis->cur.ios[BLKG_IOSTAT_WRITE]; dios = bis->cur.ios[BLKG_IOSTAT_DISCARD]; -} while (u64_stats_fetch_retry(&bis->sync, seq));Quoted 8 lines
Review static void blkcg_print_one_stat(struct · line 1158
Quoted 1 line
(quoted diff) +1
-0+}The above two are the only places which can potentially benefit from
per-blkg lock but these aren't hot paths. I'd just useblkg_stat_lock
for
the above.In
__blkcg_rstat_flush, u64 sync is used for fetching data, changing to
spinlock will increase overhead both 64 bit and 32 bit systems. 64 bit
systems do noting in u64 sync, and 32 bit systems can read data in parallel in u64 sync if no one updating data. However, it is already protected byblkg_stat_lock, so there is no parallelism now, and there
is no issue here. I think that no changes are needed here.In
blkcg_fill_root_iostats, this is where the issue happens in 32 bit SMP systems, so spinlock needs to be added to protect it.In
blkcg_print_one_stat, u64 sync is used for fetching data. Changing to spinlock will increase overhead like in__blkcg_rstat_flush. However, it is already protected by spin_lock_irq(&blkg->q-Quoted 1 line
queue_lock)in
blkcg_print_stat, so there is no parallelism now, and there is no issue here. I think that no changes are needed here.Quoted 49 lines
Review void blk_cgroup_bio_start(struct bio · line 2159
Quoted 10 lines
(quoted diff) +0
-4@@ -2152,30 +2141,29 @@ void blk_cgroup_bio_start(struct bio *bio) cpu = get_cpu(); bis = per_cpu_ptr(bio->bi_blkg->iostat_cpu, cpu); -flags = u64_stats_update_begin_irqsave(&bis->sync); - -/* - * If the bio is flagged with BIO_CGROUP_ACCT it means this is asplit
Review void blk_cgroup_bio_start(struct bio · line 2160
Quoted 1 line
(quoted diff) +0
-1- * bio and we would have already accounted for the size of thebio.
Review void blk_cgroup_bio_start(struct bio · line 2147
Quoted 9 lines
(quoted diff) +3
-6- */ -if (!bio_flagged(bio, BIO_CGROUP_ACCT)) { -bio_set_flag(bio, BIO_CGROUP_ACCT); -bis->cur.bytes[rwd] += bio->bi_iter.bi_size; -} -bis->cur.ios[rwd]++; +scoped_guard(spinlock_irqsave, &bis->spinlock) { +/* + * If the bio is flagged with BIO_CGROUP_ACCT it means this is asplit
Review void blk_cgroup_bio_start(struct bio · line 2148
Quoted 1 line
(quoted diff) +1
-0+ * bio and we would have already accounted for the size of thebio.
Quoted 20 lines
(quoted diff) +12
-6+ */ +if (!bio_flagged(bio, BIO_CGROUP_ACCT)) { +bio_set_flag(bio, BIO_CGROUP_ACCT); +bis->cur.bytes[rwd] += bio->bi_iter.bi_size; +} +bis->cur.ios[rwd]++; -/* - * If the iostat_cpu isn't in a lockless list, put it into the - * list to indicate that a stat update is pending. - */ -if (!READ_ONCE(bis->lqueued)) { -struct llist_head *lhead = this_cpu_ptr(blkcg->lhead); +/* + * If the iostat_cpu isn't in a lockless list, put it into the + * list to indicate that a stat update is pending. + */ +if (!READ_ONCE(bis->lqueued)) { +struct llist_head *lhead = this_cpu_ptr(blkcg->lhead);Quoted 11 lines
Review void blk_cgroup_bio_start(struct bio · line 2165
Quoted 5 lines
(quoted diff) +3
-2-llist_add(&bis->lnode, lhead); -WRITE_ONCE(bis->lqueued, true); +llist_add(&bis->lnode, lhead); +WRITE_ONCE(bis->lqueued, true); +}These are per-cpu stat updates which should keep using
u64_sync. We don't
want to incur locking overhead for stat updates in the hot issue path.I agree that no changes are needed in
blk_cgroup_bio_start.Quoted 4 lines
Thanks.
Signature · 1 line
tejunI think we can look back to the original issue, which is that on 32 bit
SMP systems will have concurrent problems onu64_stats_update_begin_irqsaveandu64_stats_update_end_irqrestoreinblkcg_fill_root_iostats. So, adding a lock only on 32 bits systems inblkcg_fill_root_iostatsis to resolve the concurrent issue and not affect 64 bit systems, which do not have the issue in the first place,
because u64 sync does noting in 64 bit systems and they don't need it.I think we can just fix it by following change.
(quoted diff) +3
-0@@ -1134,9 +1128,8 @@ static void blkcg_fill_root_iostats(void) cpu_dkstats->sectors[STAT_DISCARD] << 9; } +#if BITS_PER_LONG == 32 + guard(spinlock_irqsave)(&blkg_stat_lock); +#endif flags = u64_stats_update_begin_irqsave(&blkg-Quoted 1 line
iostat.sync);
Quoted 5 lines
blkg_iostat_set(&blkg->iostat.cur, &tmp); u64_stats_update_end_irqrestore(&blkg->iostat.sync, flags); }}
I think there is another way to fix it. Maybe we should address the root cause, which is that u64_stats_update_begin_irqsave and u64_stats_update_end_irqrestore do not protect the seqcount in 32 bit SMP systems. This can be fix by the following change.
(quoted diff) +0 -0
b/include/linux/u64_stats_sync.h
index 46040d66334a..94dd74b4fb2c 100644
include/linux/u64_stats_sync.h +2 -0
@@ -64,6 +64,7 @@struct u64_stats_sync {#if BITS_PER_LONG ==32 seqcount_t seq; +spinlock_t spinlock;#endif };
@@ -138,6 +139,7 @@ static inline void u64_stats_inc(u64_stats_t *p)static inline void u64_stats_init (struct u64_stats_sync *syncp) {seqcount_init (&syncp->seq); + spin_lock_init(&syncp->spinlock) } static inline void __u64_stats_update_begin(struct u64_stats_sync
*syncp)
@@ -191,6 +193,7 @@ static inline unsigned long u64_stats_update_begin_irqsave(struct u64_stats_sync
{
unsigned long flags = __u64_stats_irqsave();+ spin_lock_irq(&syncp->spinlock);
__u64_stats_update_begin(syncp);
return flags; }
@@ -199,6 +202,7 @@ static inline void u64_stats_update_end_irqrestore(struct u64_stats_sync *syncp,
unsigned long flags)
{
__u64_stats_update_end(syncp);
+ spin_unlock_irq(&syncp->spinlock);
__u64_stats_irqrestore(flags);}
--
boy.wu
Quoted 47 lines · Boy.Wu
I think there is another way to fix it. Maybe we should address the root cause, which is that
u64_stats_update_begin_irqsaveandu64_stats_update_end_irqrestoredo not protect the seqcount in 32 bit SMP systems. This can be fix by the following change.include/linux/u64_stats_sync.h +2
-0@@ -64,6 +64,7 @@struct u64_stats_sync {#if BITS_PER_LONG ==32 seqcount_t seq; +spinlock_t spinlock;#endif };@@ -138,6 +139,7 @@ static inline void u64_stats_inc(u64_stats_t *p)static inline void u64_stats_init (struct u64_stats_sync *syncp) {seqcount_init (&syncp->seq); + spin_lock_init(&syncp->spinlock) } static inline void __u64_stats_update_begin(struct u64_stats_sync*syncp)
include/linux/u64_stats_sync.h +0
-0@@ -191,6 +193,7 @@ static inline unsigned longu64_stats_update_begin_irqsave(struct
u64_stats_sync{ unsigned long flags = __u64_stats_irqsave();include/linux/u64_stats_sync.h +2
-0+spin_lock_irq (&syncp->spinlock);__u64_stats_update_begin (syncp);return flags; }@@ -199,6 +202,7 @@ static inline void u64_stats_update_end_irqrestore(struct u64_stats_sync *syncp, unsigned long flags) {__u64_stats_update_end (syncp); +spin_unlock_irq (&syncp->spinlock);__u64_stats_irqrestore (flags);}
Quoted 3 lines
--
boy.wu
Never mind, there is a usage in u64_stats_sync.h
* Usage :
*
* Stats producer (writer) should use following template granted it
already got
* an exclusive access to counters (a lock is already taken, or per cpu
* data is used [in a non preemptable context])
*
* spin_lock_bh(...) or other synchronization to get exclusive access * ... * u64_stats_update_begin(&stats->syncp); * u64_stats_add(&stats->bytes64, len); // non atomic operation * u64_stats_inc(&stats->packets64); // non atomic operation * u64_stats_update_end(&stats->syncp);
--
boy.wu
[email protected]
maintainerHello,
Does something like the following work for you?
Thanks.
block/blk-cgroup.c +15 -19
@@ -329,7 +329,6 @@ static struct blkcg_gq *blkg_alloc(struct blkcg *blkcg, struct gendisk *disk,INIT_WORK (&blkg->async_bio_work, blkg_async_bio_workfn);#endif -u64_stats_init (&blkg->iostat.sync);for_each_possible_cpu (cpu) {u64_stats_init (&per_cpu_ptr (blkg->iostat_cpu, cpu)->sync);per_cpu_ptr (blkg->iostat_cpu, cpu)->blkg = blkg;
@@ -632,24 +631,26 @@ static void blkg_iostat_set(struct blkg_iostat *dst, struct blkg_iostat *src)static void __blkg_clear_stat (struct blkg_iostat_set *bis) {struct blkg_iostat cur = {0 }; -unsigned long flags; - flags =u64_stats_update_begin_irqsave (&bis->sync);blkg_iostat_set (&bis->cur, &cur);blkg_iostat_set (&bis->last, &cur); -u64_stats_update_end_irqrestore (&bis->sync, flags); }static void blkg_clear_stat (struct blkcg_gq *blkg) { +unsigned long flags;int cpu; +raw_spin_lock_irqsave (&blkg_stat_lock, flags); +for_each_possible_cpu (cpu) {struct blkg_iostat_set *s =per_cpu_ptr (blkg->iostat_cpu, cpu);__blkg_clear_stat (s); }__blkg_clear_stat (&blkg->iostat); + +raw_spin_unlock_irqrestore (&blkg_stat_lock, flags); } static int blkcg_reset_stats(struct cgroup_subsys_state *css,
@@ -998,12 +999,10 @@ static void blkcg_iostat_update(struct blkcg_gq *blkg, struct blkg_iostat *cur,unsigned long flags;/* propagate percpu delta to global */ - flags =u64_stats_update_begin_irqsave (&blkg->iostat.sync);blkg_iostat_set (&delta, cur);blkg_iostat_sub (&delta, last);blkg_iostat_add (&blkg->iostat.cur, &delta);blkg_iostat_add (last, &delta); -u64_stats_update_end_irqrestore (&blkg->iostat.sync, flags); }static void __blkcg_rstat_flush (struct blkcg *blkcg,int cpu)
@@ -1134,9 +1133,9 @@ static void blkcg_fill_root_iostats(void) cpu_dkstats->sectors[STAT_DISCARD ] <<9 ; } - flags =u64_stats_update_begin_irqsave (&blkg->iostat.sync); +raw_spin_lock_irqsave (&blkg_stat_lock, flags);blkg_iostat_set (&blkg->iostat.cur, &tmp); -u64_stats_update_end_irqrestore (&blkg->iostat.sync, flags); +raw_spin_unlock_irqrestore (&blkg_stat_lock, flags); } }
@@ -1145,7 +1144,6 @@ static void blkcg_print_one_stat(struct blkcg_gq *blkg, struct seq_file *s)struct blkg_iostat_set *bis = &blkg->iostat;u64 rbytes, wbytes, rios, wios, dbytes, dios;const char *dname; -unsigned seq;int i;if (!blkg->online)
@@ -1157,16 +1155,14 @@ static void blkcg_print_one_stat(struct blkcg_gq *blkg, struct seq_file *s)seq_printf (s,"%s " , dname); -do { - seq =u64_stats_fetch_begin (&bis->sync); - - rbytes = bis->cur.bytes[BLKG_IOSTAT_READ ]; - wbytes = bis->cur.bytes[BLKG_IOSTAT_WRITE ]; - dbytes = bis->cur.bytes[BLKG_IOSTAT_DISCARD ]; - rios = bis->cur.ios[BLKG_IOSTAT_READ ]; - wios = bis->cur.ios[BLKG_IOSTAT_WRITE ]; - dios = bis->cur.ios[BLKG_IOSTAT_DISCARD ]; - }while (u64_stats_fetch_retry (&bis->sync, seq)); +raw_spin_lock_irq (&blkg_stat_lock); + rbytes = bis->cur.bytes[BLKG_IOSTAT_READ ]; + wbytes = bis->cur.bytes[BLKG_IOSTAT_WRITE ]; + dbytes = bis->cur.bytes[BLKG_IOSTAT_DISCARD ]; + rios = bis->cur.ios[BLKG_IOSTAT_READ ]; + wios = bis->cur.ios[BLKG_IOSTAT_WRITE ]; + dios = bis->cur.ios[BLKG_IOSTAT_DISCARD ]; +raw_spin_unlock_irq (&blkg_stat_lock, flags);if (rbytes || wbytes || rios || wios) { seq_printf(s, "rbytes=%llu wbytes=%llu rios=%llu wios=%llu dbytes=%llu dios=%llu",
Quoted 97 lines · On 7/17/24 12:55, [email protected] wrote:
Hello,
Does something like the following work for you?
Thanks.
block/blk-cgroup.c +8
-19@@ -329,7 +329,6 @@ static struct blkcg_gq *blkg_alloc(struct blkcg *blkcg, struct gendisk *disk,INIT_WORK (&blkg->async_bio_work, blkg_async_bio_workfn);#endif -u64_stats_init (&blkg->iostat.sync);for_each_possible_cpu (cpu) {u64_stats_init (&per_cpu_ptr (blkg->iostat_cpu, cpu)->sync);per_cpu_ptr (blkg->iostat_cpu, cpu)->blkg = blkg;@@ -632,24 +631,26 @@ static void blkg_iostat_set(struct blkg_iostat *dst, struct blkg_iostat *src)static void __blkg_clear_stat (struct blkg_iostat_set *bis) {struct blkg_iostat cur = {0 }; -unsigned long flags; - flags =u64_stats_update_begin_irqsave (&bis->sync);blkg_iostat_set (&bis->cur, &cur);blkg_iostat_set (&bis->last, &cur); -u64_stats_update_end_irqrestore (&bis->sync, flags); }static void blkg_clear_stat (struct blkcg_gq *blkg) { +unsigned long flags;int cpu; +raw_spin_lock_irqsave (&blkg_stat_lock, flags); +for_each_possible_cpu (cpu) {struct blkg_iostat_set *s =per_cpu_ptr (blkg->iostat_cpu, cpu);__blkg_clear_stat (s); }__blkg_clear_stat (&blkg->iostat); + +raw_spin_unlock_irqrestore (&blkg_stat_lock, flags); } static int blkcg_reset_stats(struct cgroup_subsys_state *css,@@ -998,12 +999,10 @@ static void blkcg_iostat_update(struct blkcg_gq *blkg, struct blkg_iostat *cur,unsigned long flags;/* propagate percpu delta to global */ - flags =u64_stats_update_begin_irqsave (&blkg->iostat.sync);blkg_iostat_set (&delta, cur);blkg_iostat_sub (&delta, last);blkg_iostat_add (&blkg->iostat.cur, &delta);blkg_iostat_add (last, &delta); -u64_stats_update_end_irqrestore (&blkg->iostat.sync, flags); }static void __blkcg_rstat_flush (struct blkcg *blkcg,int cpu)@@ -1134,9 +1133,9 @@ static void blkcg_fill_root_iostats(void) cpu_dkstats->sectors[STAT_DISCARD ] <<9 ; } - flags =u64_stats_update_begin_irqsave (&blkg->iostat.sync); +raw_spin_lock_irqsave (&blkg_stat_lock, flags);blkg_iostat_set (&blkg->iostat.cur, &tmp); -u64_stats_update_end_irqrestore (&blkg->iostat.sync, flags); +raw_spin_unlock_irqrestore (&blkg_stat_lock, flags); } }@@ -1145,7 +1144,6 @@ static void blkcg_print_one_stat(struct blkcg_gq *blkg, struct seq_file *s)struct blkg_iostat_set *bis = &blkg->iostat;u64 rbytes, wbytes, rios, wios, dbytes, dios;const char *dname; -unsigned seq;int i;if (!blkg->online)@@ -1157,16 +1155,14 @@ static void blkcg_print_one_stat(struct blkcg_gq *blkg, struct seq_file *s)seq_printf (s,"%s " , dname); -do { - seq =u64_stats_fetch_begin (&bis->sync); - - rbytes = bis->cur.bytes[BLKG_IOSTAT_READ ]; - wbytes = bis->cur.bytes[BLKG_IOSTAT_WRITE ]; - dbytes = bis->cur.bytes[BLKG_IOSTAT_DISCARD ]; - rios = bis->cur.ios[BLKG_IOSTAT_READ ]; - wios = bis->cur.ios[BLKG_IOSTAT_WRITE ]; - dios = bis->cur.ios[BLKG_IOSTAT_DISCARD ]; - }while (u64_stats_fetch_retry (&bis->sync, seq)); +raw_spin_lock_irq (&blkg_stat_lock);
block/blk-cgroup.c static void blkcg_print_one_stat(struct blkcg_gq *blkg, struct seq_file *s) · line 1168
Quoted 10 lines
block/blk-cgroup.c +7
-0+ rbytes = bis->cur.bytes[BLKG_IOSTAT_READ ]; + wbytes = bis->cur.bytes[BLKG_IOSTAT_WRITE ]; + dbytes = bis->cur.bytes[BLKG_IOSTAT_DISCARD ]; + rios = bis->cur.ios[BLKG_IOSTAT_READ ]; + wios = bis->cur.ios[BLKG_IOSTAT_WRITE ]; + dios = bis->cur.ios[BLKG_IOSTAT_DISCARD ]; +raw_spin_unlock_irq (&blkg_stat_lock, flags);if (rbytes || wbytes || rios || wios) { seq_printf(s, "rbytes=%llu wbytes=%llu rios=%llu wios=%llu dbytes=%llu dios=%llu",
bis->sync is still being used in blk_cgroup_bio_start(). Replacing it with a global lock may kill performance. We may have to use a per-cpu lock if we want to go this route of eliminating bis->sync.
Cheers,
Longman
[email protected]
maintainerHello, Waiman.
Quoted 3 lines · Waiman Long
bis->syncis still being used inblk_cgroup_bio_start(). Replacing it with a global lock may kill performance. We may have to use a per-cpu lock if we want to go this route of eliminatingbis->sync.
So, the idea is to keep using u64_sync for blkg->iostat_cpu and use blkg_stat_lock for blkg->iostat. The former is the only one which is updated in hot path, right?
Thanks.
Signature · 1 line
tejun
Quoted 9 lines · On 7/17/24 13:40, [email protected] wrote:
Hello, Waiman.
Quoted 3 lines · Waiman Long
bis->syncis still being used inblk_cgroup_bio_start(). Replacing it with a global lock may kill performance. We may have to use a per-cpu lock if we want to go this route of eliminatingbis->sync.So, the idea is to keep using
u64_syncforblkg->iostat_cpuand useblkg_stat_lockforblkg->iostat. The former is the only one which is updated in hot path, right?
Well, it can be confusing whether we are dealing with blkg->iostat or blkg->iostat_cpu. In many cases, we are dealing with iostat_cpu instead of iostat like __blkcg_rstat_flush() and blkg_clear_stat(). So we can't eliminate the use of u64_stats_update_begin_irqsave() in those cases.
Cheers,
Longman
[email protected]
maintainerHello,
Quoted 4 lines · Waiman Long
Well, it can be confusing whether we are dealing with
blkg->iostatorblkg->iostat_cpu. In many cases, we are dealing withiostat_cpuinstead of iostat like__blkcg_rstat_flush()andblkg_clear_stat(). So we can't eliminate the use ofu64_stats_update_begin_irqsave()in those cases.
I mean, we need to distinguish them. For 32bits, blkg->iostat has multiple updaters, so we can't use u64_sync; however, blkg->iostat_cpu has only one updater (except blkg_clear_stat() which I don't think we need to worry too much about), so u64_sync is fine.
Thanks.
Signature · 1 line
tejun
Quoted 11 lines · On 7/17/24 14:24, [email protected] wrote:
Hello,
Quoted 4 lines · Waiman Long
Well, it can be confusing whether we are dealing with
blkg->iostatorblkg->iostat_cpu. In many cases, we are dealing withiostat_cpuinstead of iostat like__blkcg_rstat_flush()andblkg_clear_stat(). So we can't eliminate the use ofu64_stats_update_begin_irqsave()in those cases.I mean, we need to distinguish them. For 32bits,
blkg->iostathas multiple updaters, so we can't useu64_sync; however,blkg->iostat_cpuhas only one updater (exceptblkg_clear_stat()which I don't think we need to worry too much about), sou64_syncis fine.
I was wrong about __blkcg_rstat_flush(). Right, the main updater of iostat_cpu is blk_cgroup_bio_start(). We do need to drop down some comment on what is protected by u64_sync and what is by blkg_stat_lock though. It can be confusing.
Cheers,
Longman
[email protected]
maintainerQuoted 4 lines · Waiman Long
I was wrong about
__blkcg_rstat_flush(). Right, the main updater ofiostat_cpuisblk_cgroup_bio_start(). We do need to drop down some comment on what is protected byu64_syncand what is byblkg_stat_lockthough. It can be confusing.
Oh yeah, definitely. It is confusing.
Signature · 1 line
tejun
Quoted 113 lines · On Wed, 2024-07-17 at 06:55 -1000, [email protected] wrote:
External email : Please do not click links or open attachments until you have verified the sender or the content.
Hello,Does something like the following work for you?
Thanks.
block/blk-cgroup.c +0
-1@@ -329,7 +329,6 @@ static struct blkcg_gq *blkg_alloc(struct blkcg *blkcg, struct gendisk *disk,INIT_WORK (&blkg->async_bio_work, blkg_async_bio_workfn);#endif -u64_stats_init (&blkg->iostat.sync);for_each_possible_cpu (cpu) {u64_stats_init (&per_cpu_ptr (blkg->iostat_cpu, cpu)->sync);
per_cpu_ptr(blkg->iostat_cpu, cpu)->blkg = blkg;block/blk-cgroup.c +5
-3@@ -632,24 +631,26 @@ static void blkg_iostat_set(struct blkg_iostat *dst, struct blkg_iostat *src)static void __blkg_clear_stat (struct blkg_iostat_set *bis) {struct blkg_iostat cur = {0 }; -unsigned long flags; -flags =u64_stats_update_begin_irqsave (&bis->sync);blkg_iostat_set (&bis->cur, &cur);blkg_iostat_set (&bis->last, &cur); -u64_stats_update_end_irqrestore (&bis->sync, flags); }static void blkg_clear_stat (struct blkcg_gq *blkg) { +unsigned long flags;int cpu; +raw_spin_lock_irqsave (&blkg_stat_lock, flags); +for_each_possible_cpu (cpu) {struct blkg_iostat_set *s =per_cpu_ptr (blkg->iostat_cpu, cpu);__blkg_clear_stat (s); }__blkg_clear_stat (&blkg->iostat); + +raw_spin_unlock_irqrestore (&blkg_stat_lock, flags); }static int blkcg_reset_stats(struct
cgroup_subsys_state*css,block/blk-cgroup.c +0
-2@@ -998,12 +999,10 @@ static void blkcg_iostat_update(struct blkcg_gq *blkg, struct blkg_iostat *cur, unsigned long flags;/* propagate percpu delta to global */ -flags =u64_stats_update_begin_irqsave (&blkg->iostat.sync);blkg_iostat_set (&delta, cur);blkg_iostat_sub (&delta, last);blkg_iostat_add (&blkg->iostat.cur, &delta);blkg_iostat_add (last, &delta); -u64_stats_update_end_irqrestore (&blkg->iostat.sync, flags); }static void
__blkcg_rstat_flush(struct blkcg *blkcg, int cpu)block/blk-cgroup.c +2
-3@@ -1134,9 +1133,9 @@ static void blkcg_fill_root_iostats(void) cpu_dkstats->sectors[STAT_DISCARD ] <<9 ; } -flags =u64_stats_update_begin_irqsave (&blkg->iostat.sync); +raw_spin_lock_irqsave (&blkg_stat_lock, flags);blkg_iostat_set (&blkg->iostat.cur, &tmp); -u64_stats_update_end_irqrestore (&blkg->iostat.sync, flags); +raw_spin_unlock_irqrestore (&blkg_stat_lock, flags); } }@@ -1145,7 +1144,6 @@ static void blkcg_print_one_stat(struct blkcg_gq *blkg, struct seq_file *s) struct blkg_iostat_set *bis = &blkg->iostat;u64 rbytes, wbytes, rios, wios, dbytes, dios;const char *dname; -unsigned seq;int i;if (!blkg->online)
block/blk-cgroup.c +8
-10@@ -1157,16 +1155,14 @@ static void blkcg_print_one_stat(struct blkcg_gq *blkg, struct seq_file *s) seq_printf(s, "%s ", dname); -do { -seq =u64_stats_fetch_begin (&bis->sync); - -rbytes = bis->cur.bytes[BLKG_IOSTAT_READ ]; -wbytes = bis->cur.bytes[BLKG_IOSTAT_WRITE ]; -dbytes = bis->cur.bytes[BLKG_IOSTAT_DISCARD ]; -rios = bis->cur.ios[BLKG_IOSTAT_READ ]; -wios = bis->cur.ios[BLKG_IOSTAT_WRITE ]; -dios = bis->cur.ios[BLKG_IOSTAT_DISCARD ]; -}while (u64_stats_fetch_retry (&bis->sync, seq)); +raw_spin_lock_irq (&blkg_stat_lock); +rbytes = bis->cur.bytes[BLKG_IOSTAT_READ ]; +wbytes = bis->cur.bytes[BLKG_IOSTAT_WRITE ]; +dbytes = bis->cur.bytes[BLKG_IOSTAT_DISCARD ]; +rios = bis->cur.ios[BLKG_IOSTAT_READ ]; +wios = bis->cur.ios[BLKG_IOSTAT_WRITE ]; +dios = bis->cur.ios[BLKG_IOSTAT_DISCARD ]; +raw_spin_unlock_irq (&blkg_stat_lock, flags);if (rbytes || wbytes || rios || wios) {
Quoted 2 lines
seq_printf(s, "rbytes=%llu wbytes=%llu rios=%llu wios=%llu
dbytes=%llu dios=%llu",
I think this will work, but as I mentioned before, this issue is only on 32 bit SMP systems. Replacing u64 sync with spinlock will increase overhead on 64 bit systems, because u64 sync does nothing on 64 bit systems. However, if it is acceptable, we can proceed with this solution.
--
boy.wu
[email protected]
maintainerHello,
On Thu, Jul 18, 2024 at 01:21:32AM +0000, Boy Wu (吳勃誼) wrote:
...
Quoted 5 lines
I think this will work, but as I mentioned before, this issue is only on 32 bit SMP systems. Replacing u64 sync with spinlock will increase overhead on 64 bit systems, because u64 sync does nothing on 64 bit systems. However, if it is acceptable, we can proceed with this solution.
We can encapsulate the spinlock in some helpers and make them conditional on 32bit. However, the u64_sync -> spinlock conversions in the suggested patch are all on cold paths, so I doubt it'd be all that noticeable especially given that the hottest path of them all is already grabbing blkg_stat_lock, but either way is fine by me.
Thanks.
Signature · 1 line
tejun