Skip to contentSkip to conversation

blk-cgroup: Replace u64 sync with spinlock for iostat update

PATCH Superseded boy.wu 3 participants 14 messages Jul 16, 2024 – Jul 18, 2024

Subsystems (4)

v3 of 4 Superseded Latest is v4 →

lore
Share & tools
lore

boy.wu

author
#0 lore raw

PATCH v3 1/1 blk-cgroup: Replace u64 sync with spinlock for iostat update

From: Boy Wu <[email protected]>

In 32bit SMP systems, if multiple CPUs call blkcg_print_stat, it may cause blkcg_fill_root_iostats to have a concurrent problem on the seqlock in u64_stats_update, which will cause a deadlock on u64_stats_fetch_begin in blkcg_print_one_stat.

Thus, replace u64 sync with spinlock to protect iostat update.

Fixes ef45fe470e1e ("blk-cgroup: show global disk stats in root cgroup io.stat")

---
Change in v2:
- update commit message
- Remove u64_sync
- Replace spin_lock_irq with guard statement
- Replace blkg->q->queue_lock with blkg_stat_lock
Change in v3:
- update commit message
- Add spinlock in blkg_iostat_set structure
- Replace all u64_sync with spinlock for iostat
- Replace blkg_stat_lock with iostat.spinlock

---
 block/blk-cgroup.c | 62 +++++++++++++++++++---------------------------
 block/blk-cgroup.h |  1 +
 2 files changed, 26 insertions(+), 37 deletions(-)
block/blk-cgroup.c +25 -37
index 37e6cc91d576..4b66f37c45a0 100644
@@ -329,7 +329,7 @@ static struct blkcg_gq *blkg_alloc(struct blkcg *blkcg, struct gendisk *disk,
 	INIT_WORK(&blkg->async_bio_work, blkg_async_bio_workfn);
 #endif
 
-	u64_stats_init(&blkg->iostat.sync);
+	spin_lock_init(&blkg->iostat.spinlock);
 	for_each_possible_cpu(cpu) {
 		u64_stats_init(&per_cpu_ptr(blkg->iostat_cpu, cpu)->sync);
 		per_cpu_ptr(blkg->iostat_cpu, cpu)->blkg = blkg;
@@ -995,15 +995,13 @@ static void blkcg_iostat_update(struct blkcg_gq *blkg, struct blkg_iostat *cur,
 				struct blkg_iostat *last)
 {
 	struct blkg_iostat delta;
-	unsigned long flags;
 
 	/* propagate percpu delta to global */
-	flags = u64_stats_update_begin_irqsave(&blkg->iostat.sync);
+	guard(spinlock_irqsave)(&blkg->iostat.spinlock);
 	blkg_iostat_set(&delta, cur);
 	blkg_iostat_sub(&delta, last);
 	blkg_iostat_add(&blkg->iostat.cur, &delta);
 	blkg_iostat_add(last, &delta);
-	u64_stats_update_end_irqrestore(&blkg->iostat.sync, flags);
 }
 
 static void __blkcg_rstat_flush(struct blkcg *blkcg, int cpu)
@@ -1034,7 +1032,6 @@ static void __blkcg_rstat_flush(struct blkcg *blkcg, int cpu)
 		struct blkcg_gq *blkg = bisc->blkg;
 		struct blkcg_gq *parent = blkg->parent;
 		struct blkg_iostat cur;
-		unsigned int seq;
 
 		/*
 		 * Order assignment of `next_bisc` from `bisc->lnode.next` in
@@ -1051,10 +1048,8 @@ static void __blkcg_rstat_flush(struct blkcg *blkcg, int cpu)
 			goto propagate_up; /* propagate up to parent only */
 
 		/* fetch the current per-cpu values */
-		do {
-			seq = u64_stats_fetch_begin(&bisc->sync);
+		scoped_guard(spinlock_irqsave, &bisc->spinlock)
 			blkg_iostat_set(&cur, &bisc->cur);
-		} while (u64_stats_fetch_retry(&bisc->sync, seq));
 
 		blkcg_iostat_update(blkg, &cur, &bisc->last);
 
@@ -1112,7 +1107,6 @@ static void blkcg_fill_root_iostats(void)
 		struct blkcg_gq *blkg = bdev->bd_disk->queue->root_blkg;
 		struct blkg_iostat tmp;
 		int cpu;
-		unsigned long flags;
 
 		memset(&tmp, 0, sizeof(tmp));
 		for_each_possible_cpu(cpu) {
@@ -1134,9 +1128,8 @@ static void blkcg_fill_root_iostats(void)
 				cpu_dkstats->sectors[STAT_DISCARD] << 9;
 		}
 
-		flags = u64_stats_update_begin_irqsave(&blkg->iostat.sync);
+		guard(spinlock_irqsave)(&blkg->iostat.spinlock);
 		blkg_iostat_set(&blkg->iostat.cur, &tmp);
-		u64_stats_update_end_irqrestore(&blkg->iostat.sync, flags);
 	}
 }
 
@@ -1145,7 +1138,6 @@ static void blkcg_print_one_stat(struct blkcg_gq *blkg, struct seq_file *s)
 	struct blkg_iostat_set *bis = &blkg->iostat;
 	u64 rbytes, wbytes, rios, wios, dbytes, dios;
 	const char *dname;
-	unsigned seq;
 	int i;
 
 	if (!blkg->online)
@@ -1157,16 +1149,14 @@ static void blkcg_print_one_stat(struct blkcg_gq *blkg, struct seq_file *s)
 
 	seq_printf(s, "%s ", dname);
 
-	do {
-		seq = u64_stats_fetch_begin(&bis->sync);
-
+	scoped_guard(spinlock_irqsave, &bis->spinlock) {
 		rbytes = bis->cur.bytes[BLKG_IOSTAT_READ];
 		wbytes = bis->cur.bytes[BLKG_IOSTAT_WRITE];
 		dbytes = bis->cur.bytes[BLKG_IOSTAT_DISCARD];
 		rios = bis->cur.ios[BLKG_IOSTAT_READ];
 		wios = bis->cur.ios[BLKG_IOSTAT_WRITE];
 		dios = bis->cur.ios[BLKG_IOSTAT_DISCARD];
-	} while (u64_stats_fetch_retry(&bis->sync, seq));
+	}
 
 	if (rbytes || wbytes || rios || wios) {
 		seq_printf(s, "rbytes=%llu wbytes=%llu rios=%llu wios=%llu dbytes=%llu dios=%llu",
@@ -2141,7 +2131,6 @@ void blk_cgroup_bio_start(struct bio *bio)
 	struct blkcg *blkcg = bio->bi_blkg->blkcg;
 	int rwd = blk_cgroup_io_type(bio), cpu;
 	struct blkg_iostat_set *bis;
-	unsigned long flags;
 
 	if (!cgroup_subsys_on_dfl(io_cgrp_subsys))
 		return;
@@ -2152,30 +2141,29 @@ void blk_cgroup_bio_start(struct bio *bio)
 
 	cpu = get_cpu();
 	bis = per_cpu_ptr(bio->bi_blkg->iostat_cpu, cpu);
-	flags = u64_stats_update_begin_irqsave(&bis->sync);
-
-	/*
-	 * If the bio is flagged with BIO_CGROUP_ACCT it means this is a split
-	 * bio and we would have already accounted for the size of the bio.
-	 */
-	if (!bio_flagged(bio, BIO_CGROUP_ACCT)) {
-		bio_set_flag(bio, BIO_CGROUP_ACCT);
-		bis->cur.bytes[rwd] += bio->bi_iter.bi_size;
-	}
-	bis->cur.ios[rwd]++;
+	scoped_guard(spinlock_irqsave, &bis->spinlock) {
+		/*
+		 * If the bio is flagged with BIO_CGROUP_ACCT it means this is a split
+		 * bio and we would have already accounted for the size of the bio.
+		 */
+		if (!bio_flagged(bio, BIO_CGROUP_ACCT)) {
+			bio_set_flag(bio, BIO_CGROUP_ACCT);
+			bis->cur.bytes[rwd] += bio->bi_iter.bi_size;
+		}
+		bis->cur.ios[rwd]++;
 
-	/*
-	 * If the iostat_cpu isn't in a lockless list, put it into the
-	 * list to indicate that a stat update is pending.
-	 */
-	if (!READ_ONCE(bis->lqueued)) {
-		struct llist_head *lhead = this_cpu_ptr(blkcg->lhead);
+		/*
+		 * If the iostat_cpu isn't in a lockless list, put it into the
+		 * list to indicate that a stat update is pending.
+		 */
+		if (!READ_ONCE(bis->lqueued)) {
+			struct llist_head *lhead = this_cpu_ptr(blkcg->lhead);
 
-		llist_add(&bis->lnode, lhead);
-		WRITE_ONCE(bis->lqueued, true);
+			llist_add(&bis->lnode, lhead);
+			WRITE_ONCE(bis->lqueued, true);
+		}
 	}
 
-	u64_stats_update_end_irqrestore(&bis->sync, flags);
 	cgroup_rstat_updated(blkcg->css.cgroup, cpu);
 	put_cpu();
 }
block/blk-cgroup.h +1 -0
index bd472a30bc61..b9544969a131 100644
@@ -44,6 +44,7 @@ struct blkg_iostat {
 };
 
 struct blkg_iostat_set {
+	spinlock_t			spinlock;
 	struct u64_stats_sync		sync;
 	struct blkcg_gq		       *blkg;
 	struct llist_node		lnode;
Signature · 1 line
2.18.0
maintainer
#1 lore raw

Hello, Boy.

So, looking at the patch, I'm not sure per-blkg lock makes sense.

Quoted 5 lines · boy.wu
block/blk-cgroup.c +0 -1
@@ -995,15 +995,13 @@ static void blkcg_iostat_update(struct blkcg_gq *blkg, struct blkg_iostat *cur,
 				struct blkg_iostat *last)
 {
 	struct blkg_iostat delta;
-	unsigned long flags;

block/blk-cgroup.c static void blkcg_iostat_update(struct blkcg_gq *blkg, struct blkg_iostat *cur, · line 1005

Quoted 10 lines
block/blk-cgroup.c +1 -2
 
 	/* propagate percpu delta to global */
-	flags = u64_stats_update_begin_irqsave(&blkg->iostat.sync);
+	guard(spinlock_irqsave)(&blkg->iostat.spinlock);
 	blkg_iostat_set(&delta, cur);
 	blkg_iostat_sub(&delta, last);
 	blkg_iostat_add(&blkg->iostat.cur, &delta);
 	blkg_iostat_add(last, &delta);
-	u64_stats_update_end_irqrestore(&blkg->iostat.sync, flags);
 }

This is already called with blkg_stat_lock held.

block/blk-cgroup.c static void __blkcg_rstat_flush(struct blkcg *blkcg, int cpu) · line 1057

Quoted 9 lines
block/blk-cgroup.c +1 -3
@@ -1051,10 +1048,8 @@ static void __blkcg_rstat_flush(struct blkcg *blkcg, int cpu)
 			goto propagate_up; /* propagate up to parent only */
 
 		/* fetch the current per-cpu values */
-		do {
-			seq = u64_stats_fetch_begin(&bisc->sync);
+		scoped_guard(spinlock_irqsave, &bisc->spinlock)
 			blkg_iostat_set(&cur, &bisc->cur);
-		} while (u64_stats_fetch_retry(&bisc->sync, seq));

This is per-cpu stat and we should keep using u64_sync for them.

Quoted 10 lines
block/blk-cgroup.c +1 -2
@@ -1134,9 +1128,8 @@ static void blkcg_fill_root_iostats(void)
 				cpu_dkstats->sectors[STAT_DISCARD] << 9;
 		}
 
-		flags = u64_stats_update_begin_irqsave(&blkg->iostat.sync);
+		guard(spinlock_irqsave)(&blkg->iostat.spinlock);
 		blkg_iostat_set(&blkg->iostat.cur, &tmp);
-		u64_stats_update_end_irqrestore(&blkg->iostat.sync, flags);
 	}
 }

...

Quoted 6 lines
block/blk-cgroup.c +0 -2
@@ -1157,16 +1149,14 @@ static void blkcg_print_one_stat(struct blkcg_gq *blkg, struct seq_file *s)
 
 	seq_printf(s, "%s ", dname);
 
-	do {
-		seq = u64_stats_fetch_begin(&bis->sync);

block/blk-cgroup.c static void blkcg_print_one_stat(struct blkcg_gq *blkg, struct seq_file *s) · line 1159

Quoted 10 lines
block/blk-cgroup.c +2 -2
-
+	scoped_guard(spinlock_irqsave, &bis->spinlock) {
 		rbytes = bis->cur.bytes[BLKG_IOSTAT_READ];
 		wbytes = bis->cur.bytes[BLKG_IOSTAT_WRITE];
 		dbytes = bis->cur.bytes[BLKG_IOSTAT_DISCARD];
 		rios = bis->cur.ios[BLKG_IOSTAT_READ];
 		wios = bis->cur.ios[BLKG_IOSTAT_WRITE];
 		dios = bis->cur.ios[BLKG_IOSTAT_DISCARD];
-	} while (u64_stats_fetch_retry(&bis->sync, seq));
+	}

The above two are the only places which can potentially benefit from per-blkg lock but these aren't hot paths. I'd just use blkg_stat_lock for the above.

Quoted 34 lines
block/blk-cgroup.c +12 -17
@@ -2152,30 +2141,29 @@ void blk_cgroup_bio_start(struct bio *bio)
 
 	cpu = get_cpu();
 	bis = per_cpu_ptr(bio->bi_blkg->iostat_cpu, cpu);
-	flags = u64_stats_update_begin_irqsave(&bis->sync);
-
-	/*
-	 * If the bio is flagged with BIO_CGROUP_ACCT it means this is a split
-	 * bio and we would have already accounted for the size of the bio.
-	 */
-	if (!bio_flagged(bio, BIO_CGROUP_ACCT)) {
-		bio_set_flag(bio, BIO_CGROUP_ACCT);
-		bis->cur.bytes[rwd] += bio->bi_iter.bi_size;
-	}
-	bis->cur.ios[rwd]++;
+	scoped_guard(spinlock_irqsave, &bis->spinlock) {
+		/*
+		 * If the bio is flagged with BIO_CGROUP_ACCT it means this is a split
+		 * bio and we would have already accounted for the size of the bio.
+		 */
+		if (!bio_flagged(bio, BIO_CGROUP_ACCT)) {
+			bio_set_flag(bio, BIO_CGROUP_ACCT);
+			bis->cur.bytes[rwd] += bio->bi_iter.bi_size;
+		}
+		bis->cur.ios[rwd]++;
 
-	/*
-	 * If the iostat_cpu isn't in a lockless list, put it into the
-	 * list to indicate that a stat update is pending.
-	 */
-	if (!READ_ONCE(bis->lqueued)) {
-		struct llist_head *lhead = this_cpu_ptr(blkcg->lhead);
+		/*
+		 * If the iostat_cpu isn't in a lockless list, put it into the

block/blk-cgroup.c void blk_cgroup_bio_start(struct bio *bio) · line 2164

Quoted 10 lines
block/blk-cgroup.c +7 -2
+		 * list to indicate that a stat update is pending.
+		 */
+		if (!READ_ONCE(bis->lqueued)) {
+			struct llist_head *lhead = this_cpu_ptr(blkcg->lhead);
 
-		llist_add(&bis->lnode, lhead);
-		WRITE_ONCE(bis->lqueued, true);
+			llist_add(&bis->lnode, lhead);
+			WRITE_ONCE(bis->lqueued, true);
+		}

These are per-cpu stat updates which should keep using u64_sync. We don't want to incur locking overhead for stat updates in the hot issue path.

Thanks.

Signature · 1 line
tejun
#2 lore raw
Quoted 15 lines · Tejun Heo

External email : Please do not click links or open attachments until you have verified the sender or the content.
Hello, Boy.

So, looking at the patch, I'm not sure per-blkg lock makes sense.

Review static void blkcg_iostat_update(struct

Quoted 1 line · boy.wu
(quoted diff) +0 -0
@@ -995,15 +995,13 @@ static void blkcg_iostat_update(struct

blkcg_gq *blkg, struct blkg_iostat *cur,

Quoted 5 lines
block/blk-cgroup.c +0 -1
 struct blkg_iostat *last)
 {
 struct blkg_iostat delta;
-unsigned long flags;
 
Quoted 11 lines

block/blk-cgroup.c static void blkcg_iostat_update(struct blkcg_gq *blkg, struct blkg_iostat *cur, · line 1005

Quoted 9 lines
block/blk-cgroup.c +1 -2
 /* propagate percpu delta to global */
-flags = u64_stats_update_begin_irqsave(&blkg->iostat.sync);
+guard(spinlock_irqsave)(&blkg->iostat.spinlock);
 blkg_iostat_set(&delta, cur);
 blkg_iostat_sub(&delta, last);
 blkg_iostat_add(&blkg->iostat.cur, &delta);
 blkg_iostat_add(last, &delta);
-u64_stats_update_end_irqrestore(&blkg->iostat.sync, flags);
 }

This is already called with blkg_stat_lock held.

In blkcg_iostat_update, it is protected by both blkg_stat_lock and u64 sync. I agree that no changes are needed here.

Quoted 39 lines

Review static void __blkcg_rstat_flush(struct blkcg

Quoted 2 lines
(quoted diff) +0 -0
@@ -1051,10 +1048,8 @@ static void __blkcg_rstat_flush(struct blkcg

*blkcg, int cpu)

block/blk-cgroup.c static void __blkcg_rstat_flush(struct blkcg *blkcg, int cpu) · line 1057

Quoted 8 lines
block/blk-cgroup.c +1 -3
 goto propagate_up; /* propagate up to parent only */
 
 /* fetch the current per-cpu values */
-do {
-seq = u64_stats_fetch_begin(&bisc->sync);
+scoped_guard(spinlock_irqsave, &bisc->spinlock)
 blkg_iostat_set(&cur, &bisc->cur);
-} while (u64_stats_fetch_retry(&bisc->sync, seq));

This is per-cpu stat and we should keep using u64_sync for them.

Quoted 10 lines
block/blk-cgroup.c +1 -2
@@ -1134,9 +1128,8 @@ static void blkcg_fill_root_iostats(void)
 cpu_dkstats->sectors[STAT_DISCARD] << 9;
 }
 
-flags = u64_stats_update_begin_irqsave(&blkg->iostat.sync);
+guard(spinlock_irqsave)(&blkg->iostat.spinlock);
 blkg_iostat_set(&blkg->iostat.cur, &tmp);
-u64_stats_update_end_irqrestore(&blkg->iostat.sync, flags);
 }
 }

...

Review static void blkcg_print_one_stat(struct

Quoted 1 line
(quoted diff) +0 -0
@@ -1157,16 +1149,14 @@ static void blkcg_print_one_stat(struct

blkcg_gq *blkg, struct seq_file *s)

Quoted 12 lines
block/blk-cgroup.c +1 -3
 seq_printf(s, "%s ", dname);
 
-do {
-seq = u64_stats_fetch_begin(&bis->sync);
-
+scoped_guard(spinlock_irqsave, &bis->spinlock) {
 rbytes = bis->cur.bytes[BLKG_IOSTAT_READ];
 wbytes = bis->cur.bytes[BLKG_IOSTAT_WRITE];
 dbytes = bis->cur.bytes[BLKG_IOSTAT_DISCARD];
 rios = bis->cur.ios[BLKG_IOSTAT_READ];
 wios = bis->cur.ios[BLKG_IOSTAT_WRITE];
Quoted 8 lines

block/blk-cgroup.c static void blkcg_print_one_stat(struct blkcg_gq *blkg, struct seq_file *s) · line 1159

Quoted 3 lines
block/blk-cgroup.c +1 -1
 dios = bis->cur.ios[BLKG_IOSTAT_DISCARD];
-} while (u64_stats_fetch_retry(&bis->sync, seq));
+}

The above two are the only places which can potentially benefit from per-blkg lock but these aren't hot paths. I'd just use blkg_stat_lock for
the above.

In __blkcg_rstat_flush, u64 sync is used for fetching data, changing to spinlock will increase overhead both 64 bit and 32 bit systems. 64 bit systems do noting in u64 sync, and 32 bit systems can read data in parallel in u64 sync if no one updating data. However, it is already protected by blkg_stat_lock, so there is no parallelism now, and there is no issue here. I think that no changes are needed here.

In blkcg_fill_root_iostats, this is where the issue happens in 32 bit SMP systems, so spinlock needs to be added to protect it.

In blkcg_print_one_stat, u64 sync is used for fetching data. Changing to spinlock will increase overhead like in __blkcg_rstat_flush. However, it is already protected by spin_lock_irq(&blkg->q->queue_lock) in blkcg_print_stat, so there is no parallelism now, and there is no issue here. I think that no changes are needed here.

Quoted 44 lines

block/blk-cgroup.c void blk_cgroup_bio_start(struct bio *bio) · line 2158

Quoted 9 lines
block/blk-cgroup.c +0 -4
@@ -2152,30 +2141,29 @@ void blk_cgroup_bio_start(struct bio *bio)
 
 cpu = get_cpu();
 bis = per_cpu_ptr(bio->bi_blkg->iostat_cpu, cpu);
-flags = u64_stats_update_begin_irqsave(&bis->sync);
-
-/*
- * If the bio is flagged with BIO_CGROUP_ACCT it means this is a

split

Review void blk_cgroup_bio_start(struct bio *bio) · line 2159

Quoted 1 line
(quoted diff) +0 -1
- * bio and we would have already accounted for the size of the

bio.

block/blk-cgroup.c void blk_cgroup_bio_start(struct bio *bio)

Quoted 9 lines
block/blk-cgroup.c +3 -6
- */
-if (!bio_flagged(bio, BIO_CGROUP_ACCT)) {
-bio_set_flag(bio, BIO_CGROUP_ACCT);
-bis->cur.bytes[rwd] += bio->bi_iter.bi_size;
-}
-bis->cur.ios[rwd]++;
+scoped_guard(spinlock_irqsave, &bis->spinlock) {
+/*
+ * If the bio is flagged with BIO_CGROUP_ACCT it means this is a

split

Review void blk_cgroup_bio_start(struct bio *bio) · line 2147

Quoted 1 line
(quoted diff) +1 -0
+ * bio and we would have already accounted for the size of the

bio.

Quoted 20 lines
block/blk-cgroup.c +12 -6
+ */
+if (!bio_flagged(bio, BIO_CGROUP_ACCT)) {
+bio_set_flag(bio, BIO_CGROUP_ACCT);
+bis->cur.bytes[rwd] += bio->bi_iter.bi_size;
+}
+bis->cur.ios[rwd]++;
 
-/*
- * If the iostat_cpu isn't in a lockless list, put it into the
- * list to indicate that a stat update is pending.
- */
-if (!READ_ONCE(bis->lqueued)) {
-struct llist_head *lhead = this_cpu_ptr(blkcg->lhead);
+/*
+ * If the iostat_cpu isn't in a lockless list, put it into the
+ * list to indicate that a stat update is pending.
+ */
+if (!READ_ONCE(bis->lqueued)) {
+struct llist_head *lhead = this_cpu_ptr(blkcg->lhead);
 
Quoted 11 lines

block/blk-cgroup.c void blk_cgroup_bio_start(struct bio *bio) · line 2164

Quoted 5 lines
block/blk-cgroup.c +3 -2
-llist_add(&bis->lnode, lhead);
-WRITE_ONCE(bis->lqueued, true);
+llist_add(&bis->lnode, lhead);
+WRITE_ONCE(bis->lqueued, true);
+}

These are per-cpu stat updates which should keep using u64_sync. We don't
want to incur locking overhead for stat updates in the hot issue path.

I agree that no changes are needed in blk_cgroup_bio_start.

Quoted 4 lines

Thanks.

Signature · 1 line
tejun

I think we can look back to the original issue, which is that on 32 bit SMP systems will have concurrent problems on u64_stats_update_begin_irqsave and u64_stats_update_end_irqrestore in blkcg_fill_root_iostats. So, adding a lock only on 32 bits systems in blkcg_fill_root_iostats is to resolve the concurrent issue and not affect 64 bit systems, which do not have the issue in the first place, because u64 sync does noting in 64 bit systems and they don't need it.

I think we can just fix it by following change.

@@ -1134,9 +1128,8 @@ static void blkcg_fill_root_iostats(void)

                                cpu_dkstats->sectors[STAT_DISCARD] <<
9;
                }

+#if BITS_PER_LONG == 32
+ guard(spinlock_irqsave)(&blkg_stat_lock);
+#endif
flags = u64_stats_update_begin_irqsave(&blkg-

Quoted 1 line

iostat.sync);

                blkg_iostat_set(&blkg->iostat.cur, &tmp);
                u64_stats_update_end_irqrestore(&blkg->iostat.sync,
flags);
        }

}

#3 lore raw
Quoted 200 lines · Boy.Wu
Quoted 17 lines · Tejun Heo

External email : Please do not click links or open attachments until
you have verified the sender or the content.
Hello, Boy.

So, looking at the patch, I'm not sure per-blkg lock makes sense.

Review static void blkcg_iostat_update(struct

Quoted 1 line · boy.wu
(quoted diff) +0 -0
@@ -995,15 +995,13 @@ static void blkcg_iostat_update(struct

blkcg_gq *blkg, struct blkg_iostat *cur,

Quoted 5 lines
(quoted diff) +0 -1
 struct blkg_iostat *last)
 {
 struct blkg_iostat delta;
-unsigned long flags;
 
Quoted 11 lines

Review static void blkcg_iostat_update(struct · line 1005

Quoted 9 lines
(quoted diff) +1 -2
 /* propagate percpu delta to global */
-flags = u64_stats_update_begin_irqsave(&blkg->iostat.sync);
+guard(spinlock_irqsave)(&blkg->iostat.spinlock);
 blkg_iostat_set(&delta, cur);
 blkg_iostat_sub(&delta, last);
 blkg_iostat_add(&blkg->iostat.cur, &delta);
 blkg_iostat_add(last, &delta);
-u64_stats_update_end_irqrestore(&blkg->iostat.sync, flags);
 }

This is already called with blkg_stat_lock held.

In blkcg_iostat_update, it is protected by both blkg_stat_lock and u64
sync. I agree that no changes are needed here.

Quoted 45 lines
Quoted 3 lines
(quoted diff) +0 -0
@@ -1051,10 +1048,8 @@ static void __blkcg_rstat_flush(struct

blkcg

*blkcg, int cpu)

Review

Quoted 8 lines

goto propagate_up; /* propagate up to parent only */

/* fetch the current per-cpu values */

(quoted diff) +1 -3
-do {
-seq = u64_stats_fetch_begin(&bisc->sync);
+scoped_guard(spinlock_irqsave, &bisc->spinlock)
 blkg_iostat_set(&cur, &bisc->cur);
-} while (u64_stats_fetch_retry(&bisc->sync, seq));

This is per-cpu stat and we should keep using u64_sync for them.

Quoted 10 lines
(quoted diff) +1 -2
@@ -1134,9 +1128,8 @@ static void blkcg_fill_root_iostats(void)
 cpu_dkstats->sectors[STAT_DISCARD] << 9;
 }
 
-flags = u64_stats_update_begin_irqsave(&blkg->iostat.sync);
+guard(spinlock_irqsave)(&blkg->iostat.spinlock);
 blkg_iostat_set(&blkg->iostat.cur, &tmp);
-u64_stats_update_end_irqrestore(&blkg->iostat.sync, flags);
 }
 }

...

Review static void blkcg_print_one_stat(struct

Quoted 1 line
(quoted diff) +0 -0
@@ -1157,16 +1149,14 @@ static void blkcg_print_one_stat(struct

blkcg_gq *blkg, struct seq_file *s)

Quoted 14 lines
(quoted diff) +1 -4
 seq_printf(s, "%s ", dname);
 
-do {
-seq = u64_stats_fetch_begin(&bis->sync);
-
+scoped_guard(spinlock_irqsave, &bis->spinlock) {
 rbytes = bis->cur.bytes[BLKG_IOSTAT_READ];
 wbytes = bis->cur.bytes[BLKG_IOSTAT_WRITE];
 dbytes = bis->cur.bytes[BLKG_IOSTAT_DISCARD];
 rios = bis->cur.ios[BLKG_IOSTAT_READ];
 wios = bis->cur.ios[BLKG_IOSTAT_WRITE];
 dios = bis->cur.ios[BLKG_IOSTAT_DISCARD];
-} while (u64_stats_fetch_retry(&bis->sync, seq));
Quoted 8 lines

Review static void blkcg_print_one_stat(struct · line 1158

Quoted 1 line
(quoted diff) +1 -0
+}

The above two are the only places which can potentially benefit from
per-blkg lock but these aren't hot paths. I'd just use blkg_stat_lock
for
the above.

In __blkcg_rstat_flush, u64 sync is used for fetching data, changing to
spinlock will increase overhead both 64 bit and 32 bit systems. 64 bit
systems do noting in u64 sync, and 32 bit systems can read data in parallel in u64 sync if no one updating data. However, it is already protected by blkg_stat_lock, so there is no parallelism now, and there
is no issue here. I think that no changes are needed here.

In blkcg_fill_root_iostats, this is where the issue happens in 32 bit SMP systems, so spinlock needs to be added to protect it.

In blkcg_print_one_stat, u64 sync is used for fetching data. Changing to spinlock will increase overhead like in __blkcg_rstat_flush. However, it is already protected by spin_lock_irq(&blkg->q-

Quoted 1 line

queue_lock)

in blkcg_print_stat, so there is no parallelism now, and there is no issue here. I think that no changes are needed here.

Quoted 49 lines

Review void blk_cgroup_bio_start(struct bio · line 2159

Quoted 10 lines
(quoted diff) +0 -4
@@ -2152,30 +2141,29 @@ void blk_cgroup_bio_start(struct bio
 *bio)
 
 cpu = get_cpu();
 bis = per_cpu_ptr(bio->bi_blkg->iostat_cpu, cpu);
-flags = u64_stats_update_begin_irqsave(&bis->sync);
-
-/*
- * If the bio is flagged with BIO_CGROUP_ACCT it means this is a

split

Review void blk_cgroup_bio_start(struct bio · line 2160

Quoted 1 line
(quoted diff) +0 -1
- * bio and we would have already accounted for the size of the

bio.

Review void blk_cgroup_bio_start(struct bio · line 2147

Quoted 9 lines
(quoted diff) +3 -6
- */
-if (!bio_flagged(bio, BIO_CGROUP_ACCT)) {
-bio_set_flag(bio, BIO_CGROUP_ACCT);
-bis->cur.bytes[rwd] += bio->bi_iter.bi_size;
-}
-bis->cur.ios[rwd]++;
+scoped_guard(spinlock_irqsave, &bis->spinlock) {
+/*
+ * If the bio is flagged with BIO_CGROUP_ACCT it means this is a

split

Review void blk_cgroup_bio_start(struct bio · line 2148

Quoted 1 line
(quoted diff) +1 -0
+ * bio and we would have already accounted for the size of the

bio.

Quoted 20 lines
(quoted diff) +12 -6
+ */
+if (!bio_flagged(bio, BIO_CGROUP_ACCT)) {
+bio_set_flag(bio, BIO_CGROUP_ACCT);
+bis->cur.bytes[rwd] += bio->bi_iter.bi_size;
+}
+bis->cur.ios[rwd]++;
 
-/*
- * If the iostat_cpu isn't in a lockless list, put it into the
- * list to indicate that a stat update is pending.
- */
-if (!READ_ONCE(bis->lqueued)) {
-struct llist_head *lhead = this_cpu_ptr(blkcg->lhead);
+/*
+ * If the iostat_cpu isn't in a lockless list, put it into the
+ * list to indicate that a stat update is pending.
+ */
+if (!READ_ONCE(bis->lqueued)) {
+struct llist_head *lhead = this_cpu_ptr(blkcg->lhead);
 
Quoted 11 lines

Review void blk_cgroup_bio_start(struct bio · line 2165

Quoted 5 lines
(quoted diff) +3 -2
-llist_add(&bis->lnode, lhead);
-WRITE_ONCE(bis->lqueued, true);
+llist_add(&bis->lnode, lhead);
+WRITE_ONCE(bis->lqueued, true);
+}

These are per-cpu stat updates which should keep using u64_sync. We don't
want to incur locking overhead for stat updates in the hot issue path.

I agree that no changes are needed in blk_cgroup_bio_start.

Quoted 4 lines

Thanks.

Signature · 1 line
tejun

I think we can look back to the original issue, which is that on 32 bit
SMP systems will have concurrent problems on u64_stats_update_begin_irqsave and u64_stats_update_end_irqrestore in blkcg_fill_root_iostats. So, adding a lock only on 32 bits systems in blkcg_fill_root_iostats is to resolve the concurrent issue and not affect 64 bit systems, which do not have the issue in the first place,
because u64 sync does noting in 64 bit systems and they don't need it.

I think we can just fix it by following change.

(quoted diff) +3 -0
@@ -1134,9 +1128,8 @@ static void blkcg_fill_root_iostats(void)
                                cpu_dkstats->sectors[STAT_DISCARD] <<
 9;
                }
 
+#if BITS_PER_LONG == 32
+               guard(spinlock_irqsave)(&blkg_stat_lock);
+#endif
                flags = u64_stats_update_begin_irqsave(&blkg-
Quoted 1 line

iostat.sync);

Quoted 5 lines
                blkg_iostat_set(&blkg->iostat.cur, &tmp);
                u64_stats_update_end_irqrestore(&blkg->iostat.sync,
flags);
        }

}

I think there is another way to fix it. Maybe we should address the root cause, which is that u64_stats_update_begin_irqsave and u64_stats_update_end_irqrestore do not protect the seqcount in 32 bit SMP systems. This can be fix by the following change.

(quoted diff) +0 -0

b/include/linux/u64_stats_sync.h
index 46040d66334a..94dd74b4fb2c 100644

include/linux/u64_stats_sync.h +2 -0
@@ -64,6 +64,7 @@
 struct u64_stats_sync {
 #if BITS_PER_LONG == 32
        seqcount_t      seq;
+       spinlock_t      spinlock;
 #endif
 };
 
@@ -138,6 +139,7 @@ static inline void u64_stats_inc(u64_stats_t *p)
 static inline void u64_stats_init(struct u64_stats_sync *syncp)
 {
        seqcount_init(&syncp->seq);
+       spin_lock_init(&syncp->spinlock)
 }
 
 static inline void __u64_stats_update_begin(struct u64_stats_sync

*syncp)
@@ -191,6 +193,7 @@ static inline unsigned long u64_stats_update_begin_irqsave(struct u64_stats_sync

 {
        unsigned long flags = __u64_stats_irqsave();
+       spin_lock_irq(&syncp->spinlock);
        __u64_stats_update_begin(syncp);
        return flags;

}
@@ -199,6 +202,7 @@ static inline void u64_stats_update_end_irqrestore(struct u64_stats_sync *syncp,
unsigned long flags)

 {
        __u64_stats_update_end(syncp);
+       spin_unlock_irq(&syncp->spinlock);
        __u64_stats_irqrestore(flags);

}

--
boy.wu

#4 lore raw
Quoted 47 lines · Boy.Wu

I think there is another way to fix it. Maybe we should address the root cause, which is that u64_stats_update_begin_irqsave and u64_stats_update_end_irqrestore do not protect the seqcount in 32 bit SMP systems. This can be fix by the following change.

include/linux/u64_stats_sync.h +2 -0
index 46040d66334a..94dd74b4fb2c 100644
@@ -64,6 +64,7 @@
 struct u64_stats_sync {
 #if BITS_PER_LONG == 32
        seqcount_t      seq;
+       spinlock_t      spinlock;
 #endif
 };
 
@@ -138,6 +139,7 @@ static inline void u64_stats_inc(u64_stats_t *p)
 static inline void u64_stats_init(struct u64_stats_sync *syncp)
 {
        seqcount_init(&syncp->seq);
+       spin_lock_init(&syncp->spinlock)
 }
 
 static inline void __u64_stats_update_begin(struct u64_stats_sync

*syncp)

include/linux/u64_stats_sync.h +0 -0
@@ -191,6 +193,7 @@ static inline unsigned long

u64_stats_update_begin_irqsave(struct u64_stats_sync

 {
        unsigned long flags = __u64_stats_irqsave();
include/linux/u64_stats_sync.h +2 -0
+       spin_lock_irq(&syncp->spinlock);
        __u64_stats_update_begin(syncp);
        return flags;
 }
@@ -199,6 +202,7 @@ static inline void
 u64_stats_update_end_irqrestore(struct u64_stats_sync *syncp,
                                                   unsigned long
 flags)
 {
        __u64_stats_update_end(syncp);
+       spin_unlock_irq(&syncp->spinlock);
        __u64_stats_irqrestore(flags);

}

Quoted 3 lines

--
boy.wu

Never mind, there is a usage in u64_stats_sync.h

* Usage :
*
* Stats producer (writer) should use following template granted it
already got
* an exclusive access to counters (a lock is already taken, or per cpu
* data is used [in a non preemptable context])
*

 *   spin_lock_bh(...) or other synchronization to get exclusive access
 *   ...
 *   u64_stats_update_begin(&stats->syncp);
 *   u64_stats_add(&stats->bytes64, len); // non atomic operation
 *   u64_stats_inc(&stats->packets64);    // non atomic operation
 *   u64_stats_update_end(&stats->syncp);

--
boy.wu

#5 lore raw

Hello,

Does something like the following work for you?

Thanks.

block/blk-cgroup.c +15 -19
index 37e6cc91d576..ec1d191f5c83 100644
@@ -329,7 +329,6 @@ static struct blkcg_gq *blkg_alloc(struct blkcg *blkcg, struct gendisk *disk,
 	INIT_WORK(&blkg->async_bio_work, blkg_async_bio_workfn);
 #endif
 
-	u64_stats_init(&blkg->iostat.sync);
 	for_each_possible_cpu(cpu) {
 		u64_stats_init(&per_cpu_ptr(blkg->iostat_cpu, cpu)->sync);
 		per_cpu_ptr(blkg->iostat_cpu, cpu)->blkg = blkg;
@@ -632,24 +631,26 @@ static void blkg_iostat_set(struct blkg_iostat *dst, struct blkg_iostat *src)
 static void __blkg_clear_stat(struct blkg_iostat_set *bis)
 {
 	struct blkg_iostat cur = {0};
-	unsigned long flags;
 
-	flags = u64_stats_update_begin_irqsave(&bis->sync);
 	blkg_iostat_set(&bis->cur, &cur);
 	blkg_iostat_set(&bis->last, &cur);
-	u64_stats_update_end_irqrestore(&bis->sync, flags);
 }
 
 static void blkg_clear_stat(struct blkcg_gq *blkg)
 {
+	unsigned long flags;
 	int cpu;
 
+	raw_spin_lock_irqsave(&blkg_stat_lock, flags);
+
 	for_each_possible_cpu(cpu) {
 		struct blkg_iostat_set *s = per_cpu_ptr(blkg->iostat_cpu, cpu);
 
 		__blkg_clear_stat(s);
 	}
 	__blkg_clear_stat(&blkg->iostat);
+
+	raw_spin_unlock_irqrestore(&blkg_stat_lock, flags);
 }
 
 static int blkcg_reset_stats(struct cgroup_subsys_state *css,
@@ -998,12 +999,10 @@ static void blkcg_iostat_update(struct blkcg_gq *blkg, struct blkg_iostat *cur,
 	unsigned long flags;
 
 	/* propagate percpu delta to global */
-	flags = u64_stats_update_begin_irqsave(&blkg->iostat.sync);
 	blkg_iostat_set(&delta, cur);
 	blkg_iostat_sub(&delta, last);
 	blkg_iostat_add(&blkg->iostat.cur, &delta);
 	blkg_iostat_add(last, &delta);
-	u64_stats_update_end_irqrestore(&blkg->iostat.sync, flags);
 }
 
 static void __blkcg_rstat_flush(struct blkcg *blkcg, int cpu)
@@ -1134,9 +1133,9 @@ static void blkcg_fill_root_iostats(void)
 				cpu_dkstats->sectors[STAT_DISCARD] << 9;
 		}
 
-		flags = u64_stats_update_begin_irqsave(&blkg->iostat.sync);
+		raw_spin_lock_irqsave(&blkg_stat_lock, flags);
 		blkg_iostat_set(&blkg->iostat.cur, &tmp);
-		u64_stats_update_end_irqrestore(&blkg->iostat.sync, flags);
+		raw_spin_unlock_irqrestore(&blkg_stat_lock, flags);
 	}
 }
 
@@ -1145,7 +1144,6 @@ static void blkcg_print_one_stat(struct blkcg_gq *blkg, struct seq_file *s)
 	struct blkg_iostat_set *bis = &blkg->iostat;
 	u64 rbytes, wbytes, rios, wios, dbytes, dios;
 	const char *dname;
-	unsigned seq;
 	int i;
 
 	if (!blkg->online)
@@ -1157,16 +1155,14 @@ static void blkcg_print_one_stat(struct blkcg_gq *blkg, struct seq_file *s)
 
 	seq_printf(s, "%s ", dname);
 
-	do {
-		seq = u64_stats_fetch_begin(&bis->sync);
-
-		rbytes = bis->cur.bytes[BLKG_IOSTAT_READ];
-		wbytes = bis->cur.bytes[BLKG_IOSTAT_WRITE];
-		dbytes = bis->cur.bytes[BLKG_IOSTAT_DISCARD];
-		rios = bis->cur.ios[BLKG_IOSTAT_READ];
-		wios = bis->cur.ios[BLKG_IOSTAT_WRITE];
-		dios = bis->cur.ios[BLKG_IOSTAT_DISCARD];
-	} while (u64_stats_fetch_retry(&bis->sync, seq));
+	raw_spin_lock_irq(&blkg_stat_lock);
+	rbytes = bis->cur.bytes[BLKG_IOSTAT_READ];
+	wbytes = bis->cur.bytes[BLKG_IOSTAT_WRITE];
+	dbytes = bis->cur.bytes[BLKG_IOSTAT_DISCARD];
+	rios = bis->cur.ios[BLKG_IOSTAT_READ];
+	wios = bis->cur.ios[BLKG_IOSTAT_WRITE];
+	dios = bis->cur.ios[BLKG_IOSTAT_DISCARD];
+	raw_spin_unlock_irq(&blkg_stat_lock, flags);
 
 	if (rbytes || wbytes || rios || wios) {
 		seq_printf(s, "rbytes=%llu wbytes=%llu rios=%llu wios=%llu dbytes=%llu dios=%llu",
#6 lore raw
Quoted 97 lines · On 7/17/24 12:55, [email protected] wrote:

Hello,

Does something like the following work for you?

Thanks.

block/blk-cgroup.c +8 -19
index 37e6cc91d576..ec1d191f5c83 100644
@@ -329,7 +329,6 @@ static struct blkcg_gq *blkg_alloc(struct blkcg *blkcg, struct gendisk *disk,
  	INIT_WORK(&blkg->async_bio_work, blkg_async_bio_workfn);
  #endif
 
-	u64_stats_init(&blkg->iostat.sync);
  	for_each_possible_cpu(cpu) {
  		u64_stats_init(&per_cpu_ptr(blkg->iostat_cpu, cpu)->sync);
  		per_cpu_ptr(blkg->iostat_cpu, cpu)->blkg = blkg;
@@ -632,24 +631,26 @@ static void blkg_iostat_set(struct blkg_iostat *dst, struct blkg_iostat *src)
  static void __blkg_clear_stat(struct blkg_iostat_set *bis)
  {
  	struct blkg_iostat cur = {0};
-	unsigned long flags;
 
-	flags = u64_stats_update_begin_irqsave(&bis->sync);
  	blkg_iostat_set(&bis->cur, &cur);
  	blkg_iostat_set(&bis->last, &cur);
-	u64_stats_update_end_irqrestore(&bis->sync, flags);
  }
 
  static void blkg_clear_stat(struct blkcg_gq *blkg)
  {
+	unsigned long flags;
  	int cpu;
 
+	raw_spin_lock_irqsave(&blkg_stat_lock, flags);
+
  	for_each_possible_cpu(cpu) {
  		struct blkg_iostat_set *s = per_cpu_ptr(blkg->iostat_cpu, cpu);
 
  		__blkg_clear_stat(s);
  	}
  	__blkg_clear_stat(&blkg->iostat);
+
+	raw_spin_unlock_irqrestore(&blkg_stat_lock, flags);
  }
 
  static int blkcg_reset_stats(struct cgroup_subsys_state *css,
@@ -998,12 +999,10 @@ static void blkcg_iostat_update(struct blkcg_gq *blkg, struct blkg_iostat *cur,
  	unsigned long flags;
 
  	/* propagate percpu delta to global */
-	flags = u64_stats_update_begin_irqsave(&blkg->iostat.sync);
  	blkg_iostat_set(&delta, cur);
  	blkg_iostat_sub(&delta, last);
  	blkg_iostat_add(&blkg->iostat.cur, &delta);
  	blkg_iostat_add(last, &delta);
-	u64_stats_update_end_irqrestore(&blkg->iostat.sync, flags);
  }
 
  static void __blkcg_rstat_flush(struct blkcg *blkcg, int cpu)
@@ -1134,9 +1133,9 @@ static void blkcg_fill_root_iostats(void)
  				cpu_dkstats->sectors[STAT_DISCARD] << 9;
  		}
 
-		flags = u64_stats_update_begin_irqsave(&blkg->iostat.sync);
+		raw_spin_lock_irqsave(&blkg_stat_lock, flags);
  		blkg_iostat_set(&blkg->iostat.cur, &tmp);
-		u64_stats_update_end_irqrestore(&blkg->iostat.sync, flags);
+		raw_spin_unlock_irqrestore(&blkg_stat_lock, flags);
  	}
  }
 
@@ -1145,7 +1144,6 @@ static void blkcg_print_one_stat(struct blkcg_gq *blkg, struct seq_file *s)
  	struct blkg_iostat_set *bis = &blkg->iostat;
  	u64 rbytes, wbytes, rios, wios, dbytes, dios;
  	const char *dname;
-	unsigned seq;
  	int i;
 
  	if (!blkg->online)
@@ -1157,16 +1155,14 @@ static void blkcg_print_one_stat(struct blkcg_gq *blkg, struct seq_file *s)
 
  	seq_printf(s, "%s ", dname);
 
-	do {
-		seq = u64_stats_fetch_begin(&bis->sync);
-
-		rbytes = bis->cur.bytes[BLKG_IOSTAT_READ];
-		wbytes = bis->cur.bytes[BLKG_IOSTAT_WRITE];
-		dbytes = bis->cur.bytes[BLKG_IOSTAT_DISCARD];
-		rios = bis->cur.ios[BLKG_IOSTAT_READ];
-		wios = bis->cur.ios[BLKG_IOSTAT_WRITE];
-		dios = bis->cur.ios[BLKG_IOSTAT_DISCARD];
-	} while (u64_stats_fetch_retry(&bis->sync, seq));
+	raw_spin_lock_irq(&blkg_stat_lock);

block/blk-cgroup.c static void blkcg_print_one_stat(struct blkcg_gq *blkg, struct seq_file *s) · line 1168

Quoted 10 lines
block/blk-cgroup.c +7 -0
+	rbytes = bis->cur.bytes[BLKG_IOSTAT_READ];
+	wbytes = bis->cur.bytes[BLKG_IOSTAT_WRITE];
+	dbytes = bis->cur.bytes[BLKG_IOSTAT_DISCARD];
+	rios = bis->cur.ios[BLKG_IOSTAT_READ];
+	wios = bis->cur.ios[BLKG_IOSTAT_WRITE];
+	dios = bis->cur.ios[BLKG_IOSTAT_DISCARD];
+	raw_spin_unlock_irq(&blkg_stat_lock, flags);
 
  	if (rbytes || wbytes || rios || wios) {
  		seq_printf(s, "rbytes=%llu wbytes=%llu rios=%llu wios=%llu dbytes=%llu dios=%llu",

bis->sync is still being used in blk_cgroup_bio_start(). Replacing it with a global lock may kill performance. We may have to use a per-cpu lock if we want to go this route of eliminating bis->sync.

Cheers,
Longman

#7 lore raw

Hello, Waiman.

Quoted 3 lines · Waiman Long

bis->sync is still being used in blk_cgroup_bio_start(). Replacing it with a global lock may kill performance. We may have to use a per-cpu lock if we want to go this route of eliminating bis->sync.

So, the idea is to keep using u64_sync for blkg->iostat_cpu and use blkg_stat_lock for blkg->iostat. The former is the only one which is updated in hot path, right?

Thanks.

Signature · 1 line
tejun
#8 lore raw
Quoted 9 lines · On 7/17/24 13:40, [email protected] wrote:

Hello, Waiman.

Quoted 3 lines · Waiman Long

bis->sync is still being used in blk_cgroup_bio_start(). Replacing it with a global lock may kill performance. We may have to use a per-cpu lock if we want to go this route of eliminating bis->sync.

So, the idea is to keep using u64_sync for blkg->iostat_cpu and use blkg_stat_lock for blkg->iostat. The former is the only one which is updated in hot path, right?

Well, it can be confusing whether we are dealing with blkg->iostat or blkg->iostat_cpu. In many cases, we are dealing with iostat_cpu instead of iostat like __blkcg_rstat_flush() and blkg_clear_stat(). So we can't eliminate the use of u64_stats_update_begin_irqsave() in those cases.

Cheers,
Longman

#9 lore raw

Hello,

Quoted 4 lines · Waiman Long

Well, it can be confusing whether we are dealing with blkg->iostat or blkg->iostat_cpu. In many cases, we are dealing with iostat_cpu instead of iostat like __blkcg_rstat_flush() and blkg_clear_stat(). So we can't eliminate the use of u64_stats_update_begin_irqsave() in those cases.

I mean, we need to distinguish them. For 32bits, blkg->iostat has multiple updaters, so we can't use u64_sync; however, blkg->iostat_cpu has only one updater (except blkg_clear_stat() which I don't think we need to worry too much about), so u64_sync is fine.

Thanks.

Signature · 1 line
tejun
#10 lore raw
Quoted 11 lines · On 7/17/24 14:24, [email protected] wrote:

Hello,

Quoted 4 lines · Waiman Long

Well, it can be confusing whether we are dealing with blkg->iostat or blkg->iostat_cpu. In many cases, we are dealing with iostat_cpu instead of iostat like __blkcg_rstat_flush() and blkg_clear_stat(). So we can't eliminate the use of u64_stats_update_begin_irqsave() in those cases.

I mean, we need to distinguish them. For 32bits, blkg->iostat has multiple updaters, so we can't use u64_sync; however, blkg->iostat_cpu has only one updater (except blkg_clear_stat() which I don't think we need to worry too much about), so u64_sync is fine.

I was wrong about __blkcg_rstat_flush(). Right, the main updater of iostat_cpu is  blk_cgroup_bio_start(). We do need to drop down some comment on what is protected by u64_sync and what is by blkg_stat_lock though. It can be confusing.

Cheers,
Longman

#11 lore raw
Quoted 4 lines · Waiman Long

I was wrong about __blkcg_rstat_flush(). Right, the main updater of iostat_cpu is  blk_cgroup_bio_start(). We do need to drop down some comment on what is protected by u64_sync and what is by blkg_stat_lock though. It can be confusing.

Oh yeah, definitely. It is confusing.

Signature · 1 line
tejun
#12 lore raw
In reply to [email protected]
Quoted 113 lines · On Wed, 2024-07-17 at 06:55 -1000, [email protected] wrote:

External email : Please do not click links or open attachments until you have verified the sender or the content.
Hello,

Does something like the following work for you?

Thanks.

block/blk-cgroup.c +0 -1
index 37e6cc91d576..ec1d191f5c83 100644
@@ -329,7 +329,6 @@ static struct blkcg_gq *blkg_alloc(struct blkcg
 *blkcg, struct gendisk *disk,
 INIT_WORK(&blkg->async_bio_work, blkg_async_bio_workfn);
 #endif
 
-u64_stats_init(&blkg->iostat.sync);
 for_each_possible_cpu(cpu) {
 u64_stats_init(&per_cpu_ptr(blkg->iostat_cpu, cpu)->sync);

per_cpu_ptr(blkg->iostat_cpu, cpu)->blkg = blkg;

block/blk-cgroup.c +5 -3
@@ -632,24 +631,26 @@ static void blkg_iostat_set(struct blkg_iostat
 *dst, struct blkg_iostat *src)
 static void __blkg_clear_stat(struct blkg_iostat_set *bis)
 {
 struct blkg_iostat cur = {0};
-unsigned long flags;
 
-flags = u64_stats_update_begin_irqsave(&bis->sync);
 blkg_iostat_set(&bis->cur, &cur);
 blkg_iostat_set(&bis->last, &cur);
-u64_stats_update_end_irqrestore(&bis->sync, flags);
 }
 
 static void blkg_clear_stat(struct blkcg_gq *blkg)
 {
+unsigned long flags;
 int cpu;
 
+raw_spin_lock_irqsave(&blkg_stat_lock, flags);
+
 for_each_possible_cpu(cpu) {
 struct blkg_iostat_set *s = per_cpu_ptr(blkg->iostat_cpu, cpu);
 
 __blkg_clear_stat(s);
 }
 __blkg_clear_stat(&blkg->iostat);
+
+raw_spin_unlock_irqrestore(&blkg_stat_lock, flags);
 }
 

static int blkcg_reset_stats(struct cgroup_subsys_state *css,

block/blk-cgroup.c +0 -2
@@ -998,12 +999,10 @@ static void blkcg_iostat_update(struct blkcg_gq
 *blkg, struct blkg_iostat *cur,
 unsigned long flags;
 
 /* propagate percpu delta to global */
-flags = u64_stats_update_begin_irqsave(&blkg->iostat.sync);
 blkg_iostat_set(&delta, cur);
 blkg_iostat_sub(&delta, last);
 blkg_iostat_add(&blkg->iostat.cur, &delta);
 blkg_iostat_add(last, &delta);
-u64_stats_update_end_irqrestore(&blkg->iostat.sync, flags);
 }
 

static void __blkcg_rstat_flush(struct blkcg *blkcg, int cpu)

block/blk-cgroup.c +2 -3
@@ -1134,9 +1133,9 @@ static void blkcg_fill_root_iostats(void)
 cpu_dkstats->sectors[STAT_DISCARD] << 9;
 }
 
-flags = u64_stats_update_begin_irqsave(&blkg->iostat.sync);
+raw_spin_lock_irqsave(&blkg_stat_lock, flags);
 blkg_iostat_set(&blkg->iostat.cur, &tmp);
-u64_stats_update_end_irqrestore(&blkg->iostat.sync, flags);
+raw_spin_unlock_irqrestore(&blkg_stat_lock, flags);
 }
 }
 
@@ -1145,7 +1144,6 @@ static void blkcg_print_one_stat(struct
 blkcg_gq *blkg, struct seq_file *s)
 struct blkg_iostat_set *bis = &blkg->iostat;
 u64 rbytes, wbytes, rios, wios, dbytes, dios;
 const char *dname;
-unsigned seq;
 int i;
 

if (!blkg->online)

block/blk-cgroup.c +8 -10
@@ -1157,16 +1155,14 @@ static void blkcg_print_one_stat(struct
 blkcg_gq *blkg, struct seq_file *s)
 
 seq_printf(s, "%s ", dname);
 
-do {
-seq = u64_stats_fetch_begin(&bis->sync);
-
-rbytes = bis->cur.bytes[BLKG_IOSTAT_READ];
-wbytes = bis->cur.bytes[BLKG_IOSTAT_WRITE];
-dbytes = bis->cur.bytes[BLKG_IOSTAT_DISCARD];
-rios = bis->cur.ios[BLKG_IOSTAT_READ];
-wios = bis->cur.ios[BLKG_IOSTAT_WRITE];
-dios = bis->cur.ios[BLKG_IOSTAT_DISCARD];
-} while (u64_stats_fetch_retry(&bis->sync, seq));
+raw_spin_lock_irq(&blkg_stat_lock);
+rbytes = bis->cur.bytes[BLKG_IOSTAT_READ];
+wbytes = bis->cur.bytes[BLKG_IOSTAT_WRITE];
+dbytes = bis->cur.bytes[BLKG_IOSTAT_DISCARD];
+rios = bis->cur.ios[BLKG_IOSTAT_READ];
+wios = bis->cur.ios[BLKG_IOSTAT_WRITE];
+dios = bis->cur.ios[BLKG_IOSTAT_DISCARD];
+raw_spin_unlock_irq(&blkg_stat_lock, flags);
 
 if (rbytes || wbytes || rios || wios) {
Quoted 2 lines

seq_printf(s, "rbytes=%llu wbytes=%llu rios=%llu wios=%llu
dbytes=%llu dios=%llu",

I think this will work, but as I mentioned before, this issue is only on 32 bit SMP systems. Replacing u64 sync with spinlock will increase overhead on 64 bit systems, because u64 sync does nothing on 64 bit systems. However, if it is acceptable, we can proceed with this solution.

--
boy.wu

#13 lore raw

Hello,

On Thu, Jul 18, 2024 at 01:21:32AM +0000, Boy Wu (吳勃誼) wrote:
...

Quoted 5 lines

I think this will work, but as I mentioned before, this issue is only on 32 bit SMP systems. Replacing u64 sync with spinlock will increase overhead on 64 bit systems, because u64 sync does nothing on 64 bit systems. However, if it is acceptable, we can proceed with this solution.

We can encapsulate the spinlock in some helpers and make them conditional on 32bit. However, the u64_sync -> spinlock conversions in the suggested patch are all on cold paths, so I doubt it'd be all that noticeable especially given that the hottest path of them all is already grabbing blkg_stat_lock, but either way is fine by me.

Thanks.

Signature · 1 line
tejun