Change some counters to percpu type
cover Change some counters to percpu type
Hi all
There are two main changes in the patch set.
The first is to change ->active_io to percpu(patch02). The second one is adding a counter for io_acct(patch03).
Xiao Ni (4):
Factor out is_md_suspended helper
Change active_io to percpu
Add mddev->io_acct_cnt for raid0_quiesce
Free writes_pending in md_stop
drivers/md/md.c | 69 ++++++++++++++++++++++++++++++++++------------ drivers/md/md.h | 11 +++++--- drivers/md/raid0.c | 6 ++++ 3 files changed, 65 insertions(+), 21 deletions(-)
Signature · 1 line
2.32.0 (Apple Git-132)
PATCH 1/4 Factor out is_md_suspended helper
This helper function will be used in next patch. It's easy for understanding.
--- drivers/md/md.c | 17 ++++++++++++----- 1 file changed, 12 insertions(+), 5 deletions(-)
drivers/md/md.c +12 -5
@@ -380,6 +380,13 @@ EXPORT_SYMBOL_GPL(md_new_event);static LIST_HEAD (all_mddevs);static DEFINE_SPINLOCK (all_mddevs_lock); +static bool is_md_suspended (struct mddev *mddev) +{ +if (mddev->suspended) +return true; +else +return false; +}/* Rather than calling directly into the personality make_request function, * IO requests come here first so that we can check if the device is * being suspended pending a reconfiguration.
@@ -389,7 +396,7 @@ static DEFINE_SPINLOCK(all_mddevs_lock);*/ static bool is_suspended (struct mddev *mddev,struct bio *bio) { -if (mddev->suspended) +if (is_md_suspended (mddev))return true;if (bio_data_dir (bio) !=WRITE )return false;
@@ -434,7 +441,7 @@ void md_handle_request(struct mddev *mddev, struct bio *bio) goto check_suspended; } -if (atomic_dec_and_test (&mddev->active_io) && mddev->suspended) +if (atomic_dec_and_test (&mddev->active_io) &&is_md_suspended (mddev))wake_up (&mddev->sb_wait); }EXPORT_SYMBOL (md_handle_request);
@@ -6217,7 +6224,7 @@ EXPORT_SYMBOL_GPL(md_stop_writes);static void mddev_detach (struct mddev *mddev) {md_bitmap_wait_behind_writes (mddev); -if (mddev->pers && mddev->pers->quiesce && !mddev->suspended) { +if (mddev->pers && mddev->pers->quiesce && !is_md_suspended (mddev)) { mddev->pers->quiesce (mddev,1 ); mddev->pers->quiesce (mddev,0 ); }
@@ -8529,7 +8536,7 @@ bool md_write_start(struct mddev *mddev, struct bio *bi)return true;wait_event (mddev->sb_wait, !test_bit (MD_SB_CHANGE_PENDING , &mddev->sb_flags) || - mddev->suspended); +is_md_suspended (mddev));if (test_bit (MD_SB_CHANGE_PENDING , &mddev->sb_flags)) {percpu_ref_put (&mddev->writes_pending);return false;
@@ -9257,7 +9264,7 @@ void md_check_recovery(struct mddev *mddev)wake_up (&mddev->sb_wait); } -if (mddev->suspended) +if (is_md_suspended (mddev))return ;if (mddev->bitmap)
Signature · 1 line
2.32.0 (Apple Git-132)
PATCH 2/4 Change active_io to percpu
In reply to Xiao NiNow the type of active_io is atomic. It's used to count how many ios are in the submitting process and it's added and decreased very time. But it only needs to check if it's zero when suspending the raid. So we can switch atomic to percpu to improve the performance.
After switching to percpu, the raid device is suspended when active_io is not in percpu mode. mddev->suspended is used to count how many users are trying to set raid to suspend state.
--- drivers/md/md.c | 39 +++++++++++++++++++++++---------------- drivers/md/md.h | 2 +- 2 files changed, 24 insertions(+), 17 deletions(-)
drivers/md/md.c +23 -16
@@ -382,10 +382,7 @@ static DEFINE_SPINLOCK(all_mddevs_lock);static bool is_md_suspended (struct mddev *mddev) { -if (mddev->suspended) -return true; -else -return false; +return percpu_ref_is_dying (&mddev->active_io); }/* Rather than calling directly into the personality make_request function, * IO requests come here first so that we can check if the device is
@@ -412,7 +409,6 @@ static bool is_suspended(struct mddev *mddev, struct bio *bio)void md_handle_request (struct mddev *mddev,struct bio *bio) { check_suspended: -rcu_read_lock ();if (is_suspended (mddev, bio)) {DEFINE_WAIT (__wait);/* Bail out if REQ_NOWAIT is set for the bio */
@@ -432,17 +428,15 @@ void md_handle_request(struct mddev *mddev, struct bio *bio) }finish_wait (&mddev->sb_wait, &__wait); } -atomic_inc (&mddev->active_io); -rcu_read_unlock (); +if (!percpu_ref_tryget_live (&mddev->active_io)) + goto check_suspended;if (!mddev->pers->make_request (mddev, bio)) { -atomic_dec (&mddev->active_io); -wake_up (&mddev->sb_wait); +percpu_ref_put (&mddev->active_io); goto check_suspended; } -if (atomic_dec_and_test (&mddev->active_io) &&is_md_suspended (mddev)) -wake_up (&mddev->sb_wait); +percpu_ref_put (&mddev->active_io); }EXPORT_SYMBOL (md_handle_request);
@@ -488,11 +482,10 @@ void mddev_suspend(struct mddev *mddev)lockdep_assert_held (&mddev->reconfig_mutex);if (mddev->suspended++)return ; -synchronize_rcu ();wake_up (&mddev->sb_wait);set_bit (MD_ALLOW_SB_UPDATE , &mddev->flags); -smp_mb__after_atomic (); -wait_event (mddev->sb_wait,atomic_read (&mddev->active_io) ==0 ); +percpu_ref_kill (&mddev->active_io); +wait_event (mddev->sb_wait,percpu_ref_is_zero (&mddev->active_io)); mddev->pers->quiesce (mddev,1 );clear_bit_unlock (MD_ALLOW_SB_UPDATE , &mddev->flags);wait_event (mddev->sb_wait, !test_bit (MD_UPDATING_SB , &mddev->flags));
@@ -510,6 +503,7 @@ void mddev_resume(struct mddev *mddev)lockdep_assert_held (&mddev->reconfig_mutex);if (--mddev->suspended)return ; +percpu_ref_resurrect (&mddev->active_io);wake_up (&mddev->sb_wait); mddev->pers->quiesce (mddev,0 );
@@ -688,7 +682,6 @@ void mddev_init(struct mddev *mddev)timer_setup (&mddev->safemode_timer, md_safemode_timeout,0 );atomic_set (&mddev->active,1 );atomic_set (&mddev->openers,0 ); -atomic_set (&mddev->active_io,0 );spin_lock_init (&mddev->lock);atomic_set (&mddev->flush_pending,0 );init_waitqueue_head (&mddev->sb_wait);
@@ -5765,6 +5758,11 @@ static void md_safemode_timeout(struct timer_list *t) }static int start_dirty_degraded; +static void active_io_release (struct percpu_ref *ref) +{ + struct mddev *mddev = container_of(ref, struct mddev, active_io); +wake_up (&mddev->sb_wait); +}int md_run (struct mddev *mddev) {
@@ -5845,10 +5843,15 @@ int md_run(struct mddev *mddev) nowait = nowait &&bdev_nowait (rdev->bdev); } + err =percpu_ref_init (&mddev->active_io, active_io_release, +PERCPU_REF_ALLOW_REINIT ,GFP_KERNEL ); +if (err) +return err; +if (!bioset_initialized (&mddev->bio_set)) { err =bioset_init (&mddev->bio_set,BIO_POOL_SIZE ,0 ,BIOSET_NEED_BVECS );if (err) -return err; + goto exit_active_io; }if (!bioset_initialized (&mddev->sync_set)) { err =bioset_init (&mddev->sync_set,BIO_POOL_SIZE ,0 ,BIOSET_NEED_BVECS );
@@ -6036,6 +6039,8 @@ int md_run(struct mddev *mddev)bioset_exit (&mddev->sync_set); exit_bio_set:bioset_exit (&mddev->bio_set); +exit_active_io: +percpu_ref_exit (&mddev->active_io);return err; }EXPORT_SYMBOL_GPL (md_run);
@@ -6260,6 +6265,7 @@ void md_stop(struct mddev *mddev)*/ __md_stop_writes (mddev);__md_stop (mddev); +percpu_ref_exit (&mddev->active_io);bioset_exit (&mddev->bio_set);bioset_exit (&mddev->sync_set); }
@@ -7833,6 +7839,7 @@ static void md_free_disk(struct gendisk *disk)struct mddev *mddev = disk->private_data;percpu_ref_exit (&mddev->writes_pending); +percpu_ref_exit (&mddev->active_io);bioset_exit (&mddev->bio_set);bioset_exit (&mddev->sync_set);
drivers/md/md.h +1 -1
@@ -315,7 +315,7 @@ struct mddev {unsigned long sb_flags;int suspended; -atomic_t active_io; +struct percpu_ref active_io;int ro;int sysfs_active;/* set when sysfs deletes * are happening, so run/
Signature · 1 line
2.32.0 (Apple Git-132)
PATCH 3/4 Add mddev->io_acct_cnt for raid0_quiesce
In reply to Xiao NiIt has added io_acct_set for raid0/raid5 io accounting and it needs to alloc md_io_acct in the i/o path. They are free when the bios come back from member disks. Now we don't have a method to monitor if those bios are all come back. In the takeover process, it needs to free the raid0 memory resource including the memory pool for md_io_acct. But maybe some bios are still not returned. When those bios are returned, it can cause panic bcause of introducing NULL pointer or invalid address.
This patch adds io_acct_cnt. So when stopping raid0, it can use this to wait until all bios come back. And I did a simple performance test with fio:
-direct=1 -ioengine=libaio -iodepth=128 -bs=64K -rw=write -numjobs=1 With the patch set: 2676MB/s, without the patch set: 2670MB/s -direct=1 -ioengine=libaio -iodepth=128 -bs=64K -rw=read -numjobs=1 With the patch set: 4676MB/s, without the patch set: 4654MB/s
--- drivers/md/md.c | 22 +++++++++++++++++++++- drivers/md/md.h | 9 ++++++--- drivers/md/raid0.c | 6 ++++++ 3 files changed, 33 insertions(+), 4 deletions(-)
drivers/md/md.c +21 -1
@@ -686,6 +686,7 @@ void mddev_init(struct mddev *mddev)atomic_set (&mddev->flush_pending,0 );init_waitqueue_head (&mddev->sb_wait);init_waitqueue_head (&mddev->recovery_wait); +init_waitqueue_head (&mddev->wait_io_acct); mddev->reshape_position = MaxSector; mddev->reshape_backwards =0 ; mddev->last_sync_action ="none" ;
@@ -8606,13 +8607,27 @@ void md_submit_discard_bio(struct mddev *mddev, struct md_rdev *rdev, }EXPORT_SYMBOL_GPL (md_submit_discard_bio); +static void io_acct_release (struct percpu_ref *ref) +{ + struct mddev *mddev = container_of(ref, struct mddev, io_acct_cnt); +wake_up (&mddev->wait_io_acct); +} +int acct_bioset_init (struct mddev *mddev) {int err =0 ; -if (!bioset_initialized (&mddev->io_acct_set)) +if (!bioset_initialized (&mddev->io_acct_set)) { + err =percpu_ref_init (&mddev->io_acct_cnt, io_acct_release, +PERCPU_REF_ALLOW_REINIT ,GFP_KERNEL ); +if (err) +return err; + err =bioset_init (&mddev->io_acct_set,BIO_POOL_SIZE , offsetof(struct md_io_acct , bio_clone),0 ); +if (err) +percpu_ref_exit (&mddev->io_acct_cnt); + }return err; }EXPORT_SYMBOL_GPL (acct_bioset_init);
@@ -8620,6 +8635,7 @@ EXPORT_SYMBOL_GPL(acct_bioset_init);void acct_bioset_exit (struct mddev *mddev) {bioset_exit (&mddev->io_acct_set); +percpu_ref_exit (&mddev->io_acct_cnt); }EXPORT_SYMBOL_GPL (acct_bioset_exit);
@@ -8627,9 +8643,11 @@ static void md_end_io_acct(struct bio *bio) {struct md_io_acct *md_io_acct = bio->bi_private;struct bio *orig_bio = md_io_acct->orig_bio; +struct mddev *mddev = md_io_acct->mddev; orig_bio->bi_status = bio->bi_status; +percpu_ref_put (&mddev->io_acct_cnt);bio_end_io_acct (orig_bio, md_io_acct->start_time);bio_put (bio);bio_endio (orig_bio);
@@ -8652,6 +8670,8 @@ void md_account_bio(struct mddev *mddev, struct bio **bio) md_io_acct = container_of(clone, struct md_io_acct, bio_clone); md_io_acct->orig_bio = *bio; md_io_acct->start_time =bio_start_io_acct (*bio); + md_io_acct->mddev = mddev; +percpu_ref_get (&mddev->io_acct_cnt); clone->bi_end_io = md_end_io_acct; clone->bi_private = md_io_acct;
drivers/md/md.h +6 -3
@@ -513,6 +513,8 @@ struct mddev {* metadata and bitmap writes */ struct bio_set io_acct_set;/* for raid0 and raid5 io accounting */ +struct percpu_ref io_acct_cnt; +wait_queue_head_t wait_io_acct;/* Generic flush handling. * The last to finish preflush schedules a worker to submit
@@ -710,9 +712,10 @@ struct md_thread { };struct md_io_acct { -struct bio *orig_bio; -unsigned long start_time; -struct bio bio_clone; +struct mddev *mddev; +struct bio *orig_bio; +unsigned long start_time; +struct bio bio_clone; };#define THREAD_WAKEUP 0
drivers/md/raid0.c +6 -0
@@ -753,6 +753,12 @@ static void *raid0_takeover(struct mddev *mddev)static void raid0_quiesce (struct mddev *mddev,int quiesce) { +if (quiesce) { +percpu_ref_kill (&mddev->io_acct_cnt); +wait_event (mddev->wait_io_acct, +percpu_ref_is_zero (&mddev->io_acct_cnt)); + }else +percpu_ref_resurrect (&mddev->io_acct_cnt); } static struct md_personality raid0_personality=
Signature · 1 line
2.32.0 (Apple Git-132)
PATCH 4/4 Free writes_pending in md_stop
In reply to Xiao Nidm raid calls md_stop to stop the raid device. It needs to free the writes_pending here.
--- drivers/md/md.c | 1 + 1 file changed, 1 insertion(+)
drivers/md/md.c +1 -0
@@ -6266,6 +6266,7 @@ void md_stop(struct mddev *mddev)*/ __md_stop_writes (mddev);__md_stop (mddev); +percpu_ref_exit (&mddev->writes_pending);percpu_ref_exit (&mddev->active_io);bioset_exit (&mddev->bio_set);bioset_exit (&mddev->sync_set);
Signature · 1 line
2.32.0 (Apple Git-132)
Song Liu
maintainerQuoted 8 lines · Xiao Ni
Hi all
There are two main changes in the patch set.
The first is to change ->active_io to percpu(patch02). The second one is adding a counter for
io_acct(patch03).
Hi Xiao,
If I understand correctly,
Quoted 3 lines
Xiao Ni (4):
Factor outis_md_suspendedhelper
Changeactive_ioto percpu
Patch 1 and 2 are performance optimization?
Quoted 2 lines
Add
mddev->io_acct_cntforraid0_quiesce
Freewrites_pendinginmd_stop
And patch 3 and 4 fixes two issues?
It is probably better to send them as 3 separate patches (sets).
Also, please provide more information on why we need these changes.
Thanks,
Song
Quoted 9 lines
drivers/md/md.c | 69 ++++++++++++++++++++++++++++++++++------------ drivers/md/md.h | 11 +++++--- drivers/md/raid0.c | 6 ++++ 3 files changed, 65 insertions(+), 21 deletions(-)--
2.32.0 (Apple Git-132)
Quoted 13 lines · Song Liu
Quoted 8 lines · Xiao Ni
Hi all
There are two main changes in the patch set.
The first is to change ->active_io to percpu(patch02). The second one is adding a counter for
io_acct(patch03).Hi Xiao,
Quoted 7 lines
If I understand correctly,
Quoted 3 lines
Xiao Ni (4):
Factor outis_md_suspendedhelper
Changeactive_ioto percpuPatch 1 and 2 are performance optimization?
Hi Song
Yes. These two patches are optimized for performance. At first, I thought to use the active_io to fix the raid0 bug. But we can't move acitve_io out from md_handle_request. So I used a new counter to monitor the bios that raid0 submits to member disks.
Quoted 7 lines
Quoted 3 lines
Add
mddev->io_acct_cntforraid0_quiesce
Freewrites_pendinginmd_stopAnd patch 3 and 4 fixes two issues?
It is probably better to send them as 3 separate patches (sets).
Yes, they fix two bugs. I'll divide them into 3 patch sets.
Quoted 3 lines
Also, please provide more information on why we need these changes.
Sure, I've written the reason in the specific patch.
Best Regards
Xiao
Quoted 14 lines
Thanks,
SongQuoted 9 lines
drivers/md/md.c | 69 ++++++++++++++++++++++++++++++++++------------ drivers/md/md.h | 11 +++++--- drivers/md/raid0.c | 6 ++++ 3 files changed, 65 insertions(+), 21 deletions(-)--
2.32.0 (Apple Git-132)