Optimize wait_for_overlap
cover Optimize wait_for_overlap
The wait_for_overlap wait queue is currently used in two cases, which are not really related:
- waiting for actual overlapping bios, which uses R5_Overlap bit,
- waiting for events related to reshape.
Handling every write request in raid5_make_request() involves adding to and removing from this wait queue, which uses a spinlock. With fast storage and multiple submitting threads the contention on this lock is noticeable.
This patch series aims to resolve this by separating the two cases mentioned above and using this wait queue only when reshape is in progress.
The results when testing 4k random writes on raid5 with null_blk (8 jobs, qd=64, group_thread_cnt=8):
before: 463k IOPS after: 523k IOPS
The improvement is not huge with this series alone but it is just one of the bottlenecks. When applied onto some other changes I'm working on, it allowed to go from 845k IOPS to 975k IOPS on the same test.
Artur Paszkiewicz (3):
md/raid5: use wait_on_bit() for R5_Overlap md/raid5: only add to wait queue if reshape is in progress md/raid5: rename wait_for_overlap to wait_for_reshape
drivers/md/raid5-cache.c | 6 +-- drivers/md/raid5.c | 95 +++++++++++++++++++++------------------- drivers/md/raid5.h | 2 +- 3 files changed, 52 insertions(+), 51 deletions(-)
Signature · 1 line
2.43.0
PATCH 1/3 md/raid5: use wait_on_bit() for R5_Overlap
Convert uses of wait_for_overlap wait queue with R5_Overlap bit to wait_on_bit() / wake_up_bit().
--- drivers/md/raid5-cache.c | 6 +--- drivers/md/raid5.c | 60 +++++++++++++++++++--------------------- 2 files changed, 30 insertions(+), 36 deletions(-)
drivers/md/raid5-cache.c +1 -5
@@ -2798,7 +2798,6 @@ void r5c_finish_stripe_write_out(struct r5conf *conf, {struct r5l_log *log =READ_ONCE (conf->log);int i; -int do_wakeup =0 ;sector_t tree_index;void __rcu **pslot;uintptr_t refcount;
@@ -2815,7 +2814,7 @@ void r5c_finish_stripe_write_out(struct r5conf *conf,for (i = sh->disks; i--; ) {clear_bit (R5_InJournal, &sh->dev[i].flags);if (test_and_clear_bit (R5_Overlap, &sh->dev[i].flags)) - do_wakeup =1 ; +wake_up_bit (&sh->dev[i].flags, R5_Overlap); }/*
@@ -2828,9 +2827,6 @@ void r5c_finish_stripe_write_out(struct r5conf *conf,if (atomic_dec_and_test (&conf->pending_full_writes))md_wakeup_thread (conf->mddev->thread); -if (do_wakeup) -wake_up (&conf->wait_for_overlap); -spin_lock_irq (&log->stripe_in_journal_lock);list_del_init (&sh->r5c);spin_unlock_irq (&log->stripe_in_journal_lock);
drivers/md/raid5.c +29 -31
168 lines not on this page · raw message
Signature · 1 line
2.43.0
PATCH 2/3 md/raid5: only add to wq if reshape is in progress
In reply to Artur PaszkiewiczNow that actual overlaps are not handled on the wait_for_overlap wq anymore, the remaining cases when we wait on this wq are limited to reshape. If reshape is not in progress, don't add to the wq in raid5_make_request() because add_wait_queue() / remove_wait_queue() operations take a spinlock and cause noticeable contention when multiple threads are submitting requests to the mddev.
--- drivers/md/raid5.c | 13 ++++++++++--- 1 file changed, 10 insertions(+), 3 deletions(-)
drivers/md/raid5.c +10 -3
@@ -6071,6 +6071,7 @@ static sector_t raid5_bio_lowest_chunk_sector(struct r5conf *conf,static bool raid5_make_request (struct mddev *mddev,struct bio * bi) {DEFINE_WAIT_FUNC (wait, woken_wake_function); +bool on_wq;struct r5conf *conf = mddev->private;sector_t logical_sector;struct stripe_request_ctx ctx = {};
@@ -6144,11 +6145,15 @@ static bool raid5_make_request(struct mddev *mddev, struct bio * bi)* sequential IO pattern. We don't bother with the optimization when * reshaping as the performance benefit is not worth the complexity. */ -if (likely (conf->reshape_progress == MaxSector)) +if (likely (conf->reshape_progress == MaxSector)) { logical_sector =raid5_bio_lowest_chunk_sector (conf, bi); + on_wq = false; + }else { +add_wait_queue (&conf->wait_for_overlap, &wait); + on_wq = true; + } s = (logical_sector - ctx.first_sector) >>RAID5_STRIPE_SHIFT (conf); -add_wait_queue (&conf->wait_for_overlap, &wait);while (1 ) { res =make_stripe_request (mddev, conf, &ctx, logical_sector, bi);
@@ -6159,6 +6164,7 @@ static bool raid5_make_request(struct mddev *mddev, struct bio * bi)continue ;if (res ==STRIPE_SCHEDULE_AND_RETRY ) { +WARN_ON_ONCE (!on_wq);/* * Must release the reference to batch_last before * scheduling and waiting for work to be done,
@@ -6183,7 +6189,8 @@ static bool raid5_make_request(struct mddev *mddev, struct bio * bi) logical_sector = ctx.first_sector + (s <<RAID5_STRIPE_SHIFT (conf)); } -remove_wait_queue (&conf->wait_for_overlap, &wait); +if (unlikely (on_wq)) +remove_wait_queue (&conf->wait_for_overlap, &wait);if (ctx.batch_last)raid5_release_stripe (ctx.batch_last);
Signature · 1 line
2.43.0
PATCH 3/3 md/raid5: rename wait_for_overlap to wait_for_reshape
In reply to Artur PaszkiewiczThe only remaining uses of wait_for_overlap are related to reshape so rename it accordingly.
--- drivers/md/raid5.c | 26 +++++++++++++------------- drivers/md/raid5.h | 2 +- 2 files changed, 14 insertions(+), 14 deletions(-)
drivers/md/raid5.c +13 -13
@@ -5255,7 +5255,7 @@ static void handle_stripe(struct stripe_head *sh) }else if (s.expanded && !sh->reconstruct_state && s.locked ==0 ) {clear_bit (STRIPE_EXPAND_READY , &sh->state);atomic_dec (&conf->reshape_stripes); -wake_up (&conf->wait_for_overlap); +wake_up (&conf->wait_for_reshape);md_done_sync (conf->mddev,RAID5_STRIPE_SECTORS (conf),1 ); }
@@ -6149,7 +6149,7 @@ static bool raid5_make_request(struct mddev *mddev, struct bio * bi) logical_sector =raid5_bio_lowest_chunk_sector (conf, bi); on_wq = false; }else { -add_wait_queue (&conf->wait_for_overlap, &wait); +add_wait_queue (&conf->wait_for_reshape, &wait); on_wq = true; } s = (logical_sector - ctx.first_sector) >>RAID5_STRIPE_SHIFT (conf);
@@ -6190,7 +6190,7 @@ static bool raid5_make_request(struct mddev *mddev, struct bio * bi) (s <<RAID5_STRIPE_SHIFT (conf)); }if (unlikely (on_wq)) -remove_wait_queue (&conf->wait_for_overlap, &wait); +remove_wait_queue (&conf->wait_for_reshape, &wait);if (ctx.batch_last)raid5_release_stripe (ctx.batch_last);
@@ -6343,7 +6343,7 @@ static sector_t reshape_request(struct mddev *mddev, sector_t sector_nr, int *sk : (safepos < writepos && readpos > writepos)) || time_after(jiffies, conf->reshape_checkpoint + 10*HZ)) {/* Cannot proceed until we've updated the superblock... */ -wait_event (conf->wait_for_overlap, +wait_event (conf->wait_for_reshape,atomic_read (&conf->reshape_stripes)==0 ||test_bit (MD_RECOVERY_INTR , &mddev->recovery));if (atomic_read (&conf->reshape_stripes) !=0 )
@@ -6369,7 +6369,7 @@ static sector_t reshape_request(struct mddev *mddev, sector_t sector_nr, int *skspin_lock_irq (&conf->device_lock); conf->reshape_safe = mddev->reshape_position;spin_unlock_irq (&conf->device_lock); -wake_up (&conf->wait_for_overlap); +wake_up (&conf->wait_for_reshape);sysfs_notify_dirent_safe (mddev->sysfs_completed); }
@@ -6452,7 +6452,7 @@ static sector_t reshape_request(struct mddev *mddev, sector_t sector_nr, int *sk (sector_nr - mddev->curr_resync_completed) * 2 >= mddev->resync_max - mddev->curr_resync_completed) {/* Cannot proceed until we've updated the superblock... */ -wait_event (conf->wait_for_overlap, +wait_event (conf->wait_for_reshape,atomic_read (&conf->reshape_stripes) ==0 ||test_bit (MD_RECOVERY_INTR , &mddev->recovery));if (atomic_read (&conf->reshape_stripes) !=0 )
@@ -6478,7 +6478,7 @@ static sector_t reshape_request(struct mddev *mddev, sector_t sector_nr, int *skspin_lock_irq (&conf->device_lock); conf->reshape_safe = mddev->reshape_position;spin_unlock_irq (&conf->device_lock); -wake_up (&conf->wait_for_overlap); +wake_up (&conf->wait_for_reshape);sysfs_notify_dirent_safe (mddev->sysfs_completed); } ret:
@@ -6513,7 +6513,7 @@ static inline sector_t raid5_sync_request(struct mddev *mddev, sector_t sector_n }/* Allow raid5_quiesce to complete */ -wait_event (conf->wait_for_overlap, conf->quiesce !=2 ); +wait_event (conf->wait_for_reshape, conf->quiesce !=2 );if (test_bit (MD_RECOVERY_RESHAPE , &mddev->recovery))return reshape_request (mddev, sector_nr, skipped);
@@ -7497,7 +7497,7 @@ static struct r5conf *setup_conf(struct mddev *mddev)init_waitqueue_head (&conf->wait_for_quiescent);init_waitqueue_head (&conf->wait_for_stripe); -init_waitqueue_head (&conf->wait_for_overlap); +init_waitqueue_head (&conf->wait_for_reshape);INIT_LIST_HEAD (&conf->handle_list);INIT_LIST_HEAD (&conf->loprio_list);INIT_LIST_HEAD (&conf->hold_list);
@@ -8555,7 +8555,7 @@ static void end_reshape(struct r5conf *conf) !test_bit(In_sync, &rdev->flags)) rdev->recovery_offset = MaxSector;spin_unlock_irq (&conf->device_lock); -wake_up (&conf->wait_for_overlap); +wake_up (&conf->wait_for_reshape);mddev_update_io_opt (conf->mddev, conf->raid_disks - conf->max_degraded);
@@ -8619,13 +8619,13 @@ static void raid5_quiesce(struct mddev *mddev, int quiesce) conf->quiesce =1 ;unlock_all_device_hash_locks_irq (conf);/* allow reshape to continue */ -wake_up (&conf->wait_for_overlap); +wake_up (&conf->wait_for_reshape); }else {/* re-enable writes */ lock_all_device_hash_locks_irq (conf); conf->quiesce =0 ;wake_up (&conf->wait_for_quiescent); -wake_up (&conf->wait_for_overlap); +wake_up (&conf->wait_for_reshape);unlock_all_device_hash_locks_irq (conf); }log_quiesce (conf, quiesce);
@@ -8944,7 +8944,7 @@ static void raid5_prepare_suspend(struct mddev *mddev) {struct r5conf *conf = mddev->private; -wake_up (&conf->wait_for_overlap); +wake_up (&conf->wait_for_reshape); } static struct md_personality raid6_personality =
drivers/md/raid5.h +1 -1
@@ -668,7 +668,7 @@ struct r5conf {struct llist_head released_stripes;wait_queue_head_t wait_for_quiescent;wait_queue_head_t wait_for_stripe; -wait_queue_head_t wait_for_overlap; +wait_queue_head_t wait_for_reshape;unsigned long cache_state;struct shrinker *shrinker;int pool_size;/* number of disks in stripeheads in pool */
Signature · 1 line
2.43.0
Song Liu
maintainerQuoted 20 lines · Artur Paszkiewicz
The
wait_for_overlapwait queue is currently used in two cases, which are not really related:
- waiting for actual overlapping bios, which usesR5_Overlapbit,
- waiting for events related to reshape.Handling every write request in
raid5_make_request()involves adding to and removing from this wait queue, which uses a spinlock. With fast storage and multiple submitting threads the contention on this lock is noticeable.This patch series aims to resolve this by separating the two cases mentioned above and using this wait queue only when reshape is in progress.
The results when testing 4k random writes on raid5 with
null_blk(8 jobs, qd=64, group_thread_cnt=8):before: 463k IOPS after: 523k IOPS
Quoted 8 lines
The improvement is not huge with this series alone but it is just one of the bottlenecks. When applied onto some other changes I'm working on, it allowed to go from 845k IOPS to 975k IOPS on the same test.
Artur Paszkiewicz (3):
md/raid5: usewait_on_bit()forR5_Overlapmd/raid5: only add to wait queue if reshape is in progress md/raid5: renamewait_for_overlaptowait_for_reshape
Thanks for the optimization! Applied the set to md-6.12 branch.
Song