fixes for btrfs async discards
cover fixes for btrfs async discards
Several fixes for async discards. The first patch might increase discard rate, drastically in some cases. That may be a suprise for those assuming that hitting iops_limit is rare and rarther outliers. Though, it still stays in allowed range, so should be fine.
Pavel Begunkov (4):
btrfs: discard: speed up discard up to iops_limit
btrfs: discard: save discard delay as ns not jiffy
btrfs: don't miss discards after override-schedule
btrfs: discard: reschedule work after param update
fs/btrfs/ctree.h | 3 ++- fs/btrfs/discard.c | 35 +++++++++++++++++++++++------------ fs/btrfs/sysfs.c | 5 +++-- 3 files changed, 28 insertions(+), 15 deletions(-)
Signature · 1 line
2.24.0
PATCH 2/4 btrfs: discard: save discard delay as ns not jiffy
Most of calculations are done in ns or ms, so store discard_ctl->delay in ms and convert the final delay to jiffies only in the end.
--- fs/btrfs/ctree.h | 2 +- fs/btrfs/discard.c | 14 +++++++------- 2 files changed, 8 insertions(+), 8 deletions(-)
fs/btrfs/ctree.h +1 -1
@@ -472,7 +472,7 @@ struct btrfs_discard_ctl {atomic_t discardable_extents;atomic64_t discardable_bytes;u64 max_discard_size; -unsigned long delay; +u64 delay_ms;u32 iops_limit;u32 kbps_limit;u64 discard_extent_bytes;
fs/btrfs/discard.c +7 -7
@@ -355,7 +355,7 @@ void btrfs_discard_schedule_work(struct btrfs_discard_ctl *discard_ctl, block_group =find_next_block_group (discard_ctl, now);if (block_group) { -unsigned long delay = discard_ctl->delay; +u64 delay = discard_ctl->delay_ms *NSEC_PER_MSEC ;u32 kbps_limit =READ_ONCE (discard_ctl->kbps_limit);/*
@@ -366,9 +366,9 @@ void btrfs_discard_schedule_work(struct btrfs_discard_ctl *discard_ctl,if (kbps_limit && discard_ctl->prev_discard) {u64 bps_limit = ((u64 )kbps_limit) *SZ_1K ;u64 bps_delay =div64_u64 (discard_ctl->prev_discard * -MSEC_PER_SEC , bps_limit); +NSEC_PER_SEC , bps_limit); - delay =max (delay,msecs_to_jiffies (bps_delay)); + delay =max (delay, bps_delay); }/*
@@ -378,11 +378,11 @@ void btrfs_discard_schedule_work(struct btrfs_discard_ctl *discard_ctl,if (now < block_group->discard_eligible_time) {u64 bg_timeout = block_group->discard_eligible_time - now; - delay =max (delay,nsecs_to_jiffies (bg_timeout)); + delay =max (delay, bg_timeout); }mod_delayed_work (discard_ctl->discard_workers, - &discard_ctl->work, delay); + &discard_ctl->work,nsecs_to_jiffies (delay)); } out:spin_unlock (&discard_ctl->lock);
@@ -555,7 +555,7 @@ void btrfs_discard_calc_delay(struct btrfs_discard_ctl *discard_ctl) delay =clamp (delay,BTRFS_DISCARD_MIN_DELAY_MSEC ,BTRFS_DISCARD_MAX_DELAY_MSEC ); - discard_ctl->delay =msecs_to_jiffies (delay); + discard_ctl->delay_ms = delay;spin_unlock (&discard_ctl->lock); }
@@ -687,7 +687,7 @@ void btrfs_discard_init(struct btrfs_fs_info *fs_info)atomic_set (&discard_ctl->discardable_extents,0 );atomic64_set (&discard_ctl->discardable_bytes,0 ); discard_ctl->max_discard_size =BTRFS_ASYNC_DISCARD_DEFAULT_MAX_SIZE ; - discard_ctl->delay =BTRFS_DISCARD_MAX_DELAY_MSEC ; + discard_ctl->delay_ms =BTRFS_DISCARD_MAX_DELAY_MSEC ; discard_ctl->iops_limit =BTRFS_DISCARD_MAX_IOPS ; discard_ctl->kbps_limit =0 ; discard_ctl->discard_extent_bytes =0 ;
Signature · 1 line
2.24.0
PATCH 1/4 btrfs: discard: speed up discard up to iops_limit
In reply to Pavel BegunkovInstead of using iops_limit only for cutting off extremes, calculate the discard delay directly from it, so it closely follows iops_limit and doesn't under-discarding even though quotas are not saturated.
--- fs/btrfs/discard.c | 10 +++++----- 1 file changed, 5 insertions(+), 5 deletions(-)
fs/btrfs/discard.c +5 -5
@@ -519,7 +519,6 @@ void btrfs_discard_calc_delay(struct btrfs_discard_ctl *discard_ctl)s64 discardable_bytes;u32 iops_limit;unsigned long delay; -unsigned long lower_limit =BTRFS_DISCARD_MIN_DELAY_MSEC ; discardable_extents =atomic_read (&discard_ctl->discardable_extents);if (!discardable_extents)
@@ -550,11 +549,12 @@ void btrfs_discard_calc_delay(struct btrfs_discard_ctl *discard_ctl) iops_limit =READ_ONCE (discard_ctl->iops_limit);if (iops_limit) - lower_limit = max_t(unsigned long, lower_limit, -MSEC_PER_SEC / iops_limit); + delay =MSEC_PER_SEC / iops_limit; +else + delay =BTRFS_DISCARD_TARGET_MSEC / discardable_extents; - delay =BTRFS_DISCARD_TARGET_MSEC / discardable_extents; - delay =clamp (delay, lower_limit,BTRFS_DISCARD_MAX_DELAY_MSEC ); + delay =clamp (delay,BTRFS_DISCARD_MIN_DELAY_MSEC , +BTRFS_DISCARD_MAX_DELAY_MSEC ); discard_ctl->delay =msecs_to_jiffies (delay);spin_unlock (&discard_ctl->lock);
Signature · 1 line
2.24.0
PATCH 3/4 btrfs: don't miss discards after override-schedule
In reply to Pavel BegunkovIf btrfs_discard_schedule_work() is called with override=true, it sets delay anew regardless how much time left until the timer should have fired. If delays are long (that can happen, for example, with low kbps_limit), they might be constantly overriden without having a chance to run the discard work.
--- fs/btrfs/ctree.h | 1 + fs/btrfs/discard.c | 11 +++++++++++ 2 files changed, 12 insertions(+)
fs/btrfs/ctree.h +1 -0
@@ -469,6 +469,7 @@ struct btrfs_discard_ctl {struct btrfs_block_group *block_group;struct list_head discard_list[BTRFS_NR_DISCARD_LISTS ];u64 prev_discard; +u64 prev_discard_time;atomic_t discardable_extents;atomic64_t discardable_bytes;u64 max_discard_size;
fs/btrfs/discard.c +11 -0
@@ -381,6 +381,15 @@ void btrfs_discard_schedule_work(struct btrfs_discard_ctl *discard_ctl, delay =max (delay, bg_timeout); } +if (override && discard_ctl->prev_discard) { +u64 elapsed = now - discard_ctl->prev_discard_time; + +if (delay > elapsed) + delay -= elapsed; +else + delay =0 ; + } +mod_delayed_work (discard_ctl->discard_workers, &discard_ctl->work,nsecs_to_jiffies (delay)); }
@@ -466,6 +475,7 @@ static void btrfs_discard_workfn(struct work_struct *work) } discard_ctl->prev_discard = trimmed; + discard_ctl->prev_discard_time =ktime_get_ns ();/* Determine next steps for a block_group */ if (block_group->discard_cursor >=btrfs_block_group_end (block_group)) {
@@ -684,6 +694,7 @@ void btrfs_discard_init(struct btrfs_fs_info *fs_info)INIT_LIST_HEAD (&discard_ctl->discard_list[i]); discard_ctl->prev_discard =0 ; + discard_ctl->prev_discard_time =0 ;atomic_set (&discard_ctl->discardable_extents,0 );atomic64_set (&discard_ctl->discardable_bytes,0 ); discard_ctl->max_discard_size =BTRFS_ASYNC_DISCARD_DEFAULT_MAX_SIZE ;
Signature · 1 line
2.24.0
PATCH 4/4 btrfs: discard: reschedule work after param update
In reply to Pavel BegunkovAfter sysfs updates discard's iops_limit or kbps_limit it also needs to adjust current timer through rescheduling, otherwise the discard work may wait for a long time for the previous timer to expier or bumped by someone else.
--- fs/btrfs/sysfs.c | 5 +++-- 1 file changed, 3 insertions(+), 2 deletions(-)
fs/btrfs/sysfs.c +3 -2
@@ -433,7 +433,8 @@ static ssize_t btrfs_discard_iops_limit_store(struct kobject *kobj,return -EINVAL ;WRITE_ONCE (discard_ctl->iops_limit, iops_limit); - +btrfs_discard_calc_delay (discard_ctl); +btrfs_discard_schedule_work (discard_ctl, true);return len; } BTRFS_ATTR_RW(discard, iops_limit, btrfs_discard_iops_limit_show,
@@ -463,7 +464,7 @@ static ssize_t btrfs_discard_kbps_limit_store(struct kobject *kobj,return -EINVAL ;WRITE_ONCE (discard_ctl->kbps_limit, kbps_limit); - +btrfs_discard_schedule_work (discard_ctl, true);return len; } BTRFS_ATTR_RW(discard, kbps_limit, btrfs_discard_kbps_limit_show,
Signature · 1 line
2.24.0
Re: [PATCH 1/4] btrfs: discard: speed up discard up to iops_limit
In reply to Pavel BegunkovQuoted 4 lines · Pavel Begunkov
Instead of using
iops_limitonly for cutting off extremes, calculate the discard delay directly from it, so it closely followsiops_limitand doesn't under-discarding even though quotas are not saturated.
This sounds like it potentially be a great performance boost, do you have any performance metrics regarding this patch?
Quoted 19 lines
Signed-off-by Pavel Begunkov <[email protected]>--- fs/btrfs/discard.c | 10 +++++----- 1 file changed, 5 insertions(+), 5 deletions(-)fs/btrfs/discard.c +0
-1@@ -519,7 +519,6 @@ void btrfs_discard_calc_delay(struct btrfs_discard_ctl *discard_ctl)s64 discardable_bytes;u32 iops_limit;unsigned long delay; -unsigned long lower_limit =BTRFS_DISCARD_MIN_DELAY_MSEC ; discardable_extents =atomic_read (&discard_ctl->discardable_extents);if (!discardable_extents)
fs/btrfs/discard.c void btrfs_discard_calc_delay(struct btrfs_discard_ctl *discard_ctl) · line 554
Quoted 8 lines
fs/btrfs/discard.c +3
-2@@ -550,11 +549,12 @@ void btrfs_discard_calc_delay(struct btrfs_discard_ctl *discard_ctl) iops_limit =READ_ONCE (discard_ctl->iops_limit);if (iops_limit) - lower_limit = max_t(unsigned long, lower_limit, -MSEC_PER_SEC / iops_limit); + delay =MSEC_PER_SEC / iops_limit; +else + delay =BTRFS_DISCARD_TARGET_MSEC / discardable_extents;
Looks good to me. I wonder why there wasn't handling of if iops_limit was unfindable
before?
Quoted 11 lines
fs/btrfs/discard.c +2
-3- delay =BTRFS_DISCARD_TARGET_MSEC / discardable_extents; - delay =clamp (delay, lower_limit,BTRFS_DISCARD_MAX_DELAY_MSEC ); + delay =clamp (delay,BTRFS_DISCARD_MIN_DELAY_MSEC , +BTRFS_DISCARD_MAX_DELAY_MSEC ); discard_ctl->delay =msecs_to_jiffies (delay);spin_unlock (&discard_ctl->lock); --2.24.0
This patch looks all great to me.
Best regards,
Amy Parker
(they/them)
Re: [PATCH 2/4] btrfs: discard: save discard delay as ns not jiffy
In reply to Pavel BegunkovQuoted 3 lines · Pavel Begunkov
Most of calculations are done in ns or ms, so store
discard_ctl->delayin ms and convert the final delay to jiffies only in the end.
Great idea.
Quoted 12 lines
Signed-off-by Pavel Begunkov <[email protected]>--- fs/btrfs/ctree.h | 2 +- fs/btrfs/discard.c | 14 +++++++------- 2 files changed, 8 insertions(+), 8 deletions(-)fs/btrfs/ctree.h +0
-0
fs/btrfs/ctree.h struct btrfs_discard_ctl { · line 475
Quoted 5 lines
fs/btrfs/ctree.h +1
-1@@ -472,7 +472,7 @@ struct btrfs_discard_ctl {atomic_t discardable_extents;atomic64_t discardable_bytes;u64 max_discard_size; -unsigned long delay; +u64 delay_ms;
Thanks for converting this from the ambiguous unsigned long to the more specific u64.
fs/btrfs/discard.c void btrfs_discard_schedule_work(struct btrfs_discard_ctl *discard_ctl, · line 358
Quoted 13 lines
fs/btrfs/ctree.h +0
-0u32 iops_limit;u32 kbps_limit;u64 discard_extent_bytes;fs/btrfs/discard.c +1
-1@@ -355,7 +355,7 @@ void btrfs_discard_schedule_work(struct btrfs_discard_ctl *discard_ctl, block_group =find_next_block_group (discard_ctl, now);if (block_group) { -unsigned long delay = discard_ctl->delay; +u64 delay = discard_ctl->delay_ms *NSEC_PER_MSEC ;
I worry about a potential performance impact with this, but it should be minimal at most.
fs/btrfs/discard.c void btrfs_discard_schedule_work(struct btrfs_discard_ctl *discard_ctl, · line 371
Quoted 12 lines
fs/btrfs/discard.c +2
-2u32 kbps_limit =READ_ONCE (discard_ctl->kbps_limit);/* @@ -366,9 +366,9 @@ void btrfs_discard_schedule_work(struct btrfs_discard_ctl *discard_ctl,if (kbps_limit && discard_ctl->prev_discard) {u64 bps_limit = ((u64 )kbps_limit) *SZ_1K ;u64 bps_delay =div64_u64 (discard_ctl->prev_discard * -MSEC_PER_SEC , bps_limit); +NSEC_PER_SEC , bps_limit); - delay =max (delay,msecs_to_jiffies (bps_delay)); + delay =max (delay, bps_delay);
Great that we got this down to just passing max() a value. Same thing on the instance below.
Quoted 36 lines
fs/btrfs/discard.c +4
-5}/* @@ -378,11 +378,11 @@ void btrfs_discard_schedule_work(struct btrfs_discard_ctl *discard_ctl,if (now < block_group->discard_eligible_time) {u64 bg_timeout = block_group->discard_eligible_time - now; - delay =max (delay,nsecs_to_jiffies (bg_timeout)); + delay =max (delay, bg_timeout); }mod_delayed_work (discard_ctl->discard_workers, - &discard_ctl->work, delay); + &discard_ctl->work,nsecs_to_jiffies (delay)); } out:spin_unlock (&discard_ctl->lock);@@ -555,7 +555,7 @@ void btrfs_discard_calc_delay(struct btrfs_discard_ctl *discard_ctl) delay =clamp (delay,BTRFS_DISCARD_MIN_DELAY_MSEC ,BTRFS_DISCARD_MAX_DELAY_MSEC ); - discard_ctl->delay =msecs_to_jiffies (delay); + discard_ctl->delay_ms = delay;spin_unlock (&discard_ctl->lock); }@@ -687,7 +687,7 @@ void btrfs_discard_init(struct btrfs_fs_info *fs_info)atomic_set (&discard_ctl->discardable_extents,0 );atomic64_set (&discard_ctl->discardable_bytes,0 ); discard_ctl->max_discard_size =BTRFS_ASYNC_DISCARD_DEFAULT_MAX_SIZE ; - discard_ctl->delay =BTRFS_DISCARD_MAX_DELAY_MSEC ; + discard_ctl->delay_ms =BTRFS_DISCARD_MAX_DELAY_MSEC ; discard_ctl->iops_limit =BTRFS_DISCARD_MAX_IOPS ; discard_ctl->kbps_limit =0 ; discard_ctl->discard_extent_bytes =0 ; --
Quoted 2 lines
2.24.0
Looks all fine to me.
Best regards,
Amy Parker
(they/them)
Re: [PATCH 2/4] btrfs: discard: save discard delay as ns not jiffy
Quoted 37 lines · Amy Parker
Quoted 3 lines · Pavel Begunkov
Most of calculations are done in ns or ms, so store
discard_ctl->delayin ms and convert the final delay to jiffies only in the end.Great idea.
Quoted 12 lines
Signed-off-by Pavel Begunkov <[email protected]>--- fs/btrfs/ctree.h | 2 +- fs/btrfs/discard.c | 14 +++++++------- 2 files changed, 8 insertions(+), 8 deletions(-)fs/btrfs/ctree.h +0
-0fs/btrfs/ctree.h struct btrfs_discard_ctl { · line 475
Quoted 5 lines
fs/btrfs/ctree.h +1
-1@@ -472,7 +472,7 @@ struct btrfs_discard_ctl {atomic_t discardable_extents;atomic64_t discardable_bytes;u64 max_discard_size; -unsigned long delay; +u64 delay_ms;Thanks for converting this from the ambiguous unsigned long to the more specific u64.
Quoted 9 lines
fs/btrfs/ctree.h +0
-0u32 iops_limit;u32 kbps_limit;u64 discard_extent_bytes;fs/btrfs/discard.c +0
-0@@ -355,7 +355,7 @@ void btrfs_discard_schedule_work(struct btrfs_discard_ctl *discard_ctl,
Quoted 7 lines
fs/btrfs/discard.c void btrfs_discard_schedule_work(struct btrfs_discard_ctl *discard_ctl, · line 358
Quoted 4 lines
fs/btrfs/discard.c +1
-1block_group =find_next_block_group (discard_ctl, now);if (block_group) { -unsigned long delay = discard_ctl->delay; +u64 delay = discard_ctl->delay_ms *NSEC_PER_MSEC ;I worry about a potential performance impact with this, but it should be minimal at most.
That's nothing, nsecs_to_jiffies() in the end is heavier, but this is not in a hot path and by all means it's heavily amortised by actual discarding.
Quoted 57 lines
fs/btrfs/discard.c void btrfs_discard_schedule_work(struct btrfs_discard_ctl *discard_ctl, · line 371
Quoted 13 lines
fs/btrfs/discard.c +2
-2u32 kbps_limit =READ_ONCE (discard_ctl->kbps_limit);/* @@ -366,9 +366,9 @@ void btrfs_discard_schedule_work(struct btrfs_discard_ctl *discard_ctl,if (kbps_limit && discard_ctl->prev_discard) {u64 bps_limit = ((u64 )kbps_limit) *SZ_1K ;u64 bps_delay =div64_u64 (discard_ctl->prev_discard * -MSEC_PER_SEC , bps_limit); +NSEC_PER_SEC , bps_limit); - delay =max (delay,msecs_to_jiffies (bps_delay)); + delay =max (delay, bps_delay);Great that we got this down to just passing
max()a value. Same thing on the instance below.Quoted 36 lines
fs/btrfs/discard.c +4
-5}/* @@ -378,11 +378,11 @@ void btrfs_discard_schedule_work(struct btrfs_discard_ctl *discard_ctl,if (now < block_group->discard_eligible_time) {u64 bg_timeout = block_group->discard_eligible_time - now; - delay =max (delay,nsecs_to_jiffies (bg_timeout)); + delay =max (delay, bg_timeout); }mod_delayed_work (discard_ctl->discard_workers, - &discard_ctl->work, delay); + &discard_ctl->work,nsecs_to_jiffies (delay)); } out:spin_unlock (&discard_ctl->lock);@@ -555,7 +555,7 @@ void btrfs_discard_calc_delay(struct btrfs_discard_ctl *discard_ctl) delay =clamp (delay,BTRFS_DISCARD_MIN_DELAY_MSEC ,BTRFS_DISCARD_MAX_DELAY_MSEC ); - discard_ctl->delay =msecs_to_jiffies (delay); + discard_ctl->delay_ms = delay;spin_unlock (&discard_ctl->lock); }@@ -687,7 +687,7 @@ void btrfs_discard_init(struct btrfs_fs_info *fs_info)atomic_set (&discard_ctl->discardable_extents,0 );atomic64_set (&discard_ctl->discardable_bytes,0 ); discard_ctl->max_discard_size =BTRFS_ASYNC_DISCARD_DEFAULT_MAX_SIZE ; - discard_ctl->delay =BTRFS_DISCARD_MAX_DELAY_MSEC ; + discard_ctl->delay_ms =BTRFS_DISCARD_MAX_DELAY_MSEC ; discard_ctl->iops_limit =BTRFS_DISCARD_MAX_IOPS ; discard_ctl->kbps_limit =0 ; discard_ctl->discard_extent_bytes =0 ; --Quoted 2 lines
2.24.0
Looks all fine to me.
Signature · 1 line
Pavel Begunkov
Re: [PATCH 2/4] btrfs: discard: save discard delay as ns not jiffy
Quoted 47 lines · Pavel Begunkov
Quoted 37 lines · Amy Parker
Quoted 3 lines · Pavel Begunkov
Most of calculations are done in ns or ms, so store
discard_ctl->delayin ms and convert the final delay to jiffies only in the end.Great idea.
Quoted 12 lines
Signed-off-by Pavel Begunkov <[email protected]>--- fs/btrfs/ctree.h | 2 +- fs/btrfs/discard.c | 14 +++++++------- 2 files changed, 8 insertions(+), 8 deletions(-)fs/btrfs/ctree.h +0
-0fs/btrfs/ctree.h struct btrfs_discard_ctl { · line 475
Quoted 5 lines
fs/btrfs/ctree.h +1
-1@@ -472,7 +472,7 @@ struct btrfs_discard_ctl {atomic_t discardable_extents;atomic64_t discardable_bytes;u64 max_discard_size; -unsigned long delay; +u64 delay_ms;Thanks for converting this from the ambiguous unsigned long to the more specific u64.
Quoted 9 lines
fs/btrfs/ctree.h +0
-0u32 iops_limit;u32 kbps_limit;u64 discard_extent_bytes;fs/btrfs/discard.c +0
-0@@ -355,7 +355,7 @@ void btrfs_discard_schedule_work(struct btrfs_discard_ctl *discard_ctl,Quoted 7 lines
fs/btrfs/discard.c void btrfs_discard_schedule_work(struct btrfs_discard_ctl *discard_ctl, · line 358
Quoted 4 lines
fs/btrfs/discard.c +1
-1block_group =find_next_block_group (discard_ctl, now);if (block_group) { -unsigned long delay = discard_ctl->delay; +u64 delay = discard_ctl->delay_ms *NSEC_PER_MSEC ;I worry about a potential performance impact with this, but it should be minimal at most.
Quoted 2 lines
That's nothing,
nsecs_to_jiffies()in the end is heavier, but this is not in a hot path and by all means it's heavily amortised by actual discarding.
Alright, sounds good.
Quoted 60 lines
Quoted 42 lines
Quoted 6 lines
fs/btrfs/discard.c +0
-0u32 kbps_limit =READ_ONCE (discard_ctl->kbps_limit);/* fs/btrfs/discard.c void btrfs_discard_schedule_work(struct btrfs_discard_ctl *discard_ctl, · line 371
Quoted 8 lines
fs/btrfs/discard.c +2
-2@@ -366,9 +366,9 @@ void btrfs_discard_schedule_work(struct btrfs_discard_ctl *discard_ctl,if (kbps_limit && discard_ctl->prev_discard) {u64 bps_limit = ((u64 )kbps_limit) *SZ_1K ;u64 bps_delay =div64_u64 (discard_ctl->prev_discard * -MSEC_PER_SEC , bps_limit); +NSEC_PER_SEC , bps_limit); - delay =max (delay,msecs_to_jiffies (bps_delay)); + delay =max (delay, bps_delay);Great that we got this down to just passing
max()a value. Same thing on the instance below.Quoted 24 lines
fs/btrfs/discard.c +3
-3}/* @@ -378,11 +378,11 @@ void btrfs_discard_schedule_work(struct btrfs_discard_ctl *discard_ctl,if (now < block_group->discard_eligible_time) {u64 bg_timeout = block_group->discard_eligible_time - now; - delay =max (delay,nsecs_to_jiffies (bg_timeout)); + delay =max (delay, bg_timeout); }mod_delayed_work (discard_ctl->discard_workers, - &discard_ctl->work, delay); + &discard_ctl->work,nsecs_to_jiffies (delay)); } out:spin_unlock (&discard_ctl->lock);@@ -555,7 +555,7 @@ void btrfs_discard_calc_delay(struct btrfs_discard_ctl *discard_ctl) delay =clamp (delay,BTRFS_DISCARD_MIN_DELAY_MSEC ,BTRFS_DISCARD_MAX_DELAY_MSEC ); - discard_ctl->delay =msecs_to_jiffies (delay); + discard_ctl->delay_ms = delay;Quoted 16 lines
Quoted 12 lines
fs/btrfs/discard.c +1
-2spin_unlock (&discard_ctl->lock); }@@ -687,7 +687,7 @@ void btrfs_discard_init(struct btrfs_fs_info *fs_info)atomic_set (&discard_ctl->discardable_extents,0 );atomic64_set (&discard_ctl->discardable_bytes,0 ); discard_ctl->max_discard_size =BTRFS_ASYNC_DISCARD_DEFAULT_MAX_SIZE ; - discard_ctl->delay =BTRFS_DISCARD_MAX_DELAY_MSEC ; + discard_ctl->delay_ms =BTRFS_DISCARD_MAX_DELAY_MSEC ; discard_ctl->iops_limit =BTRFS_DISCARD_MAX_IOPS ; discard_ctl->kbps_limit =0 ; discard_ctl->discard_extent_bytes =0 ; --Quoted 2 lines
2.24.0
Looks all fine to me.
Quoted 2 lines
--
Pavel Begunkov
In this case, I see nothing more to consider with this.
Best regards,
Amy Parker
(they/them)
Re: [PATCH 1/4] btrfs: discard: speed up discard up to iops_limit
In reply to Amy ParkerQuoted 8 lines · Amy Parker
Quoted 4 lines · Pavel Begunkov
Instead of using
iops_limitonly for cutting off extremes, calculate the discard delay directly from it, so it closely followsiops_limitand doesn't under-discarding even though quotas are not saturated.This sounds like it potentially be a great performance boost, do you have any performance metrics regarding this patch?
Boosting the discard rate and so reaping stalling blocks may be nice, but unless it holds too much memory creating lack of space it shouldn't affect throughput. Though, it's better to ask people with deeper understanding of the fs.
What I've seen is that in some cases there are extents staying queued for discarding for _too_ long. E.g. reaping a small number of very fat extents keeps delay at max and doesn't allow to reap them effectively. That could be a problem with fast drives.
Quoted 21 lines
Quoted 21 lines
Signed-off-by Pavel Begunkov <[email protected]>--- fs/btrfs/discard.c | 10 +++++----- 1 file changed, 5 insertions(+), 5 deletions(-)fs/btrfs/discard.c +0
-1@@ -519,7 +519,6 @@ void btrfs_discard_calc_delay(struct btrfs_discard_ctl *discard_ctl)s64 discardable_bytes;u32 iops_limit;unsigned long delay; -unsigned long lower_limit =BTRFS_DISCARD_MIN_DELAY_MSEC ; discardable_extents =atomic_read (&discard_ctl->discardable_extents);if (!discardable_extents)@@ -550,11 +549,12 @@ void btrfs_discard_calc_delay(struct btrfs_discard_ctl *discard_ctl)
Quoted 11 lines
fs/btrfs/discard.c void btrfs_discard_calc_delay(struct btrfs_discard_ctl *discard_ctl) · line 554
Quoted 7 lines
fs/btrfs/discard.c +3
-2iops_limit =READ_ONCE (discard_ctl->iops_limit);if (iops_limit) - lower_limit = max_t(unsigned long, lower_limit, -MSEC_PER_SEC / iops_limit); + delay =MSEC_PER_SEC / iops_limit; +else + delay =BTRFS_DISCARD_TARGET_MSEC / discardable_extents;Looks good to me. I wonder why there wasn't handling of if
iops_limitwas unfindable
before?
Not sure what you mean by unfindable, but async discard is relatively new, might be that everyone just have their hands full.
Quoted 19 lines
Quoted 10 lines
fs/btrfs/discard.c +2
-3- delay =BTRFS_DISCARD_TARGET_MSEC / discardable_extents; - delay =clamp (delay, lower_limit,BTRFS_DISCARD_MAX_DELAY_MSEC ); + delay =clamp (delay,BTRFS_DISCARD_MIN_DELAY_MSEC , +BTRFS_DISCARD_MAX_DELAY_MSEC ); discard_ctl->delay =msecs_to_jiffies (delay);spin_unlock (&discard_ctl->lock); --Quoted 2 lines
2.24.0
This patch looks all great to me.
Best regards,
Amy Parker
(they/them)
Signature · 1 line
Pavel Begunkov
Re: [PATCH 1/4] btrfs: discard: speed up discard up to iops_limit
Quoted 8 lines · Pavel Begunkov
Quoted 6 lines · Amy Parker
Quoted 4 lines · Pavel Begunkov
Instead of using
iops_limitonly for cutting off extremes, calculate the discard delay directly from it, so it closely followsiops_limitand doesn't under-discarding even though quotas are not saturated.
Quoted 7 lines
Quoted 2 lines
This sounds like it potentially be a great performance boost, do you have any performance metrics regarding this patch?
Boosting the discard rate and so reaping stalling blocks may be nice, but unless it holds too much memory creating lack of space it shouldn't affect throughput. Though, it's better to ask people with deeper understanding of the fs.
Alright, thanks for the clarification.
Quoted 4 lines
What I've seen is that in some cases there are extents staying queued for discarding for _too_ long. E.g. reaping a small number of very fat extents keeps delay at max and doesn't allow to reap them effectively. That could be a problem with fast drives.
Ah, yep. Seen this personally to a smaller extent.
Quoted 34 lines
Quoted 22 lines
Quoted 22 lines
Signed-off-by Pavel Begunkov <[email protected]>--- fs/btrfs/discard.c | 10 +++++----- 1 file changed, 5 insertions(+), 5 deletions(-)fs/btrfs/discard.c +0
-1@@ -519,7 +519,6 @@ void btrfs_discard_calc_delay(struct btrfs_discard_ctl *discard_ctl)s64 discardable_bytes;u32 iops_limit;unsigned long delay; -unsigned long lower_limit =BTRFS_DISCARD_MIN_DELAY_MSEC ; discardable_extents =atomic_read (&discard_ctl->discardable_extents);if (!discardable_extents)@@ -550,11 +549,12 @@ void btrfs_discard_calc_delay(struct btrfs_discard_ctl *discard_ctl)Quoted 11 lines
fs/btrfs/discard.c void btrfs_discard_calc_delay(struct btrfs_discard_ctl *discard_ctl) · line 554
Quoted 7 lines
fs/btrfs/discard.c +3
-2iops_limit =READ_ONCE (discard_ctl->iops_limit);if (iops_limit) - lower_limit = max_t(unsigned long, lower_limit, -MSEC_PER_SEC / iops_limit); + delay =MSEC_PER_SEC / iops_limit; +else + delay =BTRFS_DISCARD_TARGET_MSEC / discardable_extents;Looks good to me. I wonder why there wasn't handling of if
iops_limitwas unfindable
before?
Quoted 2 lines
Not sure what you mean by unfindable, but async discard is relatively new, might be that everyone just have their hands full.
By unfindable I mean if iops_limit turned up as null when reading it from discard_ctl.
Async discard was added in 5.6, correct? So yeah, makes sense then that people just had their hands full. Thanks for adding it.
Quoted 16 lines
Quoted 16 lines
Quoted 11 lines
fs/btrfs/discard.c +2
-3- delay =BTRFS_DISCARD_TARGET_MSEC / discardable_extents; - delay =clamp (delay, lower_limit,BTRFS_DISCARD_MAX_DELAY_MSEC ); + delay =clamp (delay,BTRFS_DISCARD_MIN_DELAY_MSEC , +BTRFS_DISCARD_MAX_DELAY_MSEC ); discard_ctl->delay =msecs_to_jiffies (delay);spin_unlock (&discard_ctl->lock); --Quoted 2 lines
2.24.0
This patch looks all great to me.
Quoted 7 lines
Quoted 4 lines
Best regards,
Amy Parker
(they/them)--
Pavel Begunkov
Best regards,
Amy Parker
(they/them)
Re: [PATCH 1/4] btrfs: discard: speed up discard up to iops_limit
Quoted 60 lines · Amy Parker
Quoted 8 lines · Pavel Begunkov
Quoted 6 lines · Amy Parker
Quoted 4 lines · Pavel Begunkov
Instead of using
iops_limitonly for cutting off extremes, calculate the discard delay directly from it, so it closely followsiops_limitand doesn't under-discarding even though quotas are not saturated.Quoted 7 lines
Quoted 2 lines
This sounds like it potentially be a great performance boost, do you have any performance metrics regarding this patch?
Boosting the discard rate and so reaping stalling blocks may be nice, but unless it holds too much memory creating lack of space it shouldn't affect throughput. Though, it's better to ask people with deeper understanding of the fs.
Alright, thanks for the clarification.
Quoted 4 lines
What I've seen is that in some cases there are extents staying queued for discarding for _too_ long. E.g. reaping a small number of very fat extents keeps delay at max and doesn't allow to reap them effectively. That could be a problem with fast drives.
Ah, yep. Seen this personally to a smaller extent.
Quoted 34 lines
Quoted 33 lines
Quoted 21 lines
Signed-off-by Pavel Begunkov <[email protected]>--- fs/btrfs/discard.c | 10 +++++----- 1 file changed, 5 insertions(+), 5 deletions(-)fs/btrfs/discard.c +0
-1@@ -519,7 +519,6 @@ void btrfs_discard_calc_delay(struct btrfs_discard_ctl *discard_ctl)s64 discardable_bytes;u32 iops_limit;unsigned long delay; -unsigned long lower_limit =BTRFS_DISCARD_MIN_DELAY_MSEC ; discardable_extents =atomic_read (&discard_ctl->discardable_extents);if (!discardable_extents)fs/btrfs/discard.c void btrfs_discard_calc_delay(struct btrfs_discard_ctl *discard_ctl) · line 554
Quoted 8 lines
fs/btrfs/discard.c +3
-2@@ -550,11 +549,12 @@ void btrfs_discard_calc_delay(struct btrfs_discard_ctl *discard_ctl) iops_limit =READ_ONCE (discard_ctl->iops_limit);if (iops_limit) - lower_limit = max_t(unsigned long, lower_limit, -MSEC_PER_SEC / iops_limit); + delay =MSEC_PER_SEC / iops_limit; +else + delay =BTRFS_DISCARD_TARGET_MSEC / discardable_extents;Looks good to me. I wonder why there wasn't handling of if
iops_limitwas unfindable
before?
Quoted 5 lines
Quoted 2 lines
Not sure what you mean by unfindable, but async discard is relatively new, might be that everyone just have their hands full.
By unfindable I mean if
iops_limitturned up as null when reading it fromdiscard_ctl.
Ahh, ok. It's handled and I left it as it was, that BTW is still a problem.
First it calculates a delay based on number of queued extents and than clamps it to (BTRFS_DISCARD_MIN_DELAY_MSEC, BTRFS_DISCARD_MAX_DELAY_MSEC). Without this patch it did the same but the lower bound was calculated from iops_limit.
Quoted 2 lines
Async discard was added in 5.6, correct? So yeah, makes sense then that people just had their hands full. Thanks for adding it.
b0643e59cfa609c4b5f ("btrfs: add the beginning of async discard, discard workqueue"). Dec 2019, so less than a year
Quoted 16 lines
Quoted 16 lines
Quoted 16 lines
Quoted 12 lines
fs/btrfs/discard.c +2
-3- delay =BTRFS_DISCARD_TARGET_MSEC / discardable_extents; - delay =clamp (delay, lower_limit,BTRFS_DISCARD_MAX_DELAY_MSEC ); + delay =clamp (delay,BTRFS_DISCARD_MIN_DELAY_MSEC , +BTRFS_DISCARD_MAX_DELAY_MSEC ); discard_ctl->delay =msecs_to_jiffies (delay);spin_unlock (&discard_ctl->lock); --Quoted 2 lines
2.24.0
This patch looks all great to me.
Signature · 1 line
Pavel Begunkov
Re: [PATCH 1/4] btrfs: discard: speed up discard up to iops_limit
Quoted 62 lines · Pavel Begunkov
Quoted 60 lines · Amy Parker
Quoted 8 lines · Pavel Begunkov
Quoted 6 lines · Amy Parker
Quoted 4 lines · Pavel Begunkov
Instead of using
iops_limitonly for cutting off extremes, calculate the discard delay directly from it, so it closely followsiops_limitand doesn't under-discarding even though quotas are not saturated.Quoted 7 lines
Quoted 2 lines
This sounds like it potentially be a great performance boost, do you have any performance metrics regarding this patch?
Boosting the discard rate and so reaping stalling blocks may be nice, but unless it holds too much memory creating lack of space it shouldn't affect throughput. Though, it's better to ask people with deeper understanding of the fs.
Alright, thanks for the clarification.
Quoted 4 lines
What I've seen is that in some cases there are extents staying queued for discarding for _too_ long. E.g. reaping a small number of very fat extents keeps delay at max and doesn't allow to reap them effectively. That could be a problem with fast drives.
Ah, yep. Seen this personally to a smaller extent.
Quoted 34 lines
Quoted 33 lines
Quoted 21 lines
Signed-off-by Pavel Begunkov <[email protected]>--- fs/btrfs/discard.c | 10 +++++----- 1 file changed, 5 insertions(+), 5 deletions(-)fs/btrfs/discard.c +0
-1@@ -519,7 +519,6 @@ void btrfs_discard_calc_delay(struct btrfs_discard_ctl *discard_ctl)s64 discardable_bytes;u32 iops_limit;unsigned long delay; -unsigned long lower_limit =BTRFS_DISCARD_MIN_DELAY_MSEC ; discardable_extents =atomic_read (&discard_ctl->discardable_extents);if (!discardable_extents)fs/btrfs/discard.c void btrfs_discard_calc_delay(struct btrfs_discard_ctl *discard_ctl) · line 554
Quoted 8 lines
fs/btrfs/discard.c +3
-2@@ -550,11 +549,12 @@ void btrfs_discard_calc_delay(struct btrfs_discard_ctl *discard_ctl) iops_limit =READ_ONCE (discard_ctl->iops_limit);if (iops_limit) - lower_limit = max_t(unsigned long, lower_limit, -MSEC_PER_SEC / iops_limit); + delay =MSEC_PER_SEC / iops_limit; +else + delay =BTRFS_DISCARD_TARGET_MSEC / discardable_extents;Looks good to me. I wonder why there wasn't handling of if
iops_limitwas unfindable
before?
Quoted 7 lines
Quoted 5 lines
Quoted 2 lines
Not sure what you mean by unfindable, but async discard is relatively new, might be that everyone just have their hands full.
By unfindable I mean if
iops_limitturned up as null when reading it fromdiscard_ctl.Ahh, ok. It's handled and I left it as it was, that BTW is still a problem.
How often is iops_limit unfindable?
Quoted 4 lines
First it calculates a delay based on number of queued extents and than clamps it to (
BTRFS_DISCARD_MIN_DELAY_MSEC,BTRFS_DISCARD_MAX_DELAY_MSEC). Without this patch it did the same but the lower bound was calculated fromiops_limit.
Thanks for clarifying.
Quoted 6 lines
Quoted 3 lines
Async discard was added in 5.6, correct? So yeah, makes sense then that people just had their hands full. Thanks for adding it.
b0643e59cfa609c4b5f("btrfs: add the beginning of async discard, discard workqueue"). Dec 2019, so less than a year
Thanks for finding the commit.
Quoted 16 lines
Quoted 16 lines
Quoted 16 lines
Quoted 16 lines
Quoted 15 lines
fs/btrfs/discard.c +2
-3- delay =BTRFS_DISCARD_TARGET_MSEC / discardable_extents; - delay =clamp (delay, lower_limit,BTRFS_DISCARD_MAX_DELAY_MSEC ); + delay =clamp (delay,BTRFS_DISCARD_MIN_DELAY_MSEC , +BTRFS_DISCARD_MAX_DELAY_MSEC ); discard_ctl->delay =msecs_to_jiffies (delay);spin_unlock (&discard_ctl->lock); --2.24.0
Quoted 4 lines
Quoted 1 line
Quoted 1 line
Quoted 1 line
This patch looks all great to me.
--
Pavel Begunkov
Best regards,
Amy Parker
(they/them)
Re: [PATCH 1/4] btrfs: discard: speed up discard up to iops_limit
Quoted 66 lines · Amy Parker
Quoted 65 lines · Pavel Begunkov
Quoted 63 lines · Amy Parker
Quoted 8 lines · Pavel Begunkov
Quoted 6 lines · Amy Parker
Quoted 4 lines · Pavel Begunkov
Instead of using
iops_limitonly for cutting off extremes, calculate the discard delay directly from it, so it closely followsiops_limitand doesn't under-discarding even though quotas are not saturated.Quoted 7 lines
Quoted 2 lines
This sounds like it potentially be a great performance boost, do you have any performance metrics regarding this patch?
Boosting the discard rate and so reaping stalling blocks may be nice, but unless it holds too much memory creating lack of space it shouldn't affect throughput. Though, it's better to ask people with deeper understanding of the fs.
Alright, thanks for the clarification.
Quoted 4 lines
What I've seen is that in some cases there are extents staying queued for discarding for _too_ long. E.g. reaping a small number of very fat extents keeps delay at max and doesn't allow to reap them effectively. That could be a problem with fast drives.
Ah, yep. Seen this personally to a smaller extent.
Quoted 36 lines
Quoted 22 lines
Quoted 22 lines
Signed-off-by Pavel Begunkov <[email protected]>--- fs/btrfs/discard.c | 10 +++++----- 1 file changed, 5 insertions(+), 5 deletions(-)fs/btrfs/discard.c +0
-1@@ -519,7 +519,6 @@ void btrfs_discard_calc_delay(struct btrfs_discard_ctl *discard_ctl)s64 discardable_bytes;u32 iops_limit;unsigned long delay; -unsigned long lower_limit =BTRFS_DISCARD_MIN_DELAY_MSEC ; discardable_extents =atomic_read (&discard_ctl->discardable_extents);if (!discardable_extents)@@ -550,11 +549,12 @@ void btrfs_discard_calc_delay(struct btrfs_discard_ctl *discard_ctl)Quoted 11 lines
fs/btrfs/discard.c void btrfs_discard_calc_delay(struct btrfs_discard_ctl *discard_ctl) · line 554
Quoted 7 lines
fs/btrfs/discard.c +3
-2iops_limit =READ_ONCE (discard_ctl->iops_limit);if (iops_limit) - lower_limit = max_t(unsigned long, lower_limit, -MSEC_PER_SEC / iops_limit); + delay =MSEC_PER_SEC / iops_limit; +else + delay =BTRFS_DISCARD_TARGET_MSEC / discardable_extents;Looks good to me. I wonder why there wasn't handling of if
iops_limitwas unfindable
before?Not sure what you mean by unfindable, but async discard is relatively new, might be that everyone just have their hands full.
Quoted 6 lines
Quoted 4 lines
Quoted 2 lines
By unfindable I mean if
iops_limitturned up as null when reading it fromdiscard_ctl.Ahh, ok. It's handled and I left it as it was, that BTW is still a problem.
How often is
iops_limitunfindable?
I don't know, but the default is 10, so shouldn't be too ubiquitous. Maybe someone here knows statistics.
Quoted 42 lines
Quoted 5 lines
First it calculates a delay based on number of queued extents and than clamps it to (
BTRFS_DISCARD_MIN_DELAY_MSEC,BTRFS_DISCARD_MAX_DELAY_MSEC). Without this patch it did the same but the lower bound was calculated fromiops_limit.Thanks for clarifying.
Quoted 6 lines
Quoted 3 lines
Async discard was added in 5.6, correct? So yeah, makes sense then that people just had their hands full. Thanks for adding it.
b0643e59cfa609c4b5f("btrfs: add the beginning of async discard, discard workqueue"). Dec 2019, so less than a yearThanks for finding the commit.
Quoted 16 lines
Quoted 16 lines
Quoted 16 lines
Quoted 16 lines
Quoted 15 lines
fs/btrfs/discard.c +2
-3- delay =BTRFS_DISCARD_TARGET_MSEC / discardable_extents; - delay =clamp (delay, lower_limit,BTRFS_DISCARD_MAX_DELAY_MSEC ); + delay =clamp (delay,BTRFS_DISCARD_MIN_DELAY_MSEC , +BTRFS_DISCARD_MAX_DELAY_MSEC ); discard_ctl->delay =msecs_to_jiffies (delay);spin_unlock (&discard_ctl->lock); --2.24.0
Quoted 4 lines
Quoted 1 line
Quoted 1 line
Quoted 1 line
This patch looks all great to me.
--
Pavel BegunkovBest regards,
Amy Parker
(they/them)
Signature · 1 line
Pavel Begunkov
Re: [PATCH 1/4] btrfs: discard: speed up discard up to iops_limit
Quoted 71 lines · Pavel Begunkov
Quoted 69 lines · Amy Parker
Quoted 68 lines · Pavel Begunkov
Quoted 65 lines · Amy Parker
Quoted 8 lines · Pavel Begunkov
Quoted 6 lines · Amy Parker
Quoted 4 lines · Pavel Begunkov
Instead of using
iops_limitonly for cutting off extremes, calculate the discard delay directly from it, so it closely followsiops_limitand doesn't under-discarding even though quotas are not saturated.Quoted 7 lines
Quoted 2 lines
This sounds like it potentially be a great performance boost, do you have any performance metrics regarding this patch?
Boosting the discard rate and so reaping stalling blocks may be nice, but unless it holds too much memory creating lack of space it shouldn't affect throughput. Though, it's better to ask people with deeper understanding of the fs.
Alright, thanks for the clarification.
Quoted 4 lines
What I've seen is that in some cases there are extents staying queued for discarding for _too_ long. E.g. reaping a small number of very fat extents keeps delay at max and doesn't allow to reap them effectively. That could be a problem with fast drives.
Ah, yep. Seen this personally to a smaller extent.
Quoted 34 lines
Quoted 22 lines
Quoted 22 lines
Signed-off-by Pavel Begunkov <[email protected]>--- fs/btrfs/discard.c | 10 +++++----- 1 file changed, 5 insertions(+), 5 deletions(-)fs/btrfs/discard.c +0
-1@@ -519,7 +519,6 @@ void btrfs_discard_calc_delay(struct btrfs_discard_ctl *discard_ctl)s64 discardable_bytes;u32 iops_limit;unsigned long delay; -unsigned long lower_limit =BTRFS_DISCARD_MIN_DELAY_MSEC ; discardable_extents =atomic_read (&discard_ctl->discardable_extents);if (!discardable_extents)@@ -550,11 +549,12 @@ void btrfs_discard_calc_delay(struct btrfs_discard_ctl *discard_ctl)Quoted 11 lines
fs/btrfs/discard.c void btrfs_discard_calc_delay(struct btrfs_discard_ctl *discard_ctl) · line 554
Quoted 7 lines
fs/btrfs/discard.c +3
-2iops_limit =READ_ONCE (discard_ctl->iops_limit);if (iops_limit) - lower_limit = max_t(unsigned long, lower_limit, -MSEC_PER_SEC / iops_limit); + delay =MSEC_PER_SEC / iops_limit; +else + delay =BTRFS_DISCARD_TARGET_MSEC / discardable_extents;Looks good to me. I wonder why there wasn't handling of if
iops_limitwas unfindable
before?Quoted 2 lines
Not sure what you mean by unfindable, but async discard is relatively new, might be that everyone just have their hands full.
By unfindable I mean if
iops_limitturned up as null when reading it fromdiscard_ctl.
Quoted 6 lines
Quoted 3 lines
Quoted 1 line
Ahh, ok. It's handled and I left it as it was, that BTW is still a problem.
How often is
iops_limitunfindable?I don't know, but the default is 10, so shouldn't be too ubiquitous. Maybe someone here knows statistics.
So it isn't a major issue right now, and we can just have it looked at whenever someone has the time to.
Quoted 39 lines
Quoted 39 lines
Quoted 6 lines
First it calculates a delay based on number of queued extents and than clamps it to (
BTRFS_DISCARD_MIN_DELAY_MSEC,BTRFS_DISCARD_MAX_DELAY_MSEC). Without this patch it did the same but the lower bound was calculated fromiops_limit.Thanks for clarifying.
Quoted 6 lines
Quoted 3 lines
Async discard was added in 5.6, correct? So yeah, makes sense then that people just had their hands full. Thanks for adding it.
b0643e59cfa609c4b5f("btrfs: add the beginning of async discard, discard workqueue"). Dec 2019, so less than a yearThanks for finding the commit.
Quoted 20 lines
Quoted 11 lines
Quoted 11 lines
Quoted 11 lines
Quoted 11 lines
fs/btrfs/discard.c +2
-2- delay =BTRFS_DISCARD_TARGET_MSEC / discardable_extents; - delay =clamp (delay, lower_limit,BTRFS_DISCARD_MAX_DELAY_MSEC ); + delay =clamp (delay,BTRFS_DISCARD_MIN_DELAY_MSEC , +BTRFS_DISCARD_MAX_DELAY_MSEC ); discard_ctl->delay =msecs_to_jiffies (delay);Quoted 6 lines
Quoted 6 lines
Quoted 6 lines
Quoted 4 lines
fs/btrfs/discard.c +0
-1spin_unlock (&discard_ctl->lock); --2.24.0
This patch looks all great to me.
--
Pavel Begunkov
Quoted 7 lines
Quoted 4 lines
Best regards,
Amy Parker
(they/them)--
Pavel Begunkov
Best regards,
Amy Parker
(they/them)
Re: [PATCH 1/4] btrfs: discard: speed up discard up to iops_limit
In reply to Pavel BegunkovQuoted 5 lines · Pavel Begunkov
Instead of using
iops_limitonly for cutting off extremes, calculate the discard delay directly from it, so it closely followsiops_limitand doesn't under-discarding even though quotas are not saturated.Signed-off-by Pavel Begunkov <[email protected]>
Thanks,
Josef
Re: [PATCH 2/4] btrfs: discard: save discard delay as ns not jiffy
In reply to Pavel BegunkovQuoted 4 lines · Pavel Begunkov
Most of calculations are done in ns or ms, so store
discard_ctl->delayin ms and convert the final delay to jiffies only in the end.Signed-off-by Pavel Begunkov <[email protected]>
Thanks,
Josef
Re: [PATCH 3/4] btrfs: don't miss discards after override-schedule
In reply to Pavel BegunkovQuoted 45 lines · Pavel Begunkov
If
btrfs_discard_schedule_work()is called with override=true, it sets delay anew regardless how much time left until the timer should have fired. If delays are long (that can happen, for example, with lowkbps_limit), they might be constantly overriden without having a chance to run the discard work.Signed-off-by Pavel Begunkov <[email protected]>--- fs/btrfs/ctree.h | 1 + fs/btrfs/discard.c | 11 +++++++++++ 2 files changed, 12 insertions(+)fs/btrfs/ctree.h +1
-0@@ -469,6 +469,7 @@ struct btrfs_discard_ctl {struct btrfs_block_group *block_group;struct list_head discard_list[BTRFS_NR_DISCARD_LISTS ];u64 prev_discard; +u64 prev_discard_time;atomic_t discardable_extents;atomic64_t discardable_bytes;u64 max_discard_size;fs/btrfs/discard.c +9
-0@@ -381,6 +381,15 @@ void btrfs_discard_schedule_work(struct btrfs_discard_ctl *discard_ctl, delay =max (delay, bg_timeout); } +if (override && discard_ctl->prev_discard) { +u64 elapsed = now - discard_ctl->prev_discard_time; + +if (delay > elapsed) + delay -= elapsed; +else + delay =0 ; + } +mod_delayed_work (discard_ctl->discard_workers, &discard_ctl->work,nsecs_to_jiffies (delay)); }
fs/btrfs/discard.c static void btrfs_discard_workfn(struct work_struct *work) · line 478
Quoted 4 lines
fs/btrfs/discard.c +1
-0@@ -466,6 +475,7 @@ static void btrfs_discard_workfn(struct work_struct *work) } discard_ctl->prev_discard = trimmed; + discard_ctl->prev_discard_time =ktime_get_ns ();
I noticed these weren't protected by the discard_ctl->lock, so I went to look at if that was ok. It appears to be ok, since this is the workfn, and we only read them if there's no pending work, so we're protected there. Just a note for anybody else who finds it weird, though I wouldn't argue with protecting it with a lock just to remove any ambiguity.
Thanks,
Josef
Re: [PATCH 4/4] btrfs: discard: reschedule work after param update
In reply to Pavel BegunkovQuoted 6 lines · Pavel Begunkov
After sysfs updates discard's
iops_limitorkbps_limitit also needs to adjust current timer through rescheduling, otherwise the discard work may wait for a long time for the previous timer to expier or bumped by someone else.Signed-off-by Pavel Begunkov <[email protected]>
Thanks,
Josef
Re: [PATCH 3/4] btrfs: don't miss discards after override-schedule
In reply to Josef BacikQuoted 42 lines · Josef Bacik
Quoted 41 lines · Pavel Begunkov
If
btrfs_discard_schedule_work()is called with override=true, it sets delay anew regardless how much time left until the timer should have fired. If delays are long (that can happen, for example, with lowkbps_limit), they might be constantly overriden without having a chance to run the discard work.Signed-off-by Pavel Begunkov <[email protected]>---
fs/btrfs/ctree.h | 1 +
fs/btrfs/discard.c| 11 +++++++++++2 files changed, 12 insertions(+)fs/btrfs/ctree.h +1
-0@@ -469,6 +469,7 @@ struct btrfs_discard_ctl { struct btrfs_block_group *block_group; struct list_head discard_list[BTRFS_NR_DISCARD_LISTS]; u64 prev_discard; + u64 prev_discard_time; atomic_t discardable_extents; atomic64_t discardable_bytes; u64 max_discard_size;fs/btrfs/discard.c +8
-0@@ -381,6 +381,15 @@ void btrfs_discard_schedule_work(struct btrfs_discard_ctl *discard_ctl, delay = max(delay, bg_timeout); } + if (override && discard_ctl->prev_discard) { + u64 elapsed = now - discard_ctl->prev_discard_time; + + if (delay > elapsed) + delay -= elapsed; + else + delay = 0; + } + mod_delayed_work(discard_ctl->discard_workers,
Quoted 8 lines
fs/btrfs/discard.c static void btrfs_discard_workfn(struct work_struct *work) · line 477
Quoted 6 lines
fs/btrfs/discard.c +1
-0&discard_ctl->work, nsecs_to_jiffies(delay)); }@@ -466,6 +475,7 @@ static void btrfs_discard_workfn(struct work_struct *work) } discard_ctl->prev_discard = trimmed; + discard_ctl->prev_discard_time = ktime_get_ns();I noticed these weren't protected by the
discard_ctl->lock, so I went to look at if that was ok. It appears to be ok, since this is the workfn, and we only read them if there's no pending work, so we're protected there. Just a note for anybody else who finds it weird, though I wouldn't argue with protecting it with a lock just to remove any ambiguity.
Agree, together with ->prev_discard. Or at least there should be a comment.
Quoted 2 lines
Reviewed-by Josef Bacik <[email protected]>
Thanks
Signature · 1 line
Pavel Begunkov
Quoted 4 lines · Pavel Begunkov
Several fixes for async discards. The first patch might increase discard rate, drastically in some cases. That may be a suprise for those assuming that hitting
iops_limitis rare and rarther outliers. Though, it still stays in allowed range, so should be fine.
I think this highly depends on the workload, if you really need to issue the discards fast because of the rate of the change in the regular data. That was the point of the async discard and the knobs, the defaults should be ok for most users and allow adjusting for specific loads.
My testing of the original discard patchset was tailored towards the default usecase and likely intense than yours. I did not observe the corner cases where the work queue scheduling was off, or changing the sysfs values did not poke the right kthreads.
Patches look ok to me and I've added them to topic branch for testing, going to misc-next later. Thanks.
Quoted 10 lines · David Sterba
Quoted 4 lines · Pavel Begunkov
Several fixes for async discards. The first patch might increase discard rate, drastically in some cases. That may be a suprise for those assuming that hitting
iops_limitis rare and rarther outliers. Though, it still stays in allowed range, so should be fine.I think this highly depends on the workload, if you really need to issue the discards fast because of the rate of the change in the regular data. That was the point of the async discard and the knobs, the defaults should be ok for most users and allow adjusting for specific loads.
Chris mentioned that _there are_ problems with faster drives though. The problem is that this iops_limit knot just clamps the chosen delay. Ultimately, I want to find later a better delay function than
delay = CONSTANT_INTERVAL_MS / nr_extents.
But that will take some thinking.
For instance, one of the cases I've seen is recycling large extents like deletion of subvolumes. There we have a small number of extents but each takes a lot of space, so there are, say, 100+GB queued to be discarded. But because there are few extents, delay is calculated to >10s that's then clamped to a constant max limit.
That was taking a long to recycle. Not sure though how many bytes/extents are discarded on each iteration of btrfs_discard_workfn().
Quoted 9 lines
My testing of the original discard patchset was tailored towards the default usecase and likely intense than yours. I did not observe the corner cases where the work queue scheduling was off, or changing the sysfs values did not poke the right kthreads.
Patches look ok to me and I've added them to topic branch for testing, going to misc-next later. Thanks.
Signature · 1 line
Pavel Begunkov
Quoted 20 lines · Pavel Begunkov
Quoted 10 lines · David Sterba
Quoted 4 lines · Pavel Begunkov
Several fixes for async discards. The first patch might increase discard rate, drastically in some cases. That may be a suprise for those assuming that hitting
iops_limitis rare and rarther outliers. Though, it still stays in allowed range, so should be fine.I think this highly depends on the workload, if you really need to issue the discards fast because of the rate of the change in the regular data. That was the point of the async discard and the knobs, the defaults should be ok for most users and allow adjusting for specific loads.
Chris mentioned that
_thereare_problems with faster drives though. The problem is that thisiops_limitknot just clamps the chosen delay. Ultimately, I want to find later a better delay function thandelay =
CONSTANT_INTERVAL_MS/nr_extents.But that will take some thinking.
Quoted 7 lines
For instance, one of the cases I've seen is recycling large extents like deletion of subvolumes. There we have a small number of extents but each takes a lot of space, so there are, say, 100+GB queued to be discarded. But because there are few extents, delay is calculated to >10s that's then clamped to a constant max limit.
That was taking a long to recycle. Not sure though how many bytes/extents are discarded on each iteration ofbtrfs_discard_workfn().
BTRFS_ASYNC_DISCARD_DEFAULT_MAX_SIZE 64M
So the few large extents do not fit current scheme. I thought some translation to "logical" units could do the same discard io scheduling. 100G of size would become N times maximum discard extent units (N = 100G / max) and submitted for discard until N is 0 for a given input range.
But if you know you'll have ranges that are orders of magnitude larger than the extent size, then setting the sysfs value accordingly seems like the right approach. I'm not sure if the freed ranges are coalesced before adding them to the discard list. If not, then icreasing the max size should work, otherwise the coalescing could be adjusted.
Quoted 12 lines · Pavel Begunkov
Quoted 10 lines · David Sterba
Quoted 4 lines · Pavel Begunkov
Several fixes for async discards. The first patch might increase discard rate, drastically in some cases. That may be a suprise for those assuming that hitting
iops_limitis rare and rarther outliers. Though, it still stays in allowed range, so should be fine.I think this highly depends on the workload, if you really need to issue the discards fast because of the rate of the change in the regular data. That was the point of the async discard and the knobs, the defaults should be ok for most users and allow adjusting for specific loads.
Quoted 7 lines
Chris mentioned that
_thereare_problems with faster drives though. The problem is that thisiops_limitknot just clamps the chosen delay. Ultimately, I want to find later a better delay function thandelay =
CONSTANT_INTERVAL_MS/nr_extents.But that will take some thinking.
Yeah, we have drives where latencies increase as the drive fills up. Async discard moves the knee on utilization percentage before the latencies increase, but without patches like these, we do still have garbage collection related stalls. We still need to measure if the GC related stalls turn into discard-saturation stalls, but at least now we can try.
-chris