sched/fair: don't push cfs_bandwith slack timers forward
A message that is not in the archive (sent off-list or not yet received): [email protected]
Xunlei Pang
Reviewed-byQuoted 3 lines · On 2019/6/6 AM 4:06, [email protected] wrote:
When a
cfs_rqsleeps and returns its quota, we delay for 5ms before waking any throttledcfs_rqsto coalesce with othercfs_rqsgoing to sleep, as this has has to be done outside of the rq lock we hold.
two "has".
Quoted 11 lines
The current code waits for 5ms without any sleeps, instead of waiting for 5ms from the first sleep, which can delay the unthrottle more than we want. Switch this around so that we can't push this forward forever.
This requires an extra flag rather than using
hrtimer_active, since we need to start a new timer if the current one is in the process of finishing.Signed-off-by Ben Segall <[email protected]>---
We've also suffered from this performance issue recently:
Quoted 49 lines
kernel/sched/fair.c | 7 +++++++ kernel/sched/sched.h | 1 + 2 files changed, 8 insertions(+)kernel/sched/fair.c +7
-0@@ -4729,6 +4729,11 @@ static void start_cfs_slack_bandwidth(struct cfs_bandwidth *cfs_b)if (runtime_refresh_within (cfs_b, min_left))return ; +/* don't push forwards an existing deferred unthrottle */ +if (cfs_b->slack_started) +return ; + cfs_b->slack_started = true; +hrtimer_start (&cfs_b->slack_timer,ns_to_ktime (cfs_bandwidth_slack_period),HRTIMER_MODE_REL );@@ -4782,6 +4787,7 @@ static void do_sched_cfs_slack_timer(struct cfs_bandwidth *cfs_b)/* confirm we're still not at a refresh boundary */ raw_spin_lock_irqsave (&cfs_b->lock, flags); + cfs_b->slack_started = false;if (cfs_b->distribute_running) {raw_spin_unlock_irqrestore (&cfs_b->lock, flags);return ;@@ -4920,6 +4926,7 @@ void init_cfs_bandwidth(struct cfs_bandwidth *cfs_b)hrtimer_init (&cfs_b->slack_timer,CLOCK_MONOTONIC ,HRTIMER_MODE_REL ); cfs_b->slack_timer.function = sched_cfs_slack_timer; cfs_b->distribute_running =0 ; + cfs_b->slack_started = false; }static void init_cfs_rq_runtime (struct cfs_rq *cfs_rq)kernel/sched/sched.h +1
-0@@ -356,6 +356,7 @@ struct cfs_bandwidth {u64 throttled_time;bool distribute_running; +bool slack_started;#endif };
[email protected]
reviewerPATCH v2 1/1 sched/fair: don't push cfs_bandwith slack timers forward
When a cfs_rq sleeps and returns its quota, we delay for 5ms before waking any throttled cfs_rqs to coalesce with other cfs_rqs going to sleep, as this has to be done outside of the rq lock we hold.
The current code waits for 5ms without any sleeps, instead of waiting for 5ms from the first sleep, which can delay the unthrottle more than we want. Switch this around so that we can't push this forward forever.
This requires an extra flag rather than using hrtimer_active, since we need to start a new timer if the current one is in the process of finishing.
--- kernel/sched/fair.c | 7 +++++++ kernel/sched/sched.h | 1 + 2 files changed, 8 insertions(+)
kernel/sched/fair.c +7 -0
@@ -4729,6 +4729,11 @@ static void start_cfs_slack_bandwidth(struct cfs_bandwidth *cfs_b)if (runtime_refresh_within (cfs_b, min_left))return ; +/* don't push forwards an existing deferred unthrottle */ +if (cfs_b->slack_started) +return ; + cfs_b->slack_started = true; +hrtimer_start (&cfs_b->slack_timer,ns_to_ktime (cfs_bandwidth_slack_period),HRTIMER_MODE_REL );
@@ -4782,6 +4787,7 @@ static void do_sched_cfs_slack_timer(struct cfs_bandwidth *cfs_b)/* confirm we're still not at a refresh boundary */ raw_spin_lock_irqsave (&cfs_b->lock, flags); + cfs_b->slack_started = false;if (cfs_b->distribute_running) {raw_spin_unlock_irqrestore (&cfs_b->lock, flags);return ;
@@ -4920,6 +4926,7 @@ void init_cfs_bandwidth(struct cfs_bandwidth *cfs_b)hrtimer_init (&cfs_b->slack_timer,CLOCK_MONOTONIC ,HRTIMER_MODE_REL ); cfs_b->slack_timer.function = sched_cfs_slack_timer; cfs_b->distribute_running =0 ; + cfs_b->slack_started = false; }static void init_cfs_rq_runtime (struct cfs_rq *cfs_rq)
kernel/sched/sched.h +1 -0
@@ -356,6 +356,7 @@ struct cfs_bandwidth {u64 throttled_time;bool distribute_running; +bool slack_started;#endif };
Signature · 1 line
2.22.0.rc1.257.g3120a18244-goog
Quoted 66 lines · On Thu, Jun 06, 2019 at 10:21:01AM -0700 [email protected] wrote:
When a
cfs_rqsleeps and returns its quota, we delay for 5ms before waking any throttledcfs_rqsto coalesce with othercfs_rqsgoing to sleep, as this has to be done outside of the rq lock we hold.The current code waits for 5ms without any sleeps, instead of waiting for 5ms from the first sleep, which can delay the unthrottle more than we want. Switch this around so that we can't push this forward forever.
This requires an extra flag rather than using
hrtimer_active, since we need to start a new timer if the current one is in the process of finishing.Signed-off-by Ben Segall <[email protected]>Reviewed-by Xunlei Pang <[email protected]>--- kernel/sched/fair.c | 7 +++++++ kernel/sched/sched.h | 1 + 2 files changed, 8 insertions(+)kernel/sched/fair.c +7
-0@@ -4729,6 +4729,11 @@ static void start_cfs_slack_bandwidth(struct cfs_bandwidth *cfs_b)if (runtime_refresh_within (cfs_b, min_left))return ; +/* don't push forwards an existing deferred unthrottle */ +if (cfs_b->slack_started) +return ; + cfs_b->slack_started = true; +hrtimer_start (&cfs_b->slack_timer,ns_to_ktime (cfs_bandwidth_slack_period),HRTIMER_MODE_REL );@@ -4782,6 +4787,7 @@ static void do_sched_cfs_slack_timer(struct cfs_bandwidth *cfs_b)/* confirm we're still not at a refresh boundary */ raw_spin_lock_irqsave (&cfs_b->lock, flags); + cfs_b->slack_started = false;if (cfs_b->distribute_running) {raw_spin_unlock_irqrestore (&cfs_b->lock, flags);return ;@@ -4920,6 +4926,7 @@ void init_cfs_bandwidth(struct cfs_bandwidth *cfs_b)hrtimer_init (&cfs_b->slack_timer,CLOCK_MONOTONIC ,HRTIMER_MODE_REL ); cfs_b->slack_timer.function = sched_cfs_slack_timer; cfs_b->distribute_running =0 ; + cfs_b->slack_started = false; }static void init_cfs_rq_runtime (struct cfs_rq *cfs_rq)kernel/sched/sched.h +1
-0@@ -356,6 +356,7 @@ struct cfs_bandwidth {u64 throttled_time;bool distribute_running; +bool slack_started;#endif };Signature · 1 line
2.22.0.rc1.257.g3120a18244-goog
I think this looks good. I like not delaying that further even if it does not fix Dave's use case.
It does make it glaring that I should have used false/true for setting distribute_running though :)
Peter Zijlstra
maintainerQuoted 71 lines · Phil Auld
Quoted 66 lines · On Thu, Jun 06, 2019 at 10:21:01AM -0700 [email protected] wrote:
When a
cfs_rqsleeps and returns its quota, we delay for 5ms before waking any throttledcfs_rqsto coalesce with othercfs_rqsgoing to sleep, as this has to be done outside of the rq lock we hold.The current code waits for 5ms without any sleeps, instead of waiting for 5ms from the first sleep, which can delay the unthrottle more than we want. Switch this around so that we can't push this forward forever.
This requires an extra flag rather than using
hrtimer_active, since we need to start a new timer if the current one is in the process of finishing.Signed-off-by Ben Segall <[email protected]>Reviewed-by Xunlei Pang <[email protected]>--- kernel/sched/fair.c | 7 +++++++ kernel/sched/sched.h | 1 + 2 files changed, 8 insertions(+)kernel/sched/fair.c +7
-0@@ -4729,6 +4729,11 @@ static void start_cfs_slack_bandwidth(struct cfs_bandwidth *cfs_b)if (runtime_refresh_within (cfs_b, min_left))return ; +/* don't push forwards an existing deferred unthrottle */ +if (cfs_b->slack_started) +return ; + cfs_b->slack_started = true; +hrtimer_start (&cfs_b->slack_timer,ns_to_ktime (cfs_bandwidth_slack_period),HRTIMER_MODE_REL );@@ -4782,6 +4787,7 @@ static void do_sched_cfs_slack_timer(struct cfs_bandwidth *cfs_b)/* confirm we're still not at a refresh boundary */ raw_spin_lock_irqsave (&cfs_b->lock, flags); + cfs_b->slack_started = false;if (cfs_b->distribute_running) {raw_spin_unlock_irqrestore (&cfs_b->lock, flags);return ;@@ -4920,6 +4926,7 @@ void init_cfs_bandwidth(struct cfs_bandwidth *cfs_b)hrtimer_init (&cfs_b->slack_timer,CLOCK_MONOTONIC ,HRTIMER_MODE_REL ); cfs_b->slack_timer.function = sched_cfs_slack_timer; cfs_b->distribute_running =0 ; + cfs_b->slack_started = false; }static void init_cfs_rq_runtime (struct cfs_rq *cfs_rq)kernel/sched/sched.h +1
-0@@ -356,6 +356,7 @@ struct cfs_bandwidth {u64 throttled_time;bool distribute_running; +bool slack_started;#endif };Signature · 1 line
2.22.0.rc1.257.g3120a18244-googI think this looks good. I like not delaying that further even if it does not fix Dave's use case.
Quoted 5 lines
It does make it glaring that I should have used false/true for setting
distribute_runningthough :)Acked-by Phil Auld <[email protected]>
Thanks!
Should this patch have a Fixes: tag?
Peter Zijlstra
maintainerkernel/sched/sched.h struct cfs_bandwidth { · line 361
Quoted 11 lines · On Thu, Jun 06, 2019 at 10:21:01AM -0700, [email protected] wrote:
kernel/sched/sched.h +1
-0@@ -356,6 +356,7 @@ struct cfs_bandwidth {u64 throttled_time;bool distribute_running; +bool slack_started;#endif };
I'm thinking we can this instead? afaict both idle and period_active are already effecitively booleans and don't need the full 16 bits.
kernel/sched/sched.h +4 -5
@@ -338,8 +338,10 @@ struct cfs_bandwidth {u64 runtime_expires;int expires_seq; -short idle; -short period_active; +u8 idle; +u8 period_active; +u8 distribute_running; +u8 slack_started;struct hrtimer period_timer;struct hrtimer slack_timer;struct list_head throttled_cfs_rq;
@@ -348,9 +350,6 @@ struct cfs_bandwidth {int nr_periods;int nr_throttled;u64 throttled_time; - -bool distribute_running; -bool slack_started;#endif };
Quoted 32 lines · On Tue, Jun 11, 2019 at 03:53:25PM +0200 Peter Zijlstra wrote:
kernel/sched/sched.h struct cfs_bandwidth { · line 361
Quoted 11 lines · On Thu, Jun 06, 2019 at 10:21:01AM -0700, [email protected] wrote:
kernel/sched/sched.h +1
-0@@ -356,6 +356,7 @@ struct cfs_bandwidth {u64 throttled_time;bool distribute_running; +bool slack_started;#endif };I'm thinking we can this instead? afaict both idle and
period_activeare already effecitively booleans and don't need the full 16 bits.kernel/sched/sched.h +4
-2@@ -338,8 +338,10 @@ struct cfs_bandwidth {u64 runtime_expires;int expires_seq; -short idle; -short period_active; +u8 idle; +u8 period_active; +u8 distribute_running; +u8 slack_started;struct hrtimer period_timer;struct hrtimer slack_timer;struct list_head throttled_cfs_rq;
kernel/sched/sched.h struct cfs_bandwidth { · line 355
Quoted 9 lines
kernel/sched/sched.h +0
-3@@ -348,9 +350,6 @@ struct cfs_bandwidth {int nr_periods;int nr_throttled;u64 throttled_time; - -bool distribute_running; -bool slack_started;#endif };
That looks reasonable to me.
Out of curiosity, why not bool? Is sizeof bool architecture dependent?
Peter Zijlstra
maintainerQuoted 3 lines · Phil Auld
That looks reasonable to me.
Out of curiosity, why not bool? Is sizeof bool architecture dependent?
Yeah, sizeof(_Bool) is unspecified and depends on ABI. It is mostly 1, but there are known cases where it is 4.
Quoted 8 lines · On Tue, Jun 11, 2019 at 04:24:43PM +0200 Peter Zijlstra wrote:
Quoted 3 lines · Phil Auld
That looks reasonable to me.
Out of curiosity, why not bool? Is sizeof bool architecture dependent?
Yeah,
sizeof(_Bool)is unspecified and depends on ABI. It is mostly 1, but there are known cases where it is 4.
Makes sense. Thanks!
[email protected]
reviewerQuoted 32 lines · Peter Zijlstra
kernel/sched/sched.h struct cfs_bandwidth { · line 361
Quoted 11 lines · On Thu, Jun 06, 2019 at 10:21:01AM -0700, [email protected] wrote:
kernel/sched/sched.h +1
-0@@ -356,6 +356,7 @@ struct cfs_bandwidth {u64 throttled_time;bool distribute_running; +bool slack_started;#endif };I'm thinking we can this instead? afaict both idle and
period_activeare already effecitively booleans and don't need the full 16 bits.kernel/sched/sched.h +4
-2@@ -338,8 +338,10 @@ struct cfs_bandwidth {u64 runtime_expires;int expires_seq; -short idle; -short period_active; +u8 idle; +u8 period_active; +u8 distribute_running; +u8 slack_started;struct hrtimer period_timer;struct hrtimer slack_timer;struct list_head throttled_cfs_rq;
kernel/sched/sched.h struct cfs_bandwidth { · line 355
Quoted 9 lines
kernel/sched/sched.h +0
-3@@ -348,9 +350,6 @@ struct cfs_bandwidth {int nr_periods;int nr_throttled;u64 throttled_time; - -bool distribute_running; -bool slack_started;#endif };
Yeah, that makes sense to me, should I spin up another version of the patch doing this too or do you have it from here?
tip-bot for [email protected]
merge bot[tip:sched/core] sched/fair: Don't push cfs_bandwith slack timers forward
In reply to [email protected]sched/fair: Don't push cfs_bandwith slack timers forward
Commit-ID: 66567fcbaecac455caa1b13643155d686b51ce63 Gitweb: https://git.kernel.org/tip/66567fcbaecac455caa1b13643155d686b51ce63 Author: [email protected] <[email protected]> AuthorDate: Thu, 6 Jun 2019 10:21:01 -0700 Committer: Ingo Molnar <[email protected]> CommitDate: Mon, 17 Jun 2019 12:16:01 +0200
sched/fair: Don't push cfs_bandwith slack timers forward
When a cfs_rq sleeps and returns its quota, we delay for 5ms before waking any throttled cfs_rqs to coalesce with other cfs_rqs going to sleep, as this has to be done outside of the rq lock we hold.
The current code waits for 5ms without any sleeps, instead of waiting for 5ms from the first sleep, which can delay the unthrottle more than we want. Switch this around so that we can't push this forward forever.
This requires an extra flag rather than using hrtimer_active, since we need to start a new timer if the current one is in the process of finishing.
--- kernel/sched/fair.c | 7 +++++++ kernel/sched/sched.h | 8 ++++---- 2 files changed, 11 insertions(+), 4 deletions(-)
kernel/sched/fair.c +7 -0
@@ -4729,6 +4729,11 @@ static void start_cfs_slack_bandwidth(struct cfs_bandwidth *cfs_b)if (runtime_refresh_within (cfs_b, min_left))return ; +/* don't push forwards an existing deferred unthrottle */ +if (cfs_b->slack_started) +return ; + cfs_b->slack_started = true; +hrtimer_start (&cfs_b->slack_timer,ns_to_ktime (cfs_bandwidth_slack_period),HRTIMER_MODE_REL );
@@ -4782,6 +4787,7 @@ static void do_sched_cfs_slack_timer(struct cfs_bandwidth *cfs_b)/* confirm we're still not at a refresh boundary */ raw_spin_lock_irqsave (&cfs_b->lock, flags); + cfs_b->slack_started = false;if (cfs_b->distribute_running) {raw_spin_unlock_irqrestore (&cfs_b->lock, flags);return ;
@@ -4945,6 +4951,7 @@ void init_cfs_bandwidth(struct cfs_bandwidth *cfs_b)hrtimer_init (&cfs_b->slack_timer,CLOCK_MONOTONIC ,HRTIMER_MODE_REL ); cfs_b->slack_timer.function = sched_cfs_slack_timer; cfs_b->distribute_running =0 ; + cfs_b->slack_started = false; }static void init_cfs_rq_runtime (struct cfs_rq *cfs_rq)
kernel/sched/sched.h +4 -4
@@ -338,8 +338,10 @@ struct cfs_bandwidth {u64 runtime_expires;int expires_seq; -short idle; -short period_active; +u8 idle; +u8 period_active; +u8 distribute_running; +u8 slack_started;struct hrtimer period_timer;struct hrtimer slack_timer;struct list_head throttled_cfs_rq;
@@ -348,8 +350,6 @@ struct cfs_bandwidth {int nr_periods;int nr_throttled;u64 throttled_time; - -bool distribute_running;#endif };