Skip to contentSkip to conversation

sched/fair: don't push cfs_bandwith slack timers forward

PATCH In review Xunlei Pang 5 participants 10 messages Jun 6, 2019 – Jun 17, 2019 1 message not in the archive Scheduler

lore
Share & tools
lore

A message that is not in the archive (sent off-list or not yet received): [email protected]

Xunlei Pang

author Reviewed-by
#1 lore raw
Quoted 3 lines · On 2019/6/6 AM 4:06, [email protected] wrote:

When a cfs_rq sleeps and returns its quota, we delay for 5ms before waking any throttled cfs_rqs to coalesce with other cfs_rqs going to sleep, as this has has to be done outside of the rq lock we hold.

two "has".

Quoted 11 lines

The current code waits for 5ms without any sleeps, instead of waiting for 5ms from the first sleep, which can delay the unthrottle more than we want. Switch this around so that we can't push this forward forever.

This requires an extra flag rather than using hrtimer_active, since we need to start a new timer if the current one is in the process of finishing.

---

We've also suffered from this performance issue recently:

Quoted 49 lines
 kernel/sched/fair.c  | 7 +++++++
 kernel/sched/sched.h | 1 +
 2 files changed, 8 insertions(+)
kernel/sched/fair.c +7 -0
index 8213ff6e365d..2ead252cfa32 100644
@@ -4729,6 +4729,11 @@ static void start_cfs_slack_bandwidth(struct cfs_bandwidth *cfs_b)
 	if (runtime_refresh_within(cfs_b, min_left))
 		return;
 
+	/* don't push forwards an existing deferred unthrottle */
+	if (cfs_b->slack_started)
+		return;
+	cfs_b->slack_started = true;
+
 	hrtimer_start(&cfs_b->slack_timer,
 			ns_to_ktime(cfs_bandwidth_slack_period),
 			HRTIMER_MODE_REL);
@@ -4782,6 +4787,7 @@ static void do_sched_cfs_slack_timer(struct cfs_bandwidth *cfs_b)
 
 	/* confirm we're still not at a refresh boundary */
 	raw_spin_lock_irqsave(&cfs_b->lock, flags);
+	cfs_b->slack_started = false;
 	if (cfs_b->distribute_running) {
 		raw_spin_unlock_irqrestore(&cfs_b->lock, flags);
 		return;
@@ -4920,6 +4926,7 @@ void init_cfs_bandwidth(struct cfs_bandwidth *cfs_b)
 	hrtimer_init(&cfs_b->slack_timer, CLOCK_MONOTONIC, HRTIMER_MODE_REL);
 	cfs_b->slack_timer.function = sched_cfs_slack_timer;
 	cfs_b->distribute_running = 0;
+	cfs_b->slack_started = false;
 }
 
 static void init_cfs_rq_runtime(struct cfs_rq *cfs_rq)
kernel/sched/sched.h +1 -0
index efa686eeff26..60219acda94b 100644
@@ -356,6 +356,7 @@ struct cfs_bandwidth {
 	u64			throttled_time;
 
 	bool                    distribute_running;
+	bool                    slack_started;
 #endif
 };
 
#2 lore raw

PATCH v2 1/1 sched/fair: don't push cfs_bandwith slack timers forward

When a cfs_rq sleeps and returns its quota, we delay for 5ms before waking any throttled cfs_rqs to coalesce with other cfs_rqs going to sleep, as this has to be done outside of the rq lock we hold.

The current code waits for 5ms without any sleeps, instead of waiting for 5ms from the first sleep, which can delay the unthrottle more than we want. Switch this around so that we can't push this forward forever.

This requires an extra flag rather than using hrtimer_active, since we need to start a new timer if the current one is in the process of finishing.

---
 kernel/sched/fair.c  | 7 +++++++
 kernel/sched/sched.h | 1 +
 2 files changed, 8 insertions(+)
kernel/sched/fair.c +7 -0
index 8213ff6e365d..2ead252cfa32 100644
@@ -4729,6 +4729,11 @@ static void start_cfs_slack_bandwidth(struct cfs_bandwidth *cfs_b)
 	if (runtime_refresh_within(cfs_b, min_left))
 		return;
 
+	/* don't push forwards an existing deferred unthrottle */
+	if (cfs_b->slack_started)
+		return;
+	cfs_b->slack_started = true;
+
 	hrtimer_start(&cfs_b->slack_timer,
 			ns_to_ktime(cfs_bandwidth_slack_period),
 			HRTIMER_MODE_REL);
@@ -4782,6 +4787,7 @@ static void do_sched_cfs_slack_timer(struct cfs_bandwidth *cfs_b)
 
 	/* confirm we're still not at a refresh boundary */
 	raw_spin_lock_irqsave(&cfs_b->lock, flags);
+	cfs_b->slack_started = false;
 	if (cfs_b->distribute_running) {
 		raw_spin_unlock_irqrestore(&cfs_b->lock, flags);
 		return;
@@ -4920,6 +4926,7 @@ void init_cfs_bandwidth(struct cfs_bandwidth *cfs_b)
 	hrtimer_init(&cfs_b->slack_timer, CLOCK_MONOTONIC, HRTIMER_MODE_REL);
 	cfs_b->slack_timer.function = sched_cfs_slack_timer;
 	cfs_b->distribute_running = 0;
+	cfs_b->slack_started = false;
 }
 
 static void init_cfs_rq_runtime(struct cfs_rq *cfs_rq)
kernel/sched/sched.h +1 -0
index efa686eeff26..60219acda94b 100644
@@ -356,6 +356,7 @@ struct cfs_bandwidth {
 	u64			throttled_time;
 
 	bool                    distribute_running;
+	bool                    slack_started;
 #endif
 };
 
Signature · 1 line
2.22.0.rc1.257.g3120a18244-goog
Acked-by on v2 1/1
#3 lore raw
Quoted 66 lines · On Thu, Jun 06, 2019 at 10:21:01AM -0700 [email protected] wrote:

When a cfs_rq sleeps and returns its quota, we delay for 5ms before waking any throttled cfs_rqs to coalesce with other cfs_rqs going to sleep, as this has to be done outside of the rq lock we hold.

The current code waits for 5ms without any sleeps, instead of waiting for 5ms from the first sleep, which can delay the unthrottle more than we want. Switch this around so that we can't push this forward forever.

This requires an extra flag rather than using hrtimer_active, since we need to start a new timer if the current one is in the process of finishing.

---
 kernel/sched/fair.c  | 7 +++++++
 kernel/sched/sched.h | 1 +
 2 files changed, 8 insertions(+)
kernel/sched/fair.c +7 -0
index 8213ff6e365d..2ead252cfa32 100644
@@ -4729,6 +4729,11 @@ static void start_cfs_slack_bandwidth(struct cfs_bandwidth *cfs_b)
 	if (runtime_refresh_within(cfs_b, min_left))
 		return;
 
+	/* don't push forwards an existing deferred unthrottle */
+	if (cfs_b->slack_started)
+		return;
+	cfs_b->slack_started = true;
+
 	hrtimer_start(&cfs_b->slack_timer,
 			ns_to_ktime(cfs_bandwidth_slack_period),
 			HRTIMER_MODE_REL);
@@ -4782,6 +4787,7 @@ static void do_sched_cfs_slack_timer(struct cfs_bandwidth *cfs_b)
 
 	/* confirm we're still not at a refresh boundary */
 	raw_spin_lock_irqsave(&cfs_b->lock, flags);
+	cfs_b->slack_started = false;
 	if (cfs_b->distribute_running) {
 		raw_spin_unlock_irqrestore(&cfs_b->lock, flags);
 		return;
@@ -4920,6 +4926,7 @@ void init_cfs_bandwidth(struct cfs_bandwidth *cfs_b)
 	hrtimer_init(&cfs_b->slack_timer, CLOCK_MONOTONIC, HRTIMER_MODE_REL);
 	cfs_b->slack_timer.function = sched_cfs_slack_timer;
 	cfs_b->distribute_running = 0;
+	cfs_b->slack_started = false;
 }
 
 static void init_cfs_rq_runtime(struct cfs_rq *cfs_rq)
kernel/sched/sched.h +1 -0
index efa686eeff26..60219acda94b 100644
@@ -356,6 +356,7 @@ struct cfs_bandwidth {
 	u64			throttled_time;
 
 	bool                    distribute_running;
+	bool                    slack_started;
 #endif
 };
 
Signature · 1 line
2.22.0.rc1.257.g3120a18244-goog

I think this looks good. I like not delaying that further even if it does not fix Dave's use case.

It does make it glaring that I should have used false/true for setting distribute_running though :)

maintainer
#4 lore raw
Quoted 71 lines · Phil Auld
Quoted 66 lines · On Thu, Jun 06, 2019 at 10:21:01AM -0700 [email protected] wrote:

When a cfs_rq sleeps and returns its quota, we delay for 5ms before waking any throttled cfs_rqs to coalesce with other cfs_rqs going to sleep, as this has to be done outside of the rq lock we hold.

The current code waits for 5ms without any sleeps, instead of waiting for 5ms from the first sleep, which can delay the unthrottle more than we want. Switch this around so that we can't push this forward forever.

This requires an extra flag rather than using hrtimer_active, since we need to start a new timer if the current one is in the process of finishing.

---
 kernel/sched/fair.c  | 7 +++++++
 kernel/sched/sched.h | 1 +
 2 files changed, 8 insertions(+)
kernel/sched/fair.c +7 -0
index 8213ff6e365d..2ead252cfa32 100644
@@ -4729,6 +4729,11 @@ static void start_cfs_slack_bandwidth(struct cfs_bandwidth *cfs_b)
 	if (runtime_refresh_within(cfs_b, min_left))
 		return;
 
+	/* don't push forwards an existing deferred unthrottle */
+	if (cfs_b->slack_started)
+		return;
+	cfs_b->slack_started = true;
+
 	hrtimer_start(&cfs_b->slack_timer,
 			ns_to_ktime(cfs_bandwidth_slack_period),
 			HRTIMER_MODE_REL);
@@ -4782,6 +4787,7 @@ static void do_sched_cfs_slack_timer(struct cfs_bandwidth *cfs_b)
 
 	/* confirm we're still not at a refresh boundary */
 	raw_spin_lock_irqsave(&cfs_b->lock, flags);
+	cfs_b->slack_started = false;
 	if (cfs_b->distribute_running) {
 		raw_spin_unlock_irqrestore(&cfs_b->lock, flags);
 		return;
@@ -4920,6 +4926,7 @@ void init_cfs_bandwidth(struct cfs_bandwidth *cfs_b)
 	hrtimer_init(&cfs_b->slack_timer, CLOCK_MONOTONIC, HRTIMER_MODE_REL);
 	cfs_b->slack_timer.function = sched_cfs_slack_timer;
 	cfs_b->distribute_running = 0;
+	cfs_b->slack_started = false;
 }
 
 static void init_cfs_rq_runtime(struct cfs_rq *cfs_rq)
kernel/sched/sched.h +1 -0
index efa686eeff26..60219acda94b 100644
@@ -356,6 +356,7 @@ struct cfs_bandwidth {
 	u64			throttled_time;
 
 	bool                    distribute_running;
+	bool                    slack_started;
 #endif
 };
 
Signature · 1 line
2.22.0.rc1.257.g3120a18244-goog

I think this looks good. I like not delaying that further even if it does not fix Dave's use case.

Quoted 5 lines

It does make it glaring that I should have used false/true for setting distribute_running though :)

Thanks!

Should this patch have a Fixes: tag?

maintainer
#5 lore raw
In reply to [email protected]

kernel/sched/sched.h struct cfs_bandwidth { · line 361

Quoted 11 lines · On Thu, Jun 06, 2019 at 10:21:01AM -0700, [email protected] wrote:
kernel/sched/sched.h +1 -0
index efa686eeff26..60219acda94b 100644
@@ -356,6 +356,7 @@ struct cfs_bandwidth {
 	u64			throttled_time;
 
 	bool                    distribute_running;
+	bool                    slack_started;
 #endif
 };

I'm thinking we can this instead? afaict both idle and period_active are already effecitively booleans and don't need the full 16 bits.

kernel/sched/sched.h +4 -5
@@ -338,8 +338,10 @@ struct cfs_bandwidth {
 	u64			runtime_expires;
 	int			expires_seq;
 
-	short			idle;
-	short			period_active;
+	u8			idle;
+	u8			period_active;
+	u8			distribute_running;
+	u8			slack_started;
 	struct hrtimer		period_timer;
 	struct hrtimer		slack_timer;
 	struct list_head	throttled_cfs_rq;
@@ -348,9 +350,6 @@ struct cfs_bandwidth {
 	int			nr_periods;
 	int			nr_throttled;
 	u64			throttled_time;
-
-	bool                    distribute_running;
-	bool                    slack_started;
 #endif
 };
 
#6 lore raw
Quoted 32 lines · On Tue, Jun 11, 2019 at 03:53:25PM +0200 Peter Zijlstra wrote:

kernel/sched/sched.h struct cfs_bandwidth { · line 361

Quoted 11 lines · On Thu, Jun 06, 2019 at 10:21:01AM -0700, [email protected] wrote:
kernel/sched/sched.h +1 -0
index efa686eeff26..60219acda94b 100644
@@ -356,6 +356,7 @@ struct cfs_bandwidth {
 	u64			throttled_time;
 
 	bool                    distribute_running;
+	bool                    slack_started;
 #endif
 };

I'm thinking we can this instead? afaict both idle and period_active are already effecitively booleans and don't need the full 16 bits.

kernel/sched/sched.h +4 -2
@@ -338,8 +338,10 @@ struct cfs_bandwidth {
 	u64			runtime_expires;
 	int			expires_seq;
 
-	short			idle;
-	short			period_active;
+	u8			idle;
+	u8			period_active;
+	u8			distribute_running;
+	u8			slack_started;
 	struct hrtimer		period_timer;
 	struct hrtimer		slack_timer;
 	struct list_head	throttled_cfs_rq;

kernel/sched/sched.h struct cfs_bandwidth { · line 355

Quoted 9 lines
kernel/sched/sched.h +0 -3
@@ -348,9 +350,6 @@ struct cfs_bandwidth {
 	int			nr_periods;
 	int			nr_throttled;
 	u64			throttled_time;
-
-	bool                    distribute_running;
-	bool                    slack_started;
 #endif
 };
 

That looks reasonable to me.

Out of curiosity, why not bool? Is sizeof bool architecture dependent?

maintainer
#7 lore raw
Quoted 3 lines · Phil Auld

That looks reasonable to me.

Out of curiosity, why not bool? Is sizeof bool architecture dependent?

Yeah, sizeof(_Bool) is unspecified and depends on ABI. It is mostly 1, but there are known cases where it is 4.

#8 lore raw
Quoted 8 lines · On Tue, Jun 11, 2019 at 04:24:43PM +0200 Peter Zijlstra wrote:
Quoted 3 lines · Phil Auld

That looks reasonable to me.

Out of curiosity, why not bool? Is sizeof bool architecture dependent?

Yeah, sizeof(_Bool) is unspecified and depends on ABI. It is mostly 1, but there are known cases where it is 4.

Makes sense. Thanks!

#9 lore raw
In reply to Peter Zijlstra
Quoted 32 lines · Peter Zijlstra

kernel/sched/sched.h struct cfs_bandwidth { · line 361

Quoted 11 lines · On Thu, Jun 06, 2019 at 10:21:01AM -0700, [email protected] wrote:
kernel/sched/sched.h +1 -0
index efa686eeff26..60219acda94b 100644
@@ -356,6 +356,7 @@ struct cfs_bandwidth {
 	u64			throttled_time;
 
 	bool                    distribute_running;
+	bool                    slack_started;
 #endif
 };

I'm thinking we can this instead? afaict both idle and period_active are already effecitively booleans and don't need the full 16 bits.

kernel/sched/sched.h +4 -2
@@ -338,8 +338,10 @@ struct cfs_bandwidth {
 	u64			runtime_expires;
 	int			expires_seq;
 
-	short			idle;
-	short			period_active;
+	u8			idle;
+	u8			period_active;
+	u8			distribute_running;
+	u8			slack_started;
 	struct hrtimer		period_timer;
 	struct hrtimer		slack_timer;
 	struct list_head	throttled_cfs_rq;

kernel/sched/sched.h struct cfs_bandwidth { · line 355

Quoted 9 lines
kernel/sched/sched.h +0 -3
@@ -348,9 +350,6 @@ struct cfs_bandwidth {
 	int			nr_periods;
 	int			nr_throttled;
 	u64			throttled_time;
-
-	bool                    distribute_running;
-	bool                    slack_started;
 #endif
 };
 

Yeah, that makes sense to me, should I spin up another version of the patch doing this too or do you have it from here?

#10 lore raw

[tip:sched/core] sched/fair: Don't push cfs_bandwith slack timers forward

In reply to [email protected]
sched/fair: Don't push cfs_bandwith slack timers forward
Commit-ID:  66567fcbaecac455caa1b13643155d686b51ce63
Gitweb:     https://git.kernel.org/tip/66567fcbaecac455caa1b13643155d686b51ce63
Author:     [email protected] <[email protected]>
AuthorDate: Thu, 6 Jun 2019 10:21:01 -0700
Committer:  Ingo Molnar <[email protected]>
CommitDate: Mon, 17 Jun 2019 12:16:01 +0200

sched/fair: Don't push cfs_bandwith slack timers forward

When a cfs_rq sleeps and returns its quota, we delay for 5ms before waking any throttled cfs_rqs to coalesce with other cfs_rqs going to sleep, as this has to be done outside of the rq lock we hold.

The current code waits for 5ms without any sleeps, instead of waiting for 5ms from the first sleep, which can delay the unthrottle more than we want. Switch this around so that we can't push this forward forever.

This requires an extra flag rather than using hrtimer_active, since we need to start a new timer if the current one is in the process of finishing.

---
 kernel/sched/fair.c  | 7 +++++++
 kernel/sched/sched.h | 8 ++++----
 2 files changed, 11 insertions(+), 4 deletions(-)
kernel/sched/fair.c +7 -0
index 4c8f45ed093c..3c11dcdedcbc 100644
@@ -4729,6 +4729,11 @@ static void start_cfs_slack_bandwidth(struct cfs_bandwidth *cfs_b)
 	if (runtime_refresh_within(cfs_b, min_left))
 		return;
 
+	/* don't push forwards an existing deferred unthrottle */
+	if (cfs_b->slack_started)
+		return;
+	cfs_b->slack_started = true;
+
 	hrtimer_start(&cfs_b->slack_timer,
 			ns_to_ktime(cfs_bandwidth_slack_period),
 			HRTIMER_MODE_REL);
@@ -4782,6 +4787,7 @@ static void do_sched_cfs_slack_timer(struct cfs_bandwidth *cfs_b)
 
 	/* confirm we're still not at a refresh boundary */
 	raw_spin_lock_irqsave(&cfs_b->lock, flags);
+	cfs_b->slack_started = false;
 	if (cfs_b->distribute_running) {
 		raw_spin_unlock_irqrestore(&cfs_b->lock, flags);
 		return;
@@ -4945,6 +4951,7 @@ void init_cfs_bandwidth(struct cfs_bandwidth *cfs_b)
 	hrtimer_init(&cfs_b->slack_timer, CLOCK_MONOTONIC, HRTIMER_MODE_REL);
 	cfs_b->slack_timer.function = sched_cfs_slack_timer;
 	cfs_b->distribute_running = 0;
+	cfs_b->slack_started = false;
 }
 
 static void init_cfs_rq_runtime(struct cfs_rq *cfs_rq)
kernel/sched/sched.h +4 -4
index 607859a18b2a..b08dee29ef5e 100644
@@ -338,8 +338,10 @@ struct cfs_bandwidth {
 	u64			runtime_expires;
 	int			expires_seq;
 
-	short			idle;
-	short			period_active;
+	u8			idle;
+	u8			period_active;
+	u8			distribute_running;
+	u8			slack_started;
 	struct hrtimer		period_timer;
 	struct hrtimer		slack_timer;
 	struct list_head	throttled_cfs_rq;
@@ -348,8 +350,6 @@ struct cfs_bandwidth {
 	int			nr_periods;
 	int			nr_throttled;
 	u64			throttled_time;
-
-	bool                    distribute_running;
 #endif
 };