Skip to contentSkip to conversation

sched/fair: Disable runtime_enabled on dying rq

PATCH Superseded Kirill Tkhai 4 participants 14 messages Jun 24, 2014 – Jun 26, 2014 1 message not in the archive

Subsystems (2)

v2 of 3 Superseded Latest is v3 →

lore
Share & tools
lore

A message that is not in the archive (sent off-list or not yet received): 20140624074148.8738.57690.stgit@tkhai

#1 lore raw

PATCH v2 1/3 sched/fair: Disable runtime_enabled on dying rq

We kill rq->rd on the CPU_DOWN_PREPARE stage:

	cpuset_cpu_inactive -> cpuset_update_active_cpus -> partition_sched_domains ->
	-> cpu_attach_domain -> rq_attach_root -> set_rq_offline

This unthrottles all throttled cfs_rqs.

But the cpu is still able to call schedule() till

	take_cpu_down->__cpu_disable()

is called from stop_machine.

This case the tasks from just unthrottled cfs_rqs are pickable in a standard scheduler way, and they are picked by dying cpu. The cfs_rqs becomes throttled again, and migrate_tasks() in migration_call skips their tasks (one more unthrottle in migrate_tasks()->CPU_DYING does not happen, because rq->rd is already NULL).

Patch sets runtime_enabled to zero. This guarantees, the runtime is not accounted, and the cfs_rqs won't exceed given cfs_rq->runtime_remaining = 1, and tasks will be pickable in migrate_tasks(). runtime_enabled is recalculated again when rq becomes online again.

Ben Segall also noticed, we always enable runtime in tg_set_cfs_bandwidth(). Actually, we should do that for online cpus only. To fix that, we check if a cpu is online when its rq is locked. This guarantees we do not have races with set_rq_offline(), which also requires rq->lock.

v2: Fix race with tg_set_cfs_bandwidth().

    Move cfs_rq->runtime_enabled=0 above unthrottle_cfs_rq().
CC Konstantin Khorenko <[email protected]>
CC Ben Segall <[email protected]>
CC Paul Turner <[email protected]>
CC Srikar Dronamraju <[email protected]>
CC Mike Galbraith <[email protected]>
CC Peter Zijlstra <[email protected]>
CC Ingo Molnar <[email protected]>
---
 kernel/sched/core.c |   15 +++++++++++----
 kernel/sched/fair.c |   22 ++++++++++++++++++++++
 2 files changed, 33 insertions(+), 4 deletions(-)
kernel/sched/core.c +11 -4
index 7f3063c..707a3c5 100644
@@ -7842,11 +7842,18 @@ static int tg_set_cfs_bandwidth(struct task_group *tg, u64 period, u64 quota)
 		struct rq *rq = cfs_rq->rq;
 
 		raw_spin_lock_irq(&rq->lock);
-		cfs_rq->runtime_enabled = runtime_enabled;
-		cfs_rq->runtime_remaining = 0;
+		/*
+		 * Do not enable runtime on offline runqueues. We specially
+		 * make it disabled in unthrottle_offline_cfs_rqs().
+		 */
+		if (cpu_online(i)) {
+			cfs_rq->runtime_enabled = runtime_enabled;
+			cfs_rq->runtime_remaining = 0;
+
+			if (cfs_rq->throttled)
+				unthrottle_cfs_rq(cfs_rq);
+		}
 
-		if (cfs_rq->throttled)
-			unthrottle_cfs_rq(cfs_rq);
 		raw_spin_unlock_irq(&rq->lock);
 	}
 	if (runtime_was_enabled && !runtime_enabled)
kernel/sched/fair.c +22 -0
index 1f9c457..5616d23 100644
@@ -3776,6 +3776,19 @@ static void destroy_cfs_bandwidth(struct cfs_bandwidth *cfs_b)
 	hrtimer_cancel(&cfs_b->slack_timer);
 }
 
+static void __maybe_unused update_runtime_enabled(struct rq *rq)
+{
+	struct cfs_rq *cfs_rq;
+
+	for_each_leaf_cfs_rq(rq, cfs_rq) {
+		struct cfs_bandwidth *cfs_b = &cfs_rq->tg->cfs_bandwidth;
+
+		raw_spin_lock(&cfs_b->lock);
+		cfs_rq->runtime_enabled = cfs_b->quota != RUNTIME_INF;
+		raw_spin_unlock(&cfs_b->lock);
+	}
+}
+
 static void __maybe_unused unthrottle_offline_cfs_rqs(struct rq *rq)
 {
 	struct cfs_rq *cfs_rq;
@@ -3789,6 +3802,12 @@ static void __maybe_unused unthrottle_offline_cfs_rqs(struct rq *rq)
 		 * there's some valid quota amount
 		 */
 		cfs_rq->runtime_remaining = 1;
+		/*
+		 * Offline rq is schedulable till cpu is completely disabled
+		 * in take_cpu_down(), so we prevent new cfs throttling here.
+		 */
+		cfs_rq->runtime_enabled = 0;
+
 		if (cfs_rq_throttled(cfs_rq))
 			unthrottle_cfs_rq(cfs_rq);
 	}
@@ -3832,6 +3851,7 @@ static inline struct cfs_bandwidth *tg_cfs_bandwidth(struct task_group *tg)
 	return NULL;
 }
 static inline void destroy_cfs_bandwidth(struct cfs_bandwidth *cfs_b) {}
+static inline void update_runtime_enabled(struct rq *rq) {}
 static inline void unthrottle_offline_cfs_rqs(struct rq *rq) {}
 
 #endif /* CONFIG_CFS_BANDWIDTH */
@@ -7325,6 +7345,8 @@ void trigger_load_balance(struct rq *rq)
 static void rq_online_fair(struct rq *rq)
 {
 	update_sysctl();
+
+	update_runtime_enabled(rq);
 }
 
 static void rq_offline_fair(struct rq *rq)
#2 lore raw
Quoted 59 lines · Kirill Tkhai

We kill rq->rd on the CPU_DOWN_PREPARE stage:

	cpuset_cpu_inactive -> cpuset_update_active_cpus -> partition_sched_domains ->
	-> cpu_attach_domain -> rq_attach_root -> set_rq_offline

This unthrottles all throttled cfs_rqs.

But the cpu is still able to call schedule() till

	take_cpu_down->__cpu_disable()

is called from stop_machine.

This case the tasks from just unthrottled cfs_rqs are pickable in a standard scheduler way, and they are picked by dying cpu. The cfs_rqs becomes throttled again, and migrate_tasks() in migration_call skips their tasks (one more unthrottle in migrate_tasks()->CPU_DYING does not happen, because rq->rd is already NULL).

Patch sets runtime_enabled to zero. This guarantees, the runtime is not accounted, and the cfs_rqs won't exceed given cfs_rq->runtime_remaining = 1, and tasks will be pickable in migrate_tasks(). runtime_enabled is recalculated again when rq becomes online again.

Ben Segall also noticed, we always enable runtime in tg_set_cfs_bandwidth(). Actually, we should do that for online cpus only. To fix that, we check if a cpu is online when its rq is locked. This guarantees we do not have races with set_rq_offline(), which also requires rq->lock.

v2: Fix race with tg_set_cfs_bandwidth().

    Move cfs_rq->runtime_enabled=0 above unthrottle_cfs_rq().
CC Konstantin Khorenko <[email protected]>
CC Ben Segall <[email protected]>
CC Paul Turner <[email protected]>
CC Srikar Dronamraju <[email protected]>
CC Mike Galbraith <[email protected]>
CC Peter Zijlstra <[email protected]>
CC Ingo Molnar <[email protected]>
---
 kernel/sched/core.c |   15 +++++++++++----
 kernel/sched/fair.c |   22 ++++++++++++++++++++++
 2 files changed, 33 insertions(+), 4 deletions(-)
kernel/sched/core.c +1 -2
index 7f3063c..707a3c5 100644
@@ -7842,11 +7842,18 @@ static int tg_set_cfs_bandwidth(struct task_group *tg, u64 period, u64 quota)
 		struct rq *rq = cfs_rq->rq;
 
 		raw_spin_lock_irq(&rq->lock);
-		cfs_rq->runtime_enabled = runtime_enabled;
-		cfs_rq->runtime_remaining = 0;
+		/*

kernel/sched/core.c static int tg_set_cfs_bandwidth(struct task_group *tg, u64 period, u64 quota) · line 7855

Quoted 10 lines
kernel/sched/core.c +10 -0
+		 * Do not enable runtime on offline runqueues. We specially
+		 * make it disabled in unthrottle_offline_cfs_rqs().
+		 */
+		if (cpu_online(i)) {
+			cfs_rq->runtime_enabled = runtime_enabled;
+			cfs_rq->runtime_remaining = 0;
+
+			if (cfs_rq->throttled)
+				unthrottle_cfs_rq(cfs_rq);
+		}

We can just do for_each_online_cpu, yes? Also we probably need get_online_cpus/put_online_cpus, and/or want cpu_active_mask instead right?

#3 lore raw
Quoted 67 lines · On 24.06.2014 21:03, [email protected] wrote:
Quoted 65 lines · Kirill Tkhai

We kill rq->rd on the CPU_DOWN_PREPARE stage:

	cpuset_cpu_inactive -> cpuset_update_active_cpus -> partition_sched_domains ->
	-> cpu_attach_domain -> rq_attach_root -> set_rq_offline

This unthrottles all throttled cfs_rqs.

But the cpu is still able to call schedule() till

	take_cpu_down->__cpu_disable()

is called from stop_machine.

This case the tasks from just unthrottled cfs_rqs are pickable in a standard scheduler way, and they are picked by dying cpu. The cfs_rqs becomes throttled again, and migrate_tasks() in migration_call skips their tasks (one more unthrottle in migrate_tasks()->CPU_DYING does not happen, because rq->rd is already NULL).

Patch sets runtime_enabled to zero. This guarantees, the runtime is not accounted, and the cfs_rqs won't exceed given cfs_rq->runtime_remaining = 1, and tasks will be pickable in migrate_tasks(). runtime_enabled is recalculated again when rq becomes online again.

Ben Segall also noticed, we always enable runtime in tg_set_cfs_bandwidth(). Actually, we should do that for online cpus only. To fix that, we check if a cpu is online when its rq is locked. This guarantees we do not have races with set_rq_offline(), which also requires rq->lock.

v2: Fix race with tg_set_cfs_bandwidth().

    Move cfs_rq->runtime_enabled=0 above unthrottle_cfs_rq().
CC Konstantin Khorenko <[email protected]>
CC Ben Segall <[email protected]>
CC Paul Turner <[email protected]>
CC Srikar Dronamraju <[email protected]>
CC Mike Galbraith <[email protected]>
CC Peter Zijlstra <[email protected]>
CC Ingo Molnar <[email protected]>
---
 kernel/sched/core.c |   15 +++++++++++----
 kernel/sched/fair.c |   22 ++++++++++++++++++++++
 2 files changed, 33 insertions(+), 4 deletions(-)
kernel/sched/core.c +7 -2
index 7f3063c..707a3c5 100644
@@ -7842,11 +7842,18 @@ static int tg_set_cfs_bandwidth(struct task_group *tg, u64 period, u64 quota)
 		struct rq *rq = cfs_rq->rq;
 
 		raw_spin_lock_irq(&rq->lock);
-		cfs_rq->runtime_enabled = runtime_enabled;
-		cfs_rq->runtime_remaining = 0;
+		/*
+		 * Do not enable runtime on offline runqueues. We specially
+		 * make it disabled in unthrottle_offline_cfs_rqs().
+		 */
+		if (cpu_online(i)) {
+			cfs_rq->runtime_enabled = runtime_enabled;
+			cfs_rq->runtime_remaining = 0;
Quoted 9 lines

kernel/sched/core.c static int tg_set_cfs_bandwidth(struct task_group *tg, u64 period, u64 quota) · line 7855

Quoted 4 lines
kernel/sched/core.c +4 -0
+
+			if (cfs_rq->throttled)
+				unthrottle_cfs_rq(cfs_rq);
+		}

We can just do for_each_online_cpu, yes? Also we probably need get_online_cpus/put_online_cpus, and/or want cpu_active_mask instead right?

Yes, we can use for_each_online_cpu/for_each_active_cpu with get_online_cpus() taken. But it adds one more lock dependence. This looks worse for me.

#4 lore raw
Quoted 78 lines · Kirill Tkhai
Quoted 67 lines · On 24.06.2014 21:03, [email protected] wrote:
Quoted 65 lines · Kirill Tkhai

We kill rq->rd on the CPU_DOWN_PREPARE stage:

	cpuset_cpu_inactive -> cpuset_update_active_cpus -> partition_sched_domains ->
	-> cpu_attach_domain -> rq_attach_root -> set_rq_offline

This unthrottles all throttled cfs_rqs.

But the cpu is still able to call schedule() till

	take_cpu_down->__cpu_disable()

is called from stop_machine.

This case the tasks from just unthrottled cfs_rqs are pickable in a standard scheduler way, and they are picked by dying cpu. The cfs_rqs becomes throttled again, and migrate_tasks() in migration_call skips their tasks (one more unthrottle in migrate_tasks()->CPU_DYING does not happen, because rq->rd is already NULL).

Patch sets runtime_enabled to zero. This guarantees, the runtime is not accounted, and the cfs_rqs won't exceed given cfs_rq->runtime_remaining = 1, and tasks will be pickable in migrate_tasks(). runtime_enabled is recalculated again when rq becomes online again.

Ben Segall also noticed, we always enable runtime in tg_set_cfs_bandwidth(). Actually, we should do that for online cpus only. To fix that, we check if a cpu is online when its rq is locked. This guarantees we do not have races with set_rq_offline(), which also requires rq->lock.

v2: Fix race with tg_set_cfs_bandwidth().

    Move cfs_rq->runtime_enabled=0 above unthrottle_cfs_rq().
CC Konstantin Khorenko <[email protected]>
CC Ben Segall <[email protected]>
CC Paul Turner <[email protected]>
CC Srikar Dronamraju <[email protected]>
CC Mike Galbraith <[email protected]>
CC Peter Zijlstra <[email protected]>
CC Ingo Molnar <[email protected]>
---
 kernel/sched/core.c |   15 +++++++++++----
 kernel/sched/fair.c |   22 ++++++++++++++++++++++
 2 files changed, 33 insertions(+), 4 deletions(-)
kernel/sched/core.c +7 -2
index 7f3063c..707a3c5 100644
@@ -7842,11 +7842,18 @@ static int tg_set_cfs_bandwidth(struct task_group *tg, u64 period, u64 quota)
 		struct rq *rq = cfs_rq->rq;
 
 		raw_spin_lock_irq(&rq->lock);
-		cfs_rq->runtime_enabled = runtime_enabled;
-		cfs_rq->runtime_remaining = 0;
+		/*
+		 * Do not enable runtime on offline runqueues. We specially
+		 * make it disabled in unthrottle_offline_cfs_rqs().
+		 */
+		if (cpu_online(i)) {
+			cfs_rq->runtime_enabled = runtime_enabled;
+			cfs_rq->runtime_remaining = 0;
Quoted 9 lines

kernel/sched/core.c static int tg_set_cfs_bandwidth(struct task_group *tg, u64 period, u64 quota) · line 7855

Quoted 4 lines
kernel/sched/core.c +4 -0
+
+			if (cfs_rq->throttled)
+				unthrottle_cfs_rq(cfs_rq);
+		}

We can just do for_each_online_cpu, yes? Also we probably need get_online_cpus/put_online_cpus, and/or want cpu_active_mask instead right?

Quoted 3 lines

Yes, we can use for_each_online_cpu/for_each_active_cpu with get_online_cpus() taken. But it adds one more lock dependence. This looks worse for me.

I mean, you need get_online_cpus anyway - cpu_online is just a test against the same mask that for_each_online_cpu uses, and without taking the lock you can still race with offlining and reset runtime_enabled.

#5 lore raw
Quoted 80 lines · On 24.06.2014 23:13, [email protected] wrote:
Quoted 78 lines · Kirill Tkhai
Quoted 76 lines · On 24.06.2014 21:03, [email protected] wrote:
Quoted 59 lines · Kirill Tkhai

We kill rq->rd on the CPU_DOWN_PREPARE stage:

	cpuset_cpu_inactive -> cpuset_update_active_cpus -> partition_sched_domains ->
	-> cpu_attach_domain -> rq_attach_root -> set_rq_offline

This unthrottles all throttled cfs_rqs.

But the cpu is still able to call schedule() till

	take_cpu_down->__cpu_disable()

is called from stop_machine.

This case the tasks from just unthrottled cfs_rqs are pickable in a standard scheduler way, and they are picked by dying cpu. The cfs_rqs becomes throttled again, and migrate_tasks() in migration_call skips their tasks (one more unthrottle in migrate_tasks()->CPU_DYING does not happen, because rq->rd is already NULL).

Patch sets runtime_enabled to zero. This guarantees, the runtime is not accounted, and the cfs_rqs won't exceed given cfs_rq->runtime_remaining = 1, and tasks will be pickable in migrate_tasks(). runtime_enabled is recalculated again when rq becomes online again.

Ben Segall also noticed, we always enable runtime in tg_set_cfs_bandwidth(). Actually, we should do that for online cpus only. To fix that, we check if a cpu is online when its rq is locked. This guarantees we do not have races with set_rq_offline(), which also requires rq->lock.

v2: Fix race with tg_set_cfs_bandwidth().

    Move cfs_rq->runtime_enabled=0 above unthrottle_cfs_rq().
CC Konstantin Khorenko <[email protected]>
CC Ben Segall <[email protected]>
CC Paul Turner <[email protected]>
CC Srikar Dronamraju <[email protected]>
CC Mike Galbraith <[email protected]>
CC Peter Zijlstra <[email protected]>
CC Ingo Molnar <[email protected]>
---
 kernel/sched/core.c |   15 +++++++++++----
 kernel/sched/fair.c |   22 ++++++++++++++++++++++
 2 files changed, 33 insertions(+), 4 deletions(-)
kernel/sched/core.c +1 -2
index 7f3063c..707a3c5 100644
@@ -7842,11 +7842,18 @@ static int tg_set_cfs_bandwidth(struct task_group *tg, u64 period, u64 quota)
 		struct rq *rq = cfs_rq->rq;
 
 		raw_spin_lock_irq(&rq->lock);
-		cfs_rq->runtime_enabled = runtime_enabled;
-		cfs_rq->runtime_remaining = 0;
+		/*

kernel/sched/core.c static int tg_set_cfs_bandwidth(struct task_group *tg, u64 period, u64 quota) · line 7855

Quoted 10 lines
kernel/sched/core.c +10 -0
+		 * Do not enable runtime on offline runqueues. We specially
+		 * make it disabled in unthrottle_offline_cfs_rqs().
+		 */
+		if (cpu_online(i)) {
+			cfs_rq->runtime_enabled = runtime_enabled;
+			cfs_rq->runtime_remaining = 0;
+
+			if (cfs_rq->throttled)
+				unthrottle_cfs_rq(cfs_rq);
+		}

We can just do for_each_online_cpu, yes? Also we probably need get_online_cpus/put_online_cpus, and/or want cpu_active_mask instead right?

Quoted 8 lines
Quoted 3 lines

Yes, we can use for_each_online_cpu/for_each_active_cpu with get_online_cpus() taken. But it adds one more lock dependence. This looks worse for me.

I mean, you need get_online_cpus anyway - cpu_online is just a test against the same mask that for_each_online_cpu uses, and without taking the lock you can still race with offlining and reset runtime_enabled.

Oh, I see. Thanks.

#6 lore raw

В Вт, 24/06/2014 в 23:26 +0400, Kirill Tkhai пишет:

Quoted 85 lines · On 24.06.2014 23:13, [email protected] wrote:
Quoted 84 lines
Quoted 81 lines · Kirill Tkhai
Quoted 67 lines · On 24.06.2014 21:03, [email protected] wrote:
Quoted 65 lines · Kirill Tkhai

We kill rq->rd on the CPU_DOWN_PREPARE stage:

	cpuset_cpu_inactive -> cpuset_update_active_cpus -> partition_sched_domains ->
	-> cpu_attach_domain -> rq_attach_root -> set_rq_offline

This unthrottles all throttled cfs_rqs.

But the cpu is still able to call schedule() till

	take_cpu_down->__cpu_disable()

is called from stop_machine.

This case the tasks from just unthrottled cfs_rqs are pickable in a standard scheduler way, and they are picked by dying cpu. The cfs_rqs becomes throttled again, and migrate_tasks() in migration_call skips their tasks (one more unthrottle in migrate_tasks()->CPU_DYING does not happen, because rq->rd is already NULL).

Patch sets runtime_enabled to zero. This guarantees, the runtime is not accounted, and the cfs_rqs won't exceed given cfs_rq->runtime_remaining = 1, and tasks will be pickable in migrate_tasks(). runtime_enabled is recalculated again when rq becomes online again.

Ben Segall also noticed, we always enable runtime in tg_set_cfs_bandwidth(). Actually, we should do that for online cpus only. To fix that, we check if a cpu is online when its rq is locked. This guarantees we do not have races with set_rq_offline(), which also requires rq->lock.

v2: Fix race with tg_set_cfs_bandwidth().

    Move cfs_rq->runtime_enabled=0 above unthrottle_cfs_rq().
CC Konstantin Khorenko <[email protected]>
CC Ben Segall <[email protected]>
CC Paul Turner <[email protected]>
CC Srikar Dronamraju <[email protected]>
CC Mike Galbraith <[email protected]>
CC Peter Zijlstra <[email protected]>
CC Ingo Molnar <[email protected]>
---
 kernel/sched/core.c |   15 +++++++++++----
 kernel/sched/fair.c |   22 ++++++++++++++++++++++
 2 files changed, 33 insertions(+), 4 deletions(-)
kernel/sched/core.c +7 -2
index 7f3063c..707a3c5 100644
@@ -7842,11 +7842,18 @@ static int tg_set_cfs_bandwidth(struct task_group *tg, u64 period, u64 quota)
 		struct rq *rq = cfs_rq->rq;
 
 		raw_spin_lock_irq(&rq->lock);
-		cfs_rq->runtime_enabled = runtime_enabled;
-		cfs_rq->runtime_remaining = 0;
+		/*
+		 * Do not enable runtime on offline runqueues. We specially
+		 * make it disabled in unthrottle_offline_cfs_rqs().
+		 */
+		if (cpu_online(i)) {
+			cfs_rq->runtime_enabled = runtime_enabled;
+			cfs_rq->runtime_remaining = 0;
Quoted 9 lines

kernel/sched/core.c static int tg_set_cfs_bandwidth(struct task_group *tg, u64 period, u64 quota) · line 7855

Quoted 4 lines
kernel/sched/core.c +4 -0
+
+			if (cfs_rq->throttled)
+				unthrottle_cfs_rq(cfs_rq);
+		}

We can just do for_each_online_cpu, yes? Also we probably need get_online_cpus/put_online_cpus, and/or want cpu_active_mask instead right?

Yes, we can use for_each_online_cpu/for_each_active_cpu with get_online_cpus() taken. But it adds one more lock dependence. This looks worse for me.

Quoted 6 lines
Quoted 4 lines

I mean, you need get_online_cpus anyway - cpu_online is just a test against the same mask that for_each_online_cpu uses, and without taking the lock you can still race with offlining and reset runtime_enabled.

Oh, I see. Thanks.

But we can check for rq->online, don't we? How about this?

sched/fair: Disable runtime_enabled on dying rq

We kill rq->rd on the CPU_DOWN_PREPARE stage:

cpuset_cpu_inactive -> cpuset_update_active_cpus -> partition_sched_domains -> -> cpu_attach_domain -> rq_attach_root -> set_rq_offline

This unthrottles all throttled cfs_rqs.

But the cpu is still able to call schedule() till

take_cpu_down->__cpu_disable()

is called from stop_machine.

This case the tasks from just unthrottled cfs_rqs are pickable in a standard scheduler way, and they are picked by dying cpu. The cfs_rqs becomes throttled again, and migrate_tasks() in migration_call skips their tasks (one more unthrottle in migrate_tasks()->CPU_DYING does not happen, because rq->rd is already NULL).

Patch sets runtime_enabled to zero. This guarantees, the runtime is not accounted, and the cfs_rqs won't exceed given cfs_rq->runtime_remaining = 1, and tasks will be pickable in migrate_tasks(). runtime_enabled is recalculated again when rq becomes online again.

Ben Segall also noticed, we always enable runtime in tg_set_cfs_bandwidth(). Actually, we should do that for online cpus only. To fix that, we check if a cpu is online when its rq is locked. This guarantees we do not have races with set_rq_offline(), which also requires rq->lock.

v2: Fix race with tg_set_cfs_bandwidth().
Move cfs_rq->runtime_enabled=0 above unthrottle_cfs_rq().
v3: Check for rq->online instead of cpu_active.

kernel/sched/core.c +10 -4
index ceea8d0..4c163c9 100644
@@ -7837,11 +7837,17 @@ static int tg_set_cfs_bandwidth(struct task_group *tg, u64 period, u64 quota)
 		struct rq *rq = cfs_rq->rq;
 
 		raw_spin_lock_irq(&rq->lock);
-		cfs_rq->runtime_enabled = runtime_enabled;
-		cfs_rq->runtime_remaining = 0;
+		/*
+		 * Do not enable runtime on offline runqueues. We specially
+		 * disable it in unthrottle_offline_cfs_rqs().
+		 */
+		if (rq->online) {
+			cfs_rq->runtime_enabled = runtime_enabled;
+			cfs_rq->runtime_remaining = 0;
 
-		if (cfs_rq->throttled)
-			unthrottle_cfs_rq(cfs_rq);
+			if (cfs_rq->throttled)
+				unthrottle_cfs_rq(cfs_rq);
+		}
 		raw_spin_unlock_irq(&rq->lock);
 	}
 	if (runtime_was_enabled && !runtime_enabled)
kernel/sched/fair.c +22 -0
index 1f9c457..5616d23 100644
@@ -3776,6 +3776,19 @@ static void destroy_cfs_bandwidth(struct cfs_bandwidth *cfs_b)
 	hrtimer_cancel(&cfs_b->slack_timer);
 }
 
+static void __maybe_unused update_runtime_enabled(struct rq *rq)
+{
+	struct cfs_rq *cfs_rq;
+
+	for_each_leaf_cfs_rq(rq, cfs_rq) {
+		struct cfs_bandwidth *cfs_b = &cfs_rq->tg->cfs_bandwidth;
+
+		raw_spin_lock(&cfs_b->lock);
+		cfs_rq->runtime_enabled = cfs_b->quota != RUNTIME_INF;
+		raw_spin_unlock(&cfs_b->lock);
+	}
+}
+
 static void __maybe_unused unthrottle_offline_cfs_rqs(struct rq *rq)
 {
 	struct cfs_rq *cfs_rq;
@@ -3789,6 +3802,12 @@ static void __maybe_unused unthrottle_offline_cfs_rqs(struct rq *rq)
 		 * there's some valid quota amount
 		 */
 		cfs_rq->runtime_remaining = 1;
+		/*
+		 * Offline rq is schedulable till cpu is completely disabled
+		 * in take_cpu_down(), so we prevent new cfs throttling here.
+		 */
+		cfs_rq->runtime_enabled = 0;
+
 		if (cfs_rq_throttled(cfs_rq))
 			unthrottle_cfs_rq(cfs_rq);
 	}
@@ -3832,6 +3851,7 @@ static inline struct cfs_bandwidth *tg_cfs_bandwidth(struct task_group *tg)
 	return NULL;
 }
 static inline void destroy_cfs_bandwidth(struct cfs_bandwidth *cfs_b) {}
+static inline void update_runtime_enabled(struct rq *rq) {}
 static inline void unthrottle_offline_cfs_rqs(struct rq *rq) {}
 
 #endif /* CONFIG_CFS_BANDWIDTH */
@@ -7325,6 +7345,8 @@ void trigger_load_balance(struct rq *rq)
 static void rq_online_fair(struct rq *rq)
 {
 	update_sysctl();
+
+	update_runtime_enabled(rq);
 }
 
 static void rq_offline_fair(struct rq *rq)
#7 lore raw

В Ср, 25/06/2014 в 11:53 +0400, Kirill Tkhai пишет:

Quoted 86 lines

В Вт, 24/06/2014 в 23:26 +0400, Kirill Tkhai пишет:

Quoted 85 lines · On 24.06.2014 23:13, [email protected] wrote:
Quoted 84 lines
Quoted 81 lines · Kirill Tkhai
Quoted 67 lines · On 24.06.2014 21:03, [email protected] wrote:
Quoted 65 lines · Kirill Tkhai

We kill rq->rd on the CPU_DOWN_PREPARE stage:

	cpuset_cpu_inactive -> cpuset_update_active_cpus -> partition_sched_domains ->
	-> cpu_attach_domain -> rq_attach_root -> set_rq_offline

This unthrottles all throttled cfs_rqs.

But the cpu is still able to call schedule() till

	take_cpu_down->__cpu_disable()

is called from stop_machine.

This case the tasks from just unthrottled cfs_rqs are pickable in a standard scheduler way, and they are picked by dying cpu. The cfs_rqs becomes throttled again, and migrate_tasks() in migration_call skips their tasks (one more unthrottle in migrate_tasks()->CPU_DYING does not happen, because rq->rd is already NULL).

Patch sets runtime_enabled to zero. This guarantees, the runtime is not accounted, and the cfs_rqs won't exceed given cfs_rq->runtime_remaining = 1, and tasks will be pickable in migrate_tasks(). runtime_enabled is recalculated again when rq becomes online again.

Ben Segall also noticed, we always enable runtime in tg_set_cfs_bandwidth(). Actually, we should do that for online cpus only. To fix that, we check if a cpu is online when its rq is locked. This guarantees we do not have races with set_rq_offline(), which also requires rq->lock.

v2: Fix race with tg_set_cfs_bandwidth().

    Move cfs_rq->runtime_enabled=0 above unthrottle_cfs_rq().
CC Konstantin Khorenko <[email protected]>
CC Ben Segall <[email protected]>
CC Paul Turner <[email protected]>
CC Srikar Dronamraju <[email protected]>
CC Mike Galbraith <[email protected]>
CC Peter Zijlstra <[email protected]>
CC Ingo Molnar <[email protected]>
---
 kernel/sched/core.c |   15 +++++++++++----
 kernel/sched/fair.c |   22 ++++++++++++++++++++++
 2 files changed, 33 insertions(+), 4 deletions(-)
kernel/sched/core.c +7 -2
index 7f3063c..707a3c5 100644
@@ -7842,11 +7842,18 @@ static int tg_set_cfs_bandwidth(struct task_group *tg, u64 period, u64 quota)
 		struct rq *rq = cfs_rq->rq;
 
 		raw_spin_lock_irq(&rq->lock);
-		cfs_rq->runtime_enabled = runtime_enabled;
-		cfs_rq->runtime_remaining = 0;
+		/*
+		 * Do not enable runtime on offline runqueues. We specially
+		 * make it disabled in unthrottle_offline_cfs_rqs().
+		 */
+		if (cpu_online(i)) {
+			cfs_rq->runtime_enabled = runtime_enabled;
+			cfs_rq->runtime_remaining = 0;
Quoted 9 lines

kernel/sched/core.c static int tg_set_cfs_bandwidth(struct task_group *tg, u64 period, u64 quota) · line 7850

Quoted 4 lines
kernel/sched/core.c +4 -0
+
+			if (cfs_rq->throttled)
+				unthrottle_cfs_rq(cfs_rq);
+		}

We can just do for_each_online_cpu, yes? Also we probably need get_online_cpus/put_online_cpus, and/or want cpu_active_mask instead right?

Yes, we can use for_each_online_cpu/for_each_active_cpu with get_online_cpus() taken. But it adds one more lock dependence. This looks worse for me.

Quoted 8 lines
Quoted 6 lines
Quoted 4 lines

I mean, you need get_online_cpus anyway - cpu_online is just a test against the same mask that for_each_online_cpu uses, and without taking the lock you can still race with offlining and reset runtime_enabled.

Oh, I see. Thanks.

But we can check for rq->online, don't we? How about this?

Oh, rq->online presents only in SMP case... Ok, lets do with get_online_cpus()...

Quoted 122 lines

sched/fair: Disable runtime_enabled on dying rq

We kill rq->rd on the CPU_DOWN_PREPARE stage:

cpuset_cpu_inactive -> cpuset_update_active_cpus -> partition_sched_domains -> -> cpu_attach_domain -> rq_attach_root -> set_rq_offline

This unthrottles all throttled cfs_rqs.

But the cpu is still able to call schedule() till

take_cpu_down->__cpu_disable()

is called from stop_machine.

This case the tasks from just unthrottled cfs_rqs are pickable in a standard scheduler way, and they are picked by dying cpu. The cfs_rqs becomes throttled again, and migrate_tasks() in migration_call skips their tasks (one more unthrottle in migrate_tasks()->CPU_DYING does not happen, because rq->rd is already NULL).

Patch sets runtime_enabled to zero. This guarantees, the runtime is not accounted, and the cfs_rqs won't exceed given cfs_rq->runtime_remaining = 1, and tasks will be pickable in migrate_tasks(). runtime_enabled is recalculated again when rq becomes online again.

Ben Segall also noticed, we always enable runtime in tg_set_cfs_bandwidth(). Actually, we should do that for online cpus only. To fix that, we check if a cpu is online when its rq is locked. This guarantees we do not have races with set_rq_offline(), which also requires rq->lock.

v2: Fix race with tg_set_cfs_bandwidth().
Move cfs_rq->runtime_enabled=0 above unthrottle_cfs_rq().
v3: Check for rq->online instead of cpu_active.

kernel/sched/core.c +10 -4
index ceea8d0..4c163c9 100644
@@ -7837,11 +7837,17 @@ static int tg_set_cfs_bandwidth(struct task_group *tg, u64 period, u64 quota)
 		struct rq *rq = cfs_rq->rq;
 
 		raw_spin_lock_irq(&rq->lock);
-		cfs_rq->runtime_enabled = runtime_enabled;
-		cfs_rq->runtime_remaining = 0;
+		/*
+		 * Do not enable runtime on offline runqueues. We specially
+		 * disable it in unthrottle_offline_cfs_rqs().
+		 */
+		if (rq->online) {
+			cfs_rq->runtime_enabled = runtime_enabled;
+			cfs_rq->runtime_remaining = 0;
 
-		if (cfs_rq->throttled)
-			unthrottle_cfs_rq(cfs_rq);
+			if (cfs_rq->throttled)
+				unthrottle_cfs_rq(cfs_rq);
+		}
 		raw_spin_unlock_irq(&rq->lock);
 	}
 	if (runtime_was_enabled && !runtime_enabled)
kernel/sched/fair.c +22 -0
index 1f9c457..5616d23 100644
@@ -3776,6 +3776,19 @@ static void destroy_cfs_bandwidth(struct cfs_bandwidth *cfs_b)
 	hrtimer_cancel(&cfs_b->slack_timer);
 }
 
+static void __maybe_unused update_runtime_enabled(struct rq *rq)
+{
+	struct cfs_rq *cfs_rq;
+
+	for_each_leaf_cfs_rq(rq, cfs_rq) {
+		struct cfs_bandwidth *cfs_b = &cfs_rq->tg->cfs_bandwidth;
+
+		raw_spin_lock(&cfs_b->lock);
+		cfs_rq->runtime_enabled = cfs_b->quota != RUNTIME_INF;
+		raw_spin_unlock(&cfs_b->lock);
+	}
+}
+
 static void __maybe_unused unthrottle_offline_cfs_rqs(struct rq *rq)
 {
 	struct cfs_rq *cfs_rq;
@@ -3789,6 +3802,12 @@ static void __maybe_unused unthrottle_offline_cfs_rqs(struct rq *rq)
 		 * there's some valid quota amount
 		 */
 		cfs_rq->runtime_remaining = 1;
+		/*
+		 * Offline rq is schedulable till cpu is completely disabled
+		 * in take_cpu_down(), so we prevent new cfs throttling here.
+		 */
+		cfs_rq->runtime_enabled = 0;
+
 		if (cfs_rq_throttled(cfs_rq))
 			unthrottle_cfs_rq(cfs_rq);
 	}
@@ -3832,6 +3851,7 @@ static inline struct cfs_bandwidth *tg_cfs_bandwidth(struct task_group *tg)
 	return NULL;
 }
 static inline void destroy_cfs_bandwidth(struct cfs_bandwidth *cfs_b) {}
+static inline void update_runtime_enabled(struct rq *rq) {}
 static inline void unthrottle_offline_cfs_rqs(struct rq *rq) {}
 
 #endif /* CONFIG_CFS_BANDWIDTH */
@@ -7325,6 +7345,8 @@ void trigger_load_balance(struct rq *rq)
 static void rq_online_fair(struct rq *rq)
 {
 	update_sysctl();
+
+	update_runtime_enabled(rq);
 }
 
 static void rq_offline_fair(struct rq *rq)
reviewer Reviewed-by
#8 lore raw
In reply to Kirill Tkhai
Quoted 86 lines · Kirill Tkhai

В Вт, 24/06/2014 в 23:26 +0400, Kirill Tkhai пишет:

Quoted 85 lines · On 24.06.2014 23:13, [email protected] wrote:
Quoted 84 lines
Quoted 81 lines · Kirill Tkhai
Quoted 67 lines · On 24.06.2014 21:03, [email protected] wrote:
Quoted 65 lines · Kirill Tkhai

We kill rq->rd on the CPU_DOWN_PREPARE stage:

	cpuset_cpu_inactive -> cpuset_update_active_cpus -> partition_sched_domains ->
	-> cpu_attach_domain -> rq_attach_root -> set_rq_offline

This unthrottles all throttled cfs_rqs.

But the cpu is still able to call schedule() till

	take_cpu_down->__cpu_disable()

is called from stop_machine.

This case the tasks from just unthrottled cfs_rqs are pickable in a standard scheduler way, and they are picked by dying cpu. The cfs_rqs becomes throttled again, and migrate_tasks() in migration_call skips their tasks (one more unthrottle in migrate_tasks()->CPU_DYING does not happen, because rq->rd is already NULL).

Patch sets runtime_enabled to zero. This guarantees, the runtime is not accounted, and the cfs_rqs won't exceed given cfs_rq->runtime_remaining = 1, and tasks will be pickable in migrate_tasks(). runtime_enabled is recalculated again when rq becomes online again.

Ben Segall also noticed, we always enable runtime in tg_set_cfs_bandwidth(). Actually, we should do that for online cpus only. To fix that, we check if a cpu is online when its rq is locked. This guarantees we do not have races with set_rq_offline(), which also requires rq->lock.

v2: Fix race with tg_set_cfs_bandwidth().

    Move cfs_rq->runtime_enabled=0 above unthrottle_cfs_rq().
CC Konstantin Khorenko <[email protected]>
CC Ben Segall <[email protected]>
CC Paul Turner <[email protected]>
CC Srikar Dronamraju <[email protected]>
CC Mike Galbraith <[email protected]>
CC Peter Zijlstra <[email protected]>
CC Ingo Molnar <[email protected]>
---
 kernel/sched/core.c |   15 +++++++++++----
 kernel/sched/fair.c |   22 ++++++++++++++++++++++
 2 files changed, 33 insertions(+), 4 deletions(-)
kernel/sched/core.c +7 -2
index 7f3063c..707a3c5 100644
@@ -7842,11 +7842,18 @@ static int tg_set_cfs_bandwidth(struct task_group *tg, u64 period, u64 quota)
 		struct rq *rq = cfs_rq->rq;
 
 		raw_spin_lock_irq(&rq->lock);
-		cfs_rq->runtime_enabled = runtime_enabled;
-		cfs_rq->runtime_remaining = 0;
+		/*
+		 * Do not enable runtime on offline runqueues. We specially
+		 * make it disabled in unthrottle_offline_cfs_rqs().
+		 */
+		if (cpu_online(i)) {
+			cfs_rq->runtime_enabled = runtime_enabled;
+			cfs_rq->runtime_remaining = 0;
Quoted 9 lines

kernel/sched/core.c static int tg_set_cfs_bandwidth(struct task_group *tg, u64 period, u64 quota) · line 7850

Quoted 4 lines
kernel/sched/core.c +4 -0
+
+			if (cfs_rq->throttled)
+				unthrottle_cfs_rq(cfs_rq);
+		}

We can just do for_each_online_cpu, yes? Also we probably need get_online_cpus/put_online_cpus, and/or want cpu_active_mask instead right?

Yes, we can use for_each_online_cpu/for_each_active_cpu with get_online_cpus() taken. But it adds one more lock dependence. This looks worse for me.

Quoted 8 lines
Quoted 6 lines
Quoted 4 lines

I mean, you need get_online_cpus anyway - cpu_online is just a test against the same mask that for_each_online_cpu uses, and without taking the lock you can still race with offlining and reset runtime_enabled.

Oh, I see. Thanks.

But we can check for rq->online, don't we? How about this?

Yeah, that should work.

Quoted 35 lines

sched/fair: Disable runtime_enabled on dying rq

We kill rq->rd on the CPU_DOWN_PREPARE stage:

cpuset_cpu_inactive -> cpuset_update_active_cpus -> partition_sched_domains -> -> cpu_attach_domain -> rq_attach_root -> set_rq_offline

This unthrottles all throttled cfs_rqs.

But the cpu is still able to call schedule() till

take_cpu_down->__cpu_disable()

is called from stop_machine.

This case the tasks from just unthrottled cfs_rqs are pickable in a standard scheduler way, and they are picked by dying cpu. The cfs_rqs becomes throttled again, and migrate_tasks() in migration_call skips their tasks (one more unthrottle in migrate_tasks()->CPU_DYING does not happen, because rq->rd is already NULL).

Patch sets runtime_enabled to zero. This guarantees, the runtime is not accounted, and the cfs_rqs won't exceed given cfs_rq->runtime_remaining = 1, and tasks will be pickable in migrate_tasks(). runtime_enabled is recalculated again when rq becomes online again.

Ben Segall also noticed, we always enable runtime in tg_set_cfs_bandwidth(). Actually, we should do that for online cpus only. To fix that, we check if a cpu is online when its rq is locked. This guarantees we do not have races with set_rq_offline(), which also requires rq->lock.

Quoted 5 lines

v2: Fix race with tg_set_cfs_bandwidth().
Move cfs_rq->runtime_enabled=0 above unthrottle_cfs_rq().
v3: Check for rq->online instead of cpu_active.

#9 lore raw

В Ср, 25/06/2014 в 09:52 -0700, [email protected] пишет:

Quoted 93 lines
Quoted 91 lines · Kirill Tkhai

В Вт, 24/06/2014 в 23:26 +0400, Kirill Tkhai пишет:

Quoted 90 lines · On 24.06.2014 23:13, [email protected] wrote:
Quoted 88 lines
Quoted 78 lines · Kirill Tkhai
Quoted 67 lines · On 24.06.2014 21:03, [email protected] wrote:
Quoted 65 lines · Kirill Tkhai

We kill rq->rd on the CPU_DOWN_PREPARE stage:

	cpuset_cpu_inactive -> cpuset_update_active_cpus -> partition_sched_domains ->
	-> cpu_attach_domain -> rq_attach_root -> set_rq_offline

This unthrottles all throttled cfs_rqs.

But the cpu is still able to call schedule() till

	take_cpu_down->__cpu_disable()

is called from stop_machine.

This case the tasks from just unthrottled cfs_rqs are pickable in a standard scheduler way, and they are picked by dying cpu. The cfs_rqs becomes throttled again, and migrate_tasks() in migration_call skips their tasks (one more unthrottle in migrate_tasks()->CPU_DYING does not happen, because rq->rd is already NULL).

Patch sets runtime_enabled to zero. This guarantees, the runtime is not accounted, and the cfs_rqs won't exceed given cfs_rq->runtime_remaining = 1, and tasks will be pickable in migrate_tasks(). runtime_enabled is recalculated again when rq becomes online again.

Ben Segall also noticed, we always enable runtime in tg_set_cfs_bandwidth(). Actually, we should do that for online cpus only. To fix that, we check if a cpu is online when its rq is locked. This guarantees we do not have races with set_rq_offline(), which also requires rq->lock.

v2: Fix race with tg_set_cfs_bandwidth().

    Move cfs_rq->runtime_enabled=0 above unthrottle_cfs_rq().
CC Konstantin Khorenko <[email protected]>
CC Ben Segall <[email protected]>
CC Paul Turner <[email protected]>
CC Srikar Dronamraju <[email protected]>
CC Mike Galbraith <[email protected]>
CC Peter Zijlstra <[email protected]>
CC Ingo Molnar <[email protected]>
---
 kernel/sched/core.c |   15 +++++++++++----
 kernel/sched/fair.c |   22 ++++++++++++++++++++++
 2 files changed, 33 insertions(+), 4 deletions(-)
kernel/sched/core.c +7 -2
index 7f3063c..707a3c5 100644
@@ -7842,11 +7842,18 @@ static int tg_set_cfs_bandwidth(struct task_group *tg, u64 period, u64 quota)
 		struct rq *rq = cfs_rq->rq;
 
 		raw_spin_lock_irq(&rq->lock);
-		cfs_rq->runtime_enabled = runtime_enabled;
-		cfs_rq->runtime_remaining = 0;
+		/*
+		 * Do not enable runtime on offline runqueues. We specially
+		 * make it disabled in unthrottle_offline_cfs_rqs().
+		 */
+		if (cpu_online(i)) {
+			cfs_rq->runtime_enabled = runtime_enabled;
+			cfs_rq->runtime_remaining = 0;
Quoted 9 lines

kernel/sched/core.c static int tg_set_cfs_bandwidth(struct task_group *tg, u64 period, u64 quota) · line 7850

Quoted 4 lines
kernel/sched/core.c +4 -0
+
+			if (cfs_rq->throttled)
+				unthrottle_cfs_rq(cfs_rq);
+		}

We can just do for_each_online_cpu, yes? Also we probably need get_online_cpus/put_online_cpus, and/or want cpu_active_mask instead right?

Quoted 3 lines

Yes, we can use for_each_online_cpu/for_each_active_cpu with get_online_cpus() taken. But it adds one more lock dependence. This looks worse for me.

I mean, you need get_online_cpus anyway - cpu_online is just a test against the same mask that for_each_online_cpu uses, and without taking the lock you can still race with offlining and reset runtime_enabled.

Quoted 5 lines
Quoted 3 lines
Quoted 1 line

Oh, I see. Thanks.

But we can check for rq->online, don't we? How about this?

Yeah, that should work.

We can't base on it because rq->offline is not available in !SMP. Could you review the message from [PATCH v3 1/3] topic?

Quoted 42 lines
Quoted 35 lines

sched/fair: Disable runtime_enabled on dying rq

We kill rq->rd on the CPU_DOWN_PREPARE stage:

cpuset_cpu_inactive -> cpuset_update_active_cpus -> partition_sched_domains -> -> cpu_attach_domain -> rq_attach_root -> set_rq_offline

This unthrottles all throttled cfs_rqs.

But the cpu is still able to call schedule() till

take_cpu_down->__cpu_disable()

is called from stop_machine.

This case the tasks from just unthrottled cfs_rqs are pickable in a standard scheduler way, and they are picked by dying cpu. The cfs_rqs becomes throttled again, and migrate_tasks() in migration_call skips their tasks (one more unthrottle in migrate_tasks()->CPU_DYING does not happen, because rq->rd is already NULL).

Patch sets runtime_enabled to zero. This guarantees, the runtime is not accounted, and the cfs_rqs won't exceed given cfs_rq->runtime_remaining = 1, and tasks will be pickable in migrate_tasks(). runtime_enabled is recalculated again when rq becomes online again.

Ben Segall also noticed, we always enable runtime in tg_set_cfs_bandwidth(). Actually, we should do that for online cpus only. To fix that, we check if a cpu is online when its rq is locked. This guarantees we do not have races with set_rq_offline(), which also requires rq->lock.

Quoted 5 lines

v2: Fix race with tg_set_cfs_bandwidth().
Move cfs_rq->runtime_enabled=0 above unthrottle_cfs_rq().
v3: Check for rq->online instead of cpu_active.

#10 lore raw
Quoted 94 lines · Kirill Tkhai

В Ср, 25/06/2014 в 09:52 -0700, [email protected] пишет:

Quoted 93 lines
Quoted 91 lines · Kirill Tkhai

В Вт, 24/06/2014 в 23:26 +0400, Kirill Tkhai пишет:

Quoted 90 lines · On 24.06.2014 23:13, [email protected] wrote:
Quoted 88 lines
Quoted 78 lines · Kirill Tkhai
Quoted 67 lines · On 24.06.2014 21:03, [email protected] wrote:
Quoted 65 lines · Kirill Tkhai

We kill rq->rd on the CPU_DOWN_PREPARE stage:

	cpuset_cpu_inactive -> cpuset_update_active_cpus -> partition_sched_domains ->
	-> cpu_attach_domain -> rq_attach_root -> set_rq_offline

This unthrottles all throttled cfs_rqs.

But the cpu is still able to call schedule() till

	take_cpu_down->__cpu_disable()

is called from stop_machine.

This case the tasks from just unthrottled cfs_rqs are pickable in a standard scheduler way, and they are picked by dying cpu. The cfs_rqs becomes throttled again, and migrate_tasks() in migration_call skips their tasks (one more unthrottle in migrate_tasks()->CPU_DYING does not happen, because rq->rd is already NULL).

Patch sets runtime_enabled to zero. This guarantees, the runtime is not accounted, and the cfs_rqs won't exceed given cfs_rq->runtime_remaining = 1, and tasks will be pickable in migrate_tasks(). runtime_enabled is recalculated again when rq becomes online again.

Ben Segall also noticed, we always enable runtime in tg_set_cfs_bandwidth(). Actually, we should do that for online cpus only. To fix that, we check if a cpu is online when its rq is locked. This guarantees we do not have races with set_rq_offline(), which also requires rq->lock.

v2: Fix race with tg_set_cfs_bandwidth().

    Move cfs_rq->runtime_enabled=0 above unthrottle_cfs_rq().
CC Konstantin Khorenko <[email protected]>
CC Ben Segall <[email protected]>
CC Paul Turner <[email protected]>
CC Srikar Dronamraju <[email protected]>
CC Mike Galbraith <[email protected]>
CC Peter Zijlstra <[email protected]>
CC Ingo Molnar <[email protected]>
---
 kernel/sched/core.c |   15 +++++++++++----
 kernel/sched/fair.c |   22 ++++++++++++++++++++++
 2 files changed, 33 insertions(+), 4 deletions(-)
kernel/sched/core.c +7 -2
index 7f3063c..707a3c5 100644
@@ -7842,11 +7842,18 @@ static int tg_set_cfs_bandwidth(struct task_group *tg, u64 period, u64 quota)
 		struct rq *rq = cfs_rq->rq;
 
 		raw_spin_lock_irq(&rq->lock);
-		cfs_rq->runtime_enabled = runtime_enabled;
-		cfs_rq->runtime_remaining = 0;
+		/*
+		 * Do not enable runtime on offline runqueues. We specially
+		 * make it disabled in unthrottle_offline_cfs_rqs().
+		 */
+		if (cpu_online(i)) {
+			cfs_rq->runtime_enabled = runtime_enabled;
+			cfs_rq->runtime_remaining = 0;
Quoted 9 lines

kernel/sched/core.c static int tg_set_cfs_bandwidth(struct task_group *tg, u64 period, u64 quota) · line 7850

Quoted 4 lines
kernel/sched/core.c +4 -0
+
+			if (cfs_rq->throttled)
+				unthrottle_cfs_rq(cfs_rq);
+		}

We can just do for_each_online_cpu, yes? Also we probably need get_online_cpus/put_online_cpus, and/or want cpu_active_mask instead right?

Quoted 3 lines

Yes, we can use for_each_online_cpu/for_each_active_cpu with get_online_cpus() taken. But it adds one more lock dependence. This looks worse for me.

I mean, you need get_online_cpus anyway - cpu_online is just a test against the same mask that for_each_online_cpu uses, and without taking the lock you can still race with offlining and reset runtime_enabled.

Quoted 8 lines
Quoted 5 lines
Quoted 3 lines
Quoted 1 line

Oh, I see. Thanks.

But we can check for rq->online, don't we? How about this?

Yeah, that should work.

We can't base on it because rq->offline is not available in !SMP. Could you review the message from [PATCH v3 1/3] topic?

I'm not sure what you mean here. The patch just checking cpu_online won't work, is there another version you want me to look at?

#11 lore raw

В Ср, 25/06/2014 в 10:40 -0700, [email protected] пишет:

Quoted 100 lines
Quoted 98 lines · Kirill Tkhai

В Ср, 25/06/2014 в 09:52 -0700, [email protected] пишет:

Quoted 97 lines
Quoted 94 lines · Kirill Tkhai

В Вт, 24/06/2014 в 23:26 +0400, Kirill Tkhai пишет:

Quoted 85 lines · On 24.06.2014 23:13, [email protected] wrote:
Quoted 84 lines
Quoted 81 lines · Kirill Tkhai
Quoted 67 lines · On 24.06.2014 21:03, [email protected] wrote:
Quoted 65 lines · Kirill Tkhai

We kill rq->rd on the CPU_DOWN_PREPARE stage:

	cpuset_cpu_inactive -> cpuset_update_active_cpus -> partition_sched_domains ->
	-> cpu_attach_domain -> rq_attach_root -> set_rq_offline

This unthrottles all throttled cfs_rqs.

But the cpu is still able to call schedule() till

	take_cpu_down->__cpu_disable()

is called from stop_machine.

This case the tasks from just unthrottled cfs_rqs are pickable in a standard scheduler way, and they are picked by dying cpu. The cfs_rqs becomes throttled again, and migrate_tasks() in migration_call skips their tasks (one more unthrottle in migrate_tasks()->CPU_DYING does not happen, because rq->rd is already NULL).

Patch sets runtime_enabled to zero. This guarantees, the runtime is not accounted, and the cfs_rqs won't exceed given cfs_rq->runtime_remaining = 1, and tasks will be pickable in migrate_tasks(). runtime_enabled is recalculated again when rq becomes online again.

Ben Segall also noticed, we always enable runtime in tg_set_cfs_bandwidth(). Actually, we should do that for online cpus only. To fix that, we check if a cpu is online when its rq is locked. This guarantees we do not have races with set_rq_offline(), which also requires rq->lock.

v2: Fix race with tg_set_cfs_bandwidth().

    Move cfs_rq->runtime_enabled=0 above unthrottle_cfs_rq().
CC Konstantin Khorenko <[email protected]>
CC Ben Segall <[email protected]>
CC Paul Turner <[email protected]>
CC Srikar Dronamraju <[email protected]>
CC Mike Galbraith <[email protected]>
CC Peter Zijlstra <[email protected]>
CC Ingo Molnar <[email protected]>
---
 kernel/sched/core.c |   15 +++++++++++----
 kernel/sched/fair.c |   22 ++++++++++++++++++++++
 2 files changed, 33 insertions(+), 4 deletions(-)
kernel/sched/core.c +7 -2
index 7f3063c..707a3c5 100644
@@ -7842,11 +7842,18 @@ static int tg_set_cfs_bandwidth(struct task_group *tg, u64 period, u64 quota)
 		struct rq *rq = cfs_rq->rq;
 
 		raw_spin_lock_irq(&rq->lock);
-		cfs_rq->runtime_enabled = runtime_enabled;
-		cfs_rq->runtime_remaining = 0;
+		/*
+		 * Do not enable runtime on offline runqueues. We specially
+		 * make it disabled in unthrottle_offline_cfs_rqs().
+		 */
+		if (cpu_online(i)) {
+			cfs_rq->runtime_enabled = runtime_enabled;
+			cfs_rq->runtime_remaining = 0;
Quoted 9 lines

kernel/sched/core.c static int tg_set_cfs_bandwidth(struct task_group *tg, u64 period, u64 quota) · line 7850

Quoted 4 lines
kernel/sched/core.c +4 -0
+
+			if (cfs_rq->throttled)
+				unthrottle_cfs_rq(cfs_rq);
+		}

We can just do for_each_online_cpu, yes? Also we probably need get_online_cpus/put_online_cpus, and/or want cpu_active_mask instead right?

Yes, we can use for_each_online_cpu/for_each_active_cpu with get_online_cpus() taken. But it adds one more lock dependence. This looks worse for me.

Quoted 6 lines
Quoted 4 lines

I mean, you need get_online_cpus anyway - cpu_online is just a test against the same mask that for_each_online_cpu uses, and without taking the lock you can still race with offlining and reset runtime_enabled.

Oh, I see. Thanks.

But we can check for rq->online, don't we? How about this?

Quoted 7 lines
Quoted 4 lines
Quoted 1 line

Yeah, that should work.

We can't base on it because rq->offline is not available in !SMP. Could you review the message from [PATCH v3 1/3] topic?

I'm not sure what you mean here. The patch just checking cpu_online won't work, is there another version you want me to look at?

I mean this one: https://lkml.org/lkml/2014/6/25/123

Have you received it? My email client shows you are properly CCed.

Reviewed-by on v2 1/3
#12 lore raw
In reply to Kirill Tkhai
Quoted 35 lines · Kirill Tkhai

We kill rq->rd on the CPU_DOWN_PREPARE stage:

	cpuset_cpu_inactive -> cpuset_update_active_cpus -> partition_sched_domains ->
	-> cpu_attach_domain -> rq_attach_root -> set_rq_offline

This unthrottles all throttled cfs_rqs.

But the cpu is still able to call schedule() till

	take_cpu_down->__cpu_disable()

is called from stop_machine.

This case the tasks from just unthrottled cfs_rqs are pickable in a standard scheduler way, and they are picked by dying cpu. The cfs_rqs becomes throttled again, and migrate_tasks() in migration_call skips their tasks (one more unthrottle in migrate_tasks()->CPU_DYING does not happen, because rq->rd is already NULL).

Patch sets runtime_enabled to zero. This guarantees, the runtime is not accounted, and the cfs_rqs won't exceed given cfs_rq->runtime_remaining = 1, and tasks will be pickable in migrate_tasks(). runtime_enabled is recalculated again when rq becomes online again.

Ben Segall also noticed, we always enable runtime in tg_set_cfs_bandwidth(). Actually, we should do that for online cpus only. To fix that, we check if a cpu is online when its rq is locked. This guarantees we do not have races with set_rq_offline(), which also requires rq->lock.

v2: Fix race with tg_set_cfs_bandwidth().

    Move cfs_rq->runtime_enabled=0 above unthrottle_cfs_rq().
Quoted 8 lines
CC Konstantin Khorenko <[email protected]>
CC Ben Segall <[email protected]>
CC Paul Turner <[email protected]>
CC Srikar Dronamraju <[email protected]>
CC Mike Galbraith <[email protected]>
CC Peter Zijlstra <[email protected]>
CC Ingo Molnar <[email protected]>

looks good to me.

Signature · 2 lines
Thanks and Regards
Srikar Dronamraju
#13 lore raw

PATCH v2 2/3 sched/rt: __disable_runtime: Enqueue just unthrottled rt_rq back on the stack

In reply to a message not in the archive

Make rt_rq available for pick_next_task(). Otherwise, their tasks stay prisoned long time till dead cpu becomes alive again.

CC Konstantin Khorenko <[email protected]>
CC Ben Segall <[email protected]>
CC Paul Turner <[email protected]>
CC Srikar Dronamraju <[email protected]>
CC Mike Galbraith <[email protected]>
CC Peter Zijlstra <[email protected]>
CC Ingo Molnar <[email protected]>
---
 kernel/sched/rt.c |    3 +++
 1 file changed, 3 insertions(+)
kernel/sched/rt.c +3 -0
index a490831..671a8b5 100644
@@ -740,6 +740,9 @@ static void __disable_runtime(struct rq *rq)
 		rt_rq->rt_throttled = 0;
 		raw_spin_unlock(&rt_rq->rt_runtime_lock);
 		raw_spin_unlock(&rt_b->rt_runtime_lock);
+
+		/* Make rt_rq available for pick_next_task() */
+		sched_rt_rq_enqueue(rt_rq);
 	}
 }
 
#14 lore raw

PATCH v2 3/3 sched: Rework check_for_tasks()

In reply to a message not in the archive

1)Iterate throw all of threads in the system.
Check for all threads, not only for group leaders.

2)Check for p->on_rq instead of p->state and cputime.
Preempted task in !TASK_RUNNING state OR just created task may be queued, that we want to be reported too.

3)Use read_lock() instead of write_lock().
This function does not change any structures, and read_lock() is enough.

CC Konstantin Khorenko <[email protected]>
CC Ben Segall <[email protected]>
CC Paul Turner <[email protected]>
CC Srikar Dronamraju <[email protected]>
CC Mike Galbraith <[email protected]>
CC Peter Zijlstra <[email protected]>
CC Ingo Molnar <[email protected]>
---
 kernel/cpu.c |   33 ++++++++++++++++++++-------------
 1 file changed, 20 insertions(+), 13 deletions(-)
kernel/cpu.c +20 -13
index a343bde..81e2a38 100644
@@ -274,21 +274,28 @@ void clear_tasks_mm_cpumask(int cpu)
 	rcu_read_unlock();
 }
 
-static inline void check_for_tasks(int cpu)
+static inline void check_for_tasks(int dead_cpu)
 {
-	struct task_struct *p;
-	cputime_t utime, stime;
+	struct task_struct *g, *p;
 
-	write_lock_irq(&tasklist_lock);
-	for_each_process(p) {
-		task_cputime(p, &utime, &stime);
-		if (task_cpu(p) == cpu && p->state == TASK_RUNNING &&
-		    (utime || stime))
-			pr_warn("Task %s (pid = %d) is on cpu %d (state = %ld, flags = %x)\n",
-				p->comm, task_pid_nr(p), cpu,
-				p->state, p->flags);
-	}
-	write_unlock_irq(&tasklist_lock);
+	read_lock_irq(&tasklist_lock);
+	do_each_thread(g, p) {
+		if (!p->on_rq)
+			continue;
+		/*
+		 * We do the check with unlocked task_rq(p)->lock.
+		 * Order the reading to do not warn about a task,
+		 * which was running on this cpu in the past, and
+		 * it's just been woken on another cpu.
+		 */
+		rmb();
+		if (task_cpu(p) != dead_cpu)
+			continue;
+
+		pr_warn("Task %s (pid=%d) is on cpu %d (state=%ld, flags=%x)\n",
+			p->comm, task_pid_nr(p), dead_cpu, p->state, p->flags);
+	} while_each_thread(g, p);
+	read_unlock_irq(&tasklist_lock);
 }
 
 struct take_cpu_down_param {