sched: Allow per-cpu kernel threads to run on online && !active

PATCH In review Peter Zijlstra 3 participants 7 messages Mar 1, 2016 – May 6, 2016

Subsystems (5)

PowerPC Scheduler s390 CPU hotplug +1 more subsystems

lore

Share & tools

lore

Peter Zijlstra

author maintainer

2016-03-01 15:23 UTC #0 lore raw

PATCH 1/1 sched: Allow per-cpu kernel threads to run on online && !active

In order to enable symmetric hotplug, we must mirror the online && !active state of cpu-down on the cpu-up side.

However, to retain sanity, limit this state to per-cpu kthreads.

Aside from the change to set_cpus_allowed_ptr(), which allow moving the per-cpu kthreads on, the other critical piece is the cpu selection for pinned tasks in select_task_rq(). This avoids dropping into select_fallback_rq().

select_fallback_rq() cannot be allowed to select !active cpus because its used to migrate user tasks away. And we do not want to move user tasks onto cpus that are in transition.

Cc Lai Jiangshan <[email protected]>

Cc Jan H. Schönherr <[email protected]>

Cc Oleg Nesterov <[email protected]>

Requested-by Thomas Gleixner <[email protected]>

Tested-by Thomas Gleixner <[email protected]>

Signed-off-by Peter Zijlstra (Intel) <[email protected]>

---
 arch/powerpc/kernel/smp.c |    2 -
 arch/s390/kernel/smp.c    |    2 -
 include/linux/cpumask.h   |    6 +----
 kernel/sched/core.c       |   49 +++++++++++++++++++++++++++++++++++++++-------
 4 files changed, 46 insertions(+), 13 deletions(-)

arch/powerpc/kernel/smp.c +1 -1

@@ -542,7 +542,7 @@ int __cpu_up(unsigned int cpu, struct ta
 		smp_ops->give_timebase();
 
 	/* Wait until cpu puts itself in the online & active maps */
-	while (!cpu_online(cpu) || !cpu_active(cpu))
+	while (!cpu_online(cpu))
 		cpu_relax();
 
 	return 0;

arch/s390/kernel/smp.c +1 -1

@@ -832,7 +832,7 @@ int __cpu_up(unsigned int cpu, struct ta
 	pcpu_attach_task(pcpu, tidle);
 	pcpu_start_fn(pcpu, smp_start_secondary, NULL);
 	/* Wait until cpu puts itself in the online & active maps */
-	while (!cpu_online(cpu) || !cpu_active(cpu))
+	while (!cpu_online(cpu))
 		cpu_relax();
 	return 0;
 }

include/linux/cpumask.h +2 -4

@@ -745,12 +745,10 @@ set_cpu_present(unsigned int cpu, bool p
 static inline void
 set_cpu_online(unsigned int cpu, bool online)
 {
-	if (online) {
+	if (online)
 		cpumask_set_cpu(cpu, &__cpu_online_mask);
-		cpumask_set_cpu(cpu, &__cpu_active_mask);
-	} else {
+	else
 		cpumask_clear_cpu(cpu, &__cpu_online_mask);
-	}
 }
 
 static inline void

kernel/sched/core.c +42 -7

101 lines not on this page · raw message

Thomas Gleixner

2016-03-03 11:45 UTC #1 lore raw

Quoted 1 line · Peter Zijlstra

Tested-by Thomas Gleixner <[email protected]>

Works nicely especially when we move the set_cpu_active() calls as the last state in the state machine. See patch below.

Thanks,

tglx

8<--------------------------

Subject: cpu/hotplug: Handle cpu active as last state
From: Thomas Gleixner <[email protected]>
Date: Thu, 03 Mar 2016 12:33:37 +0100

We want to make sure that everything is initialized before we allow scheduling of arbitrary work on a upcoming CPU and we don't want to have no more random work on it when we shut it down.

Now that the scheduler handles this nicely via the cpu_active_mask we can put set_cpu_active() as last action when a cpu is brought up and as first action when it goes down.

Signed-off-by Thomas Gleixner <[email protected]>

Cc Peter Zijlstra <[email protected]>

---
 include/linux/cpuhotplug.h |    1 +
 kernel/cpu.c               |   19 +++++++++++++++++--
 kernel/sched/core.c        |   18 ------------------
 3 files changed, 18 insertions(+), 20 deletions(-)

include/linux/cpuhotplug.h +1 -0

@@ -16,6 +16,7 @@ enum cpuhp_state {
 	CPUHP_AP_NOTIFY_ONLINE,
 	CPUHP_AP_ONLINE_DYN,
 	CPUHP_AP_ONLINE_DYN_END		= CPUHP_AP_ONLINE_DYN + 30,
+	CPUHP_AP_ACTIVE,
 	CPUHP_ONLINE,
 };

kernel/cpu.c +17 -2

@@ -904,8 +904,6 @@ void cpuhp_online_idle(enum cpuhp_state
 
 	st->state = CPUHP_AP_ONLINE_IDLE;
 
-	/* The cpu is marked online, set it active now */
-	set_cpu_active(cpu, true);
 	/* Unpark the stopper thread and the hotplug thread of this cpu */
 	stop_machine_unpark(cpu);
 	kthread_unpark(st->thread);

@@ -917,6 +915,18 @@ void cpuhp_online_idle(enum cpuhp_state
 		complete(&st->done);
 }
 
+static int cpu_activate(unsigned int cpu)
+{
+	set_cpu_active(cpu, true);
+	return 0;
+}
+
+static int cpu_deactivate(unsigned int cpu)
+{
+	set_cpu_active(cpu, false);
+	return 0;
+}
+
 /* Requires cpu_add_remove_lock to be held */
 static int _cpu_up(unsigned int cpu, int tasks_frozen, enum cpuhp_state target)
 {

@@ -1213,6 +1223,11 @@ static struct cpuhp_step cpuhp_ap_states
 		.startup		= notify_online,
 		.teardown		= notify_down_prepare,
 	},
+	[CPUHP_AP_ACTIVE] = {
+		.name			= "active",
+		.startup		= cpu_activate,
+		.teardown		= cpu_deactivate,
+	},
 #endif
 	[CPUHP_ONLINE] = {
 		.name			= "online",

kernel/sched/core.c +0 ~~-18~~

@@ -5726,23 +5726,6 @@ static int sched_cpu_active(struct notif
 	case CPU_STARTING:
 		set_cpu_rq_start_time();
 		return NOTIFY_OK;
-
-	case CPU_DOWN_FAILED:
-		set_cpu_active(cpu, true);
-		return NOTIFY_OK;
-
-	default:
-		return NOTIFY_DONE;
-	}
-}
-
-static int sched_cpu_inactive(struct notifier_block *nfb,
-					unsigned long action, void *hcpu)
-{
-	switch (action & ~CPU_TASKS_FROZEN) {
-	case CPU_DOWN_PREPARE:
-		set_cpu_active((long)hcpu, false);
-		return NOTIFY_OK;
 	default:
 		return NOTIFY_DONE;
 	}

@@ -5761,7 +5744,6 @@ static int __init migration_init(void)
 
 	/* Register cpu active notifiers */
 	cpu_notifier(sched_cpu_active, CPU_PRI_SCHED_ACTIVE);
-	cpu_notifier(sched_cpu_inactive, CPU_PRI_SCHED_INACTIVE);
 
 	return 0;
 }

Peter Zijlstra

author maintainer Acked-by

2016-03-03 11:56 UTC #2 lore raw

Quoted 8 lines · Thomas Gleixner

Subject: cpu/hotplug: Handle cpu active as last state
From: Thomas Gleixner <[email protected]>
Date: Thu, 03 Mar 2016 12:33:37 +0100
We want to make sure that everything is initialized before we allow scheduling of arbitrary work on a upcoming CPU and we don't want to have no more random work on it when we shut it down.

Quoted 6 lines

Now that the scheduler handles this nicely via the cpu_active_mask we can put set_cpu_active() as last action when a cpu is brought up and as first action when it goes down.
Signed-off-by Thomas Gleixner <[email protected]>
Cc Peter Zijlstra <[email protected]>

Nice!

Acked-by Peter Zijlstra (Intel) <[email protected]>

kernel/sched/core.c static int sched_cpu_active(struct notif · line 5727

Quoted 10 lines

---
 include/linux/cpuhotplug.h |    1 +
 kernel/cpu.c               |   19 +++++++++++++++++--
 kernel/sched/core.c        |   18 ------------------
 3 files changed, 18 insertions(+), 20 deletions(-)

+++ b/kernel/sched/core.c

kernel/sched/core.c +0 -0

@@ -5726,23 +5726,6 @@ static int sched_cpu_active(struct notif
 	case CPU_STARTING:
 		set_cpu_rq_start_time();

If we stick that in notify_starting() we can kill all of this.

Quoted 29 lines

kernel/sched/core.c +0 ~~-18~~

 		return NOTIFY_OK;
-
-	case CPU_DOWN_FAILED:
-		set_cpu_active(cpu, true);
-		return NOTIFY_OK;
-
-	default:
-		return NOTIFY_DONE;
-	}
-}
-
-static int sched_cpu_inactive(struct notifier_block *nfb,
-					unsigned long action, void *hcpu)
-{
-	switch (action & ~CPU_TASKS_FROZEN) {
-	case CPU_DOWN_PREPARE:
-		set_cpu_active((long)hcpu, false);
-		return NOTIFY_OK;
 	default:
 		return NOTIFY_DONE;
 	}

@@ -5761,7 +5744,6 @@ static int __init migration_init(void)
 
 	/* Register cpu active notifiers */
 	cpu_notifier(sched_cpu_active, CPU_PRI_SCHED_ACTIVE);
-	cpu_notifier(sched_cpu_inactive, CPU_PRI_SCHED_INACTIVE);
 
 	return 0;
 }

Thomas Gleixner

2016-03-03 11:59 UTC #3 lore raw

Quoted 6 lines · Peter Zijlstra

kernel/sched/core.c static int sched_cpu_active(struct notif · line 5727
Quoted 4 lines
+++ b/kernel/sched/core.c
kernel/sched/core.c +0 -0
@@ -5726,23 +5726,6 @@ static int sched_cpu_active(struct notif
 	case CPU_STARTING:
 		set_cpu_rq_start_time();
If we stick that in notify_starting() we can kill all of this.

Yep. I'm going to kill that one by one :)

Thomas Gleixner

2016-03-03 17:46 UTC #4 lore raw

Quoted 9 lines · Thomas Gleixner

Quoted 6 lines · Peter Zijlstra
kernel/sched/core.c static int sched_cpu_active(struct notif · line 5727
Quoted 4 lines
+++ b/kernel/sched/core.c
kernel/sched/core.c +0 -0
@@ -5726,23 +5726,6 @@ static int sched_cpu_active(struct notif
 	case CPU_STARTING:
 		set_cpu_rq_start_time();
If we stick that in notify_starting() we can kill all of this.
Yep. I'm going to kill that one by one :)

After I'm done fixing the wreckage which is caused by moving set_active() alone. The numa and cpuset muck rely on it ....

But now that we can delay random workload on the upcoming/outgoing cpu we can move that stuff alltogether.

Thanks,

tglx

tip-bot for Peter Zijlstra (Intel)

merge bot

2016-05-05 11:22 UTC #5 lore raw

In reply to Peter Zijlstra

sched: Allow per-cpu kernel threads to run on online && !active

Commit-ID:  618d6e31623149c6203b46850e2e76ee0f29e577
Gitweb:     http://git.kernel.org/tip/618d6e31623149c6203b46850e2e76ee0f29e577
Author:     Peter Zijlstra (Intel) <[email protected]>
AuthorDate: Thu, 10 Mar 2016 12:54:08 +0100
Committer:  Thomas Gleixner <[email protected]>
CommitDate: Thu, 5 May 2016 13:17:52 +0200

sched: Allow per-cpu kernel threads to run on online && !active

In order to enable symmetric hotplug, we must mirror the online && !active state of cpu-down on the cpu-up side.

However, to retain sanity, limit this state to per-cpu kthreads.

select_fallback_rq() cannot be allowed to select !active cpus because its used to migrate user tasks away. And we do not want to move user tasks onto cpus that are in transition.

Requested-by Thomas Gleixner <[email protected]>

Signed-off-by Peter Zijlstra (Intel) <[email protected]>

Tested-by Thomas Gleixner <[email protected]>

Cc Lai Jiangshan <[email protected]>

Cc Jan H. Schönherr <[email protected]>

Cc Oleg Nesterov <[email protected]>

Cc [email protected]

Link http://lkml.kernel.org/r/[email protected] open here

Signed-off-by Thomas Gleixner <[email protected]>

---
 arch/powerpc/kernel/smp.c |  2 +-
 arch/s390/kernel/smp.c    |  2 +-
 include/linux/cpumask.h   |  6 ++----
 kernel/sched/core.c       | 49 ++++++++++++++++++++++++++++++++++++++++-------
 4 files changed, 46 insertions(+), 13 deletions(-)

arch/powerpc/kernel/smp.c +1 -1

index 8cac1eb..55c924b 100644
@@ -565,7 +565,7 @@ int __cpu_up(unsigned int cpu, struct task_struct *tidle)
 		smp_ops->give_timebase();
 
 	/* Wait until cpu puts itself in the online & active maps */
-	while (!cpu_online(cpu) || !cpu_active(cpu))
+	while (!cpu_online(cpu))
 		cpu_relax();
 
 	return 0;

arch/s390/kernel/smp.c +1 -1

index 40a6b4f..7b89a75 100644
@@ -832,7 +832,7 @@ int __cpu_up(unsigned int cpu, struct task_struct *tidle)
 	pcpu_attach_task(pcpu, tidle);
 	pcpu_start_fn(pcpu, smp_start_secondary, NULL);
 	/* Wait until cpu puts itself in the online & active maps */
-	while (!cpu_online(cpu) || !cpu_active(cpu))
+	while (!cpu_online(cpu))
 		cpu_relax();
 	return 0;
 }

include/linux/cpumask.h +2 -4

index 40cee6b..e828cf6 100644
@@ -743,12 +743,10 @@ set_cpu_present(unsigned int cpu, bool present)
 static inline void
 set_cpu_online(unsigned int cpu, bool online)
 {
-	if (online) {
+	if (online)
 		cpumask_set_cpu(cpu, &__cpu_online_mask);
-		cpumask_set_cpu(cpu, &__cpu_active_mask);
-	} else {
+	else
 		cpumask_clear_cpu(cpu, &__cpu_online_mask);
-	}
 }
 
 static inline void

kernel/sched/core.c +42 -7

101 lines not on this page · raw message

tip-bot for Peter Zijlstra (Intel)

merge bot

2016-05-06 13:04 UTC #6 lore raw

In reply to Peter Zijlstra

sched: Allow per-cpu kernel threads to run on online && !active

Commit-ID:  e9d867a67fd03ccc07248ca4e9c2f74fed494d5b
Gitweb:     http://git.kernel.org/tip/e9d867a67fd03ccc07248ca4e9c2f74fed494d5b
Author:     Peter Zijlstra (Intel) <[email protected]>
AuthorDate: Thu, 10 Mar 2016 12:54:08 +0100
Committer:  Thomas Gleixner <[email protected]>
CommitDate: Fri, 6 May 2016 14:58:22 +0200

sched: Allow per-cpu kernel threads to run on online && !active

In order to enable symmetric hotplug, we must mirror the online && !active state of cpu-down on the cpu-up side.

However, to retain sanity, limit this state to per-cpu kthreads.

select_fallback_rq() cannot be allowed to select !active cpus because its used to migrate user tasks away. And we do not want to move user tasks onto cpus that are in transition.

Requested-by Thomas Gleixner <[email protected]>

Signed-off-by Peter Zijlstra (Intel) <[email protected]>

Tested-by Thomas Gleixner <[email protected]>

Cc Lai Jiangshan <[email protected]>

Cc Jan H. Schönherr <[email protected]>

Cc Oleg Nesterov <[email protected]>

Cc [email protected]

Link http://lkml.kernel.org/r/[email protected] open here

Signed-off-by Thomas Gleixner <[email protected]>

---
 arch/powerpc/kernel/smp.c |  2 +-
 arch/s390/kernel/smp.c    |  2 +-
 include/linux/cpumask.h   |  6 ++----
 kernel/sched/core.c       | 49 ++++++++++++++++++++++++++++++++++++++++-------
 4 files changed, 46 insertions(+), 13 deletions(-)

arch/powerpc/kernel/smp.c +1 -1

index 8cac1eb..55c924b 100644
@@ -565,7 +565,7 @@ int __cpu_up(unsigned int cpu, struct task_struct *tidle)
 		smp_ops->give_timebase();
 
 	/* Wait until cpu puts itself in the online & active maps */
-	while (!cpu_online(cpu) || !cpu_active(cpu))
+	while (!cpu_online(cpu))
 		cpu_relax();
 
 	return 0;

arch/s390/kernel/smp.c +1 -1

index 40a6b4f..7b89a75 100644
@@ -832,7 +832,7 @@ int __cpu_up(unsigned int cpu, struct task_struct *tidle)
 	pcpu_attach_task(pcpu, tidle);
 	pcpu_start_fn(pcpu, smp_start_secondary, NULL);
 	/* Wait until cpu puts itself in the online & active maps */
-	while (!cpu_online(cpu) || !cpu_active(cpu))
+	while (!cpu_online(cpu))
 		cpu_relax();
 	return 0;
 }

include/linux/cpumask.h +2 -4

index 40cee6b..e828cf6 100644
@@ -743,12 +743,10 @@ set_cpu_present(unsigned int cpu, bool present)
 static inline void
 set_cpu_online(unsigned int cpu, bool online)
 {
-	if (online) {
+	if (online)
 		cpumask_set_cpu(cpu, &__cpu_online_mask);
-		cpumask_set_cpu(cpu, &__cpu_active_mask);
-	} else {
+	else
 		cpumask_clear_cpu(cpu, &__cpu_online_mask);
-	}
 }
 
 static inline void

kernel/sched/core.c +42 -7

101 lines not on this page · raw message