Skip to contentSkip to conversation

sched: Core Scheduling

PATCH Applied Peter Zijlstra 13 participants 103 messages Apr 22, 2021 – Aug 23, 2021 Page 2 of 2

Subsystems (5)
lore
Share & tools
lore
#62 lore raw

[tip: sched/core] sched/fair: Snapshot the min_vruntime of CPUs on force idle

In reply to Peter Zijlstra
The following commit has been merged into the sched/core branch of tip:

The following commit has been merged into the sched/core branch of tip:

Commit-ID:     c6047c2e3af68dae23ad884249e0d42ff28d2d1b
Gitweb:        https://git.kernel.org/tip/c6047c2e3af68dae23ad884249e0d42ff28d2d1b
Author:        Joel Fernandes (Google) <[email protected]>
AuthorDate:    Tue, 17 Nov 2020 18:19:39 -05:00
Committer:     Peter Zijlstra <[email protected]>
CommitterDate: Wed, 12 May 2021 11:43:29 +02:00

sched/fair: Snapshot the min_vruntime of CPUs on force idle

During force-idle, we end up doing cross-cpu comparison of vruntimes during pick_next_task. If we simply compare (vruntime-min_vruntime) across CPUs, and if the CPUs only have 1 task each, we will always end up comparing 0 with 0 and pick just one of the tasks all the time. This starves the task that was not picked. To fix this, take a snapshot of the min_vruntime when entering force idle and use it for comparison. This min_vruntime snapshot will only be used for cross-CPU vruntime comparison, and nothing else.

A note about the min_vruntime snapshot and force idling:

During selection:

When we're not fi, we need to update snapshot.
when we're fi and we were not fi, we must update snapshot. When we're fi and we were already fi, we must not update snapshot.

Which gives:

  fib     fi      update
  0       0       1
  0       1       1
  1       0       1
  1       1       0

Where:

fi: force-idled now
fib: force-idled before

So the min_vruntime snapshot needs to be updated when: !(fib && fi).

Also, the cfs_prio_less() function needs to be aware of whether the core is in force idle or not, since it will be use this information to know whether to advance a cfs_rq's min_vruntime_fi in the hierarchy. So pass this information along via pick_task() -> prio_less().

---
 kernel/sched/core.c  | 59 +++++++++++++++++++---------------
 kernel/sched/fair.c  | 75 +++++++++++++++++++++++++++++++++++++++++++-
 kernel/sched/sched.h |  8 +++++-
 3 files changed, 117 insertions(+), 25 deletions(-)
kernel/sched/core.c +34 -25
index e506d9d..e45c1d2 100644
@@ -111,7 +111,7 @@ static inline int __task_prio(struct task_struct *p)
  */
 
 /* real prio, less is less */
-static inline bool prio_less(struct task_struct *a, struct task_struct *b)
+static inline bool prio_less(struct task_struct *a, struct task_struct *b, bool in_fi)
 {
 
 	int pa = __task_prio(a), pb = __task_prio(b);
@@ -125,19 +125,8 @@ static inline bool prio_less(struct task_struct *a, struct task_struct *b)
 	if (pa == -1) /* dl_prio() doesn't work because of stop_class above */
 		return !dl_time_before(a->dl.deadline, b->dl.deadline);
 
-	if (pa == MAX_RT_PRIO + MAX_NICE)  { /* fair */
-		u64 vruntime = b->se.vruntime;
-
-		/*
-		 * Normalize the vruntime if tasks are in different cpus.
-		 */
-		if (task_cpu(a) != task_cpu(b)) {
-			vruntime -= task_cfs_rq(b)->min_vruntime;
-			vruntime += task_cfs_rq(a)->min_vruntime;
-		}
-
-		return !((s64)(a->se.vruntime - vruntime) <= 0);
-	}
+	if (pa == MAX_RT_PRIO + MAX_NICE)	/* fair */
+		return cfs_prio_less(a, b, in_fi);
 
 	return false;
 }
@@ -151,7 +140,7 @@ static inline bool __sched_core_less(struct task_struct *a, struct task_struct *
 		return false;
 
 	/* flip prio, so high prio is leftmost */
-	if (prio_less(b, a))
+	if (prio_less(b, a, task_rq(a)->core->core_forceidle))
 		return true;
 
 	return false;
@@ -5350,7 +5339,7 @@ static inline bool cookie_match(struct task_struct *a, struct task_struct *b)
  * - Else returns idle_task.
  */
 static struct task_struct *
-pick_task(struct rq *rq, const struct sched_class *class, struct task_struct *max)
+pick_task(struct rq *rq, const struct sched_class *class, struct task_struct *max, bool in_fi)
 {
 	struct task_struct *class_pick, *cookie_pick;
 	unsigned long cookie = rq->core->core_cookie;
@@ -5365,7 +5354,7 @@ pick_task(struct rq *rq, const struct sched_class *class, struct task_struct *ma
 		 * higher priority than max.
 		 */
 		if (max && class_pick->core_cookie &&
-		    prio_less(class_pick, max))
+		    prio_less(class_pick, max, in_fi))
 			return idle_sched_class.pick_task(rq);
 
 		return class_pick;
@@ -5384,19 +5373,22 @@ pick_task(struct rq *rq, const struct sched_class *class, struct task_struct *ma
 	 * the core (so far) and it must be selected, otherwise we must go with
 	 * the cookie pick in order to satisfy the constraint.
 	 */
-	if (prio_less(cookie_pick, class_pick) &&
-	    (!max || prio_less(max, class_pick)))
+	if (prio_less(cookie_pick, class_pick, in_fi) &&
+	    (!max || prio_less(max, class_pick, in_fi)))
 		return class_pick;
 
 	return cookie_pick;
 }
 
+extern void task_vruntime_update(struct rq *rq, struct task_struct *p, bool in_fi);
+
 static struct task_struct *
 pick_next_task(struct rq *rq, struct task_struct *prev, struct rq_flags *rf)
 {
 	struct task_struct *next, *max = NULL;
 	const struct sched_class *class;
 	const struct cpumask *smt_mask;
+	bool fi_before = false;
 	bool need_sync;
 	int i, j, cpu;
 
@@ -5478,9 +5470,14 @@ pick_next_task(struct rq *rq, struct task_struct *prev, struct rq_flags *rf)
 
 		if (!next->core_cookie) {
 			rq->core_pick = NULL;
+			/*
+			 * For robustness, update the min_vruntime_fi for
+			 * unconstrained picks as well.
+			 */
+			WARN_ON_ONCE(fi_before);
+			task_vruntime_update(rq, next, false);
 			goto done;
 		}
-		need_sync = true;
 	}
 
 	for_each_cpu(i, smt_mask) {
@@ -5511,11 +5508,16 @@ again:
 			 * highest priority task already selected for this
 			 * core.
 			 */
-			p = pick_task(rq_i, class, max);
+			p = pick_task(rq_i, class, max, fi_before);
 			if (!p)
 				continue;
 
 			rq_i->core_pick = p;
+			if (rq_i->idle == p && rq_i->nr_running) {
+				rq->core->core_forceidle = true;
+				if (!fi_before)
+					rq->core->core_forceidle_seq++;
+			}
 
 			/*
 			 * If this new candidate is of higher priority than the
@@ -5534,6 +5536,7 @@ again:
 				max = p;
 
 				if (old_max) {
+					rq->core->core_forceidle = false;
 					for_each_cpu(j, smt_mask) {
 						if (j == i)
 							continue;
@@ -5574,10 +5577,16 @@ again:
 		if (!rq_i->core_pick)
 			continue;
 
-		if (is_task_rq_idle(rq_i->core_pick) && rq_i->nr_running &&
-		    !rq_i->core->core_forceidle) {
-			rq_i->core->core_forceidle = true;
-		}
+		/*
+		 * Update for new !FI->FI transitions, or if continuing to be in !FI:
+		 * fi_before     fi      update?
+		 *  0            0       1
+		 *  0            1       1
+		 *  1            0       1
+		 *  1            1       0
+		 */
+		if (!(fi_before && rq->core->core_forceidle))
+			task_vruntime_update(rq_i, rq_i->core_pick, rq->core->core_forceidle);
 
 		if (i == cpu) {
 			rq_i->core_pick = NULL;
kernel/sched/fair.c +75 -0

81 lines not on this page · raw message

kernel/sched/sched.h +8 -0
index db55514..4a898ab 100644
@@ -526,6 +526,11 @@ struct cfs_rq {
 
 	u64			exec_clock;
 	u64			min_vruntime;
+#ifdef CONFIG_SCHED_CORE
+	unsigned int		forceidle_seq;
+	u64			min_vruntime_fi;
+#endif
+
 #ifndef CONFIG_64BIT
 	u64			min_vruntime_copy;
 #endif
@@ -1089,6 +1094,7 @@ struct rq {
 	unsigned int		core_pick_seq;
 	unsigned long		core_cookie;
 	unsigned char		core_forceidle;
+	unsigned int		core_forceidle_seq;
 #endif
 };
 
@@ -1162,6 +1168,8 @@ static inline raw_spinlock_t *__rq_lockp(struct rq *rq)
 	return &rq->__lock;
 }
 
+bool cfs_prio_less(struct task_struct *a, struct task_struct *b, bool fi);
+
 #else /* !CONFIG_SCHED_CORE */
 
 static inline bool sched_core_enabled(struct rq *rq)
author maintainer
#63 lore raw

PATCH 14/19 sched: Trivial forced-newidle balancer

In reply to Peter Zijlstra

When a sibling is forced-idle to match the core-cookie; search for matching tasks to fill the core.

rcu_read_unlock() can incur an infrequent deadlock in sched_core_balance(). Fix this by using the RCU-sched flavor instead.

---
 include/linux/sched.h |    1 
 kernel/sched/core.c   |  130 +++++++++++++++++++++++++++++++++++++++++++++++++-
 kernel/sched/idle.c   |    1 
 kernel/sched/sched.h  |    6 ++
 4 files changed, 137 insertions(+), 1 deletion(-)
include/linux/sched.h +1 -0
@@ -704,6 +704,7 @@ struct task_struct {
 #ifdef CONFIG_SCHED_CORE
 	struct rb_node			core_node;
 	unsigned long			core_cookie;
+	unsigned int			core_occupation;
 #endif
 
 #ifdef CONFIG_CGROUP_SCHED
kernel/sched/core.c +129 -1

167 lines not on this page · raw message

kernel/sched/idle.c +1 -0
@@ -437,6 +437,7 @@ static void set_next_task_idle(struct rq
 {
 	update_idle_core(rq);
 	schedstat_inc(rq->sched_goidle);
+	queue_core_balance(rq);
 }
 
 #ifdef CONFIG_SMP
kernel/sched/sched.h +6 -0
@@ -1170,6 +1170,8 @@ static inline raw_spinlock_t *__rq_lockp
 
 bool cfs_prio_less(struct task_struct *a, struct task_struct *b, bool fi);
 
+extern void queue_core_balance(struct rq *rq);
+
 #else /* !CONFIG_SCHED_CORE */
 
 static inline bool sched_core_enabled(struct rq *rq)
@@ -1187,6 +1189,10 @@ static inline raw_spinlock_t *rq_lockp(s
 	return &rq->__lock;
 }
 
+static inline void queue_core_balance(struct rq *rq)
+{
+}
+
 #endif /* CONFIG_SCHED_CORE */
 
 static inline void lockdep_assert_rq_held(struct rq *rq)
merge bot Applied
#64 lore raw

[tip: sched/core] sched: Trivial forced-newidle balancer

The following commit has been merged into the sched/core branch of tip:

The following commit has been merged into the sched/core branch of tip:

Commit-ID:     d2dfa17bc7de67e99685c4d6557837bf801a102c
Gitweb:        https://git.kernel.org/tip/d2dfa17bc7de67e99685c4d6557837bf801a102c
Author:        Peter Zijlstra <[email protected]>
AuthorDate:    Tue, 17 Nov 2020 18:19:43 -05:00
Committer:     Peter Zijlstra <[email protected]>
CommitterDate: Wed, 12 May 2021 11:43:30 +02:00

sched: Trivial forced-newidle balancer

When a sibling is forced-idle to match the core-cookie; search for matching tasks to fill the core.

rcu_read_unlock() can incur an infrequent deadlock in sched_core_balance(). Fix this by using the RCU-sched flavor instead.

---
 include/linux/sched.h |   1 +-
 kernel/sched/core.c   | 130 ++++++++++++++++++++++++++++++++++++++++-
 kernel/sched/idle.c   |   1 +-
 kernel/sched/sched.h  |   6 ++-
 4 files changed, 137 insertions(+), 1 deletion(-)
include/linux/sched.h +1 -0
index 45eedcc..9b822e3 100644
@@ -705,6 +705,7 @@ struct task_struct {
 #ifdef CONFIG_SCHED_CORE
 	struct rb_node			core_node;
 	unsigned long			core_cookie;
+	unsigned int			core_occupation;
 #endif
 
 #ifdef CONFIG_CGROUP_SCHED
kernel/sched/core.c +129 -1

167 lines not on this page · raw message

kernel/sched/idle.c +1 -0
index 43646e7..912b47a 100644
@@ -437,6 +437,7 @@ static void set_next_task_idle(struct rq *rq, struct task_struct *next, bool fir
 {
 	update_idle_core(rq);
 	schedstat_inc(rq->sched_goidle);
+	queue_core_balance(rq);
 }
 
 #ifdef CONFIG_SMP
kernel/sched/sched.h +6 -0
index 4a898ab..91ca1fe 100644
@@ -1170,6 +1170,8 @@ static inline raw_spinlock_t *__rq_lockp(struct rq *rq)
 
 bool cfs_prio_less(struct task_struct *a, struct task_struct *b, bool fi);
 
+extern void queue_core_balance(struct rq *rq);
+
 #else /* !CONFIG_SCHED_CORE */
 
 static inline bool sched_core_enabled(struct rq *rq)
@@ -1192,6 +1194,10 @@ static inline raw_spinlock_t *__rq_lockp(struct rq *rq)
 	return &rq->__lock;
 }
 
+static inline void queue_core_balance(struct rq *rq)
+{
+}
+
 #endif /* CONFIG_SCHED_CORE */
 
 static inline void lockdep_assert_rq_held(struct rq *rq)
author maintainer
#65 lore raw

PATCH 15/19 sched: Migration changes for core scheduling

In reply to Peter Zijlstra

From: Aubrey Li <[email protected]>

- Don't migrate if there is a cookie mismatch
Load balance tries to move task from busiest CPU to the destination CPU. When core scheduling is enabled, if the task's cookie does not match with the destination CPU's core cookie, this task may be skipped by this CPU. This mitigates the forced idle time on the destination CPU.

- Select cookie matched idle CPU
In the fast path of task wakeup, select the first cookie matched idle CPU instead of the first idle CPU.

- Find cookie matched idlest CPU
In the slow path of task wakeup, find the idlest CPU whose core cookie matches with task's cookie

---
 kernel/sched/fair.c  |   29 ++++++++++++++++----
 kernel/sched/sched.h |   73 +++++++++++++++++++++++++++++++++++++++++++++++++++
 2 files changed, 96 insertions(+), 6 deletions(-)
kernel/sched/fair.c +23 -6
@@ -5889,11 +5889,15 @@ find_idlest_group_cpu(struct sched_group
 
 	/* Traverse only the allowed CPUs */
 	for_each_cpu_and(i, sched_group_span(group), p->cpus_ptr) {
+		struct rq *rq = cpu_rq(i);
+
+		if (!sched_core_cookie_match(rq, p))
+			continue;
+
 		if (sched_idle_cpu(i))
 			return i;
 
 		if (available_idle_cpu(i)) {
-			struct rq *rq = cpu_rq(i);
 			struct cpuidle_state *idle = idle_get_state(rq);
 			if (idle && idle->exit_latency < min_exit_latency) {
 				/*
@@ -5979,9 +5983,10 @@ static inline int find_idlest_cpu(struct
 	return new_cpu;
 }
 
-static inline int __select_idle_cpu(int cpu)
+static inline int __select_idle_cpu(int cpu, struct task_struct *p)
 {
-	if (available_idle_cpu(cpu) || sched_idle_cpu(cpu))
+	if ((available_idle_cpu(cpu) || sched_idle_cpu(cpu)) &&
+	    sched_cpu_cookie_match(cpu_rq(cpu), p))
 		return cpu;
 
 	return -1;
@@ -6051,7 +6056,7 @@ static int select_idle_core(struct task_
 	int cpu;
 
 	if (!static_branch_likely(&sched_smt_present))
-		return __select_idle_cpu(core);
+		return __select_idle_cpu(core, p);
 
 	for_each_cpu(cpu, cpu_smt_mask(core)) {
 		if (!available_idle_cpu(cpu)) {
@@ -6107,7 +6112,7 @@ static inline bool test_idle_cores(int c
 
 static inline int select_idle_core(struct task_struct *p, int core, struct cpumask *cpus, int *idle_cpu)
 {
-	return __select_idle_cpu(core);
+	return __select_idle_cpu(core, p);
 }
 
 static inline int select_idle_smt(struct task_struct *p, struct sched_domain *sd, int target)
@@ -6164,7 +6169,7 @@ static int select_idle_cpu(struct task_s
 		} else {
 			if (!--nr)
 				return -1;
-			idle_cpu = __select_idle_cpu(cpu);
+			idle_cpu = __select_idle_cpu(cpu, p);
 			if ((unsigned int)idle_cpu < nr_cpumask_bits)
 				break;
 		}
@@ -7517,6 +7522,14 @@ static int task_hot(struct task_struct *
 
 	if (sysctl_sched_migration_cost == -1)
 		return 1;
+
+	/*
+	 * Don't migrate task if the task's cookie does not match
+	 * with the destination CPU's core cookie.
+	 */
+	if (!sched_core_cookie_match(cpu_rq(env->dst_cpu), p))
+		return 1;
+
 	if (sysctl_sched_migration_cost == 0)
 		return 0;
 
@@ -8847,6 +8860,10 @@ find_idlest_group(struct sched_domain *s
 					p->cpus_ptr))
 			continue;
 
+		/* Skip over this group if no cookie matched */
+		if (!sched_group_cookie_match(cpu_rq(this_cpu), p, group))
+			continue;
+
 		local_group = cpumask_test_cpu(this_cpu,
 					       sched_group_span(group));
 
kernel/sched/sched.h +73 -0

92 lines not on this page · raw message

merge bot Applied
#66 lore raw

[tip: sched/core] sched: Migration changes for core scheduling

The following commit has been merged into the sched/core branch of tip:

The following commit has been merged into the sched/core branch of tip:

Commit-ID:     97886d9dcd86820bdbc1fa73b455982809cbc8c2
Gitweb:        https://git.kernel.org/tip/97886d9dcd86820bdbc1fa73b455982809cbc8c2
Author:        Aubrey Li <[email protected]>
AuthorDate:    Wed, 24 Mar 2021 17:40:13 -04:00
Committer:     Peter Zijlstra <[email protected]>
CommitterDate: Wed, 12 May 2021 11:43:30 +02:00

sched: Migration changes for core scheduling

- Don't migrate if there is a cookie mismatch
Load balance tries to move task from busiest CPU to the destination CPU. When core scheduling is enabled, if the task's cookie does not match with the destination CPU's core cookie, this task may be skipped by this CPU. This mitigates the forced idle time on the destination CPU.

- Select cookie matched idle CPU
In the fast path of task wakeup, select the first cookie matched idle CPU instead of the first idle CPU.

- Find cookie matched idlest CPU
In the slow path of task wakeup, find the idlest CPU whose core cookie matches with task's cookie

---
 kernel/sched/fair.c  | 29 +++++++++++++----
 kernel/sched/sched.h | 73 +++++++++++++++++++++++++++++++++++++++++++-
 2 files changed, 96 insertions(+), 6 deletions(-)
kernel/sched/fair.c +23 -6
index 5948dc1..2635e10 100644
@@ -5889,11 +5889,15 @@ find_idlest_group_cpu(struct sched_group *group, struct task_struct *p, int this
 
 	/* Traverse only the allowed CPUs */
 	for_each_cpu_and(i, sched_group_span(group), p->cpus_ptr) {
+		struct rq *rq = cpu_rq(i);
+
+		if (!sched_core_cookie_match(rq, p))
+			continue;
+
 		if (sched_idle_cpu(i))
 			return i;
 
 		if (available_idle_cpu(i)) {
-			struct rq *rq = cpu_rq(i);
 			struct cpuidle_state *idle = idle_get_state(rq);
 			if (idle && idle->exit_latency < min_exit_latency) {
 				/*
@@ -5979,9 +5983,10 @@ static inline int find_idlest_cpu(struct sched_domain *sd, struct task_struct *p
 	return new_cpu;
 }
 
-static inline int __select_idle_cpu(int cpu)
+static inline int __select_idle_cpu(int cpu, struct task_struct *p)
 {
-	if (available_idle_cpu(cpu) || sched_idle_cpu(cpu))
+	if ((available_idle_cpu(cpu) || sched_idle_cpu(cpu)) &&
+	    sched_cpu_cookie_match(cpu_rq(cpu), p))
 		return cpu;
 
 	return -1;
@@ -6051,7 +6056,7 @@ static int select_idle_core(struct task_struct *p, int core, struct cpumask *cpu
 	int cpu;
 
 	if (!static_branch_likely(&sched_smt_present))
-		return __select_idle_cpu(core);
+		return __select_idle_cpu(core, p);
 
 	for_each_cpu(cpu, cpu_smt_mask(core)) {
 		if (!available_idle_cpu(cpu)) {
@@ -6107,7 +6112,7 @@ static inline bool test_idle_cores(int cpu, bool def)
 
 static inline int select_idle_core(struct task_struct *p, int core, struct cpumask *cpus, int *idle_cpu)
 {
-	return __select_idle_cpu(core);
+	return __select_idle_cpu(core, p);
 }
 
 static inline int select_idle_smt(struct task_struct *p, struct sched_domain *sd, int target)
@@ -6164,7 +6169,7 @@ static int select_idle_cpu(struct task_struct *p, struct sched_domain *sd, bool 
 		} else {
 			if (!--nr)
 				return -1;
-			idle_cpu = __select_idle_cpu(cpu);
+			idle_cpu = __select_idle_cpu(cpu, p);
 			if ((unsigned int)idle_cpu < nr_cpumask_bits)
 				break;
 		}
@@ -7527,6 +7532,14 @@ static int task_hot(struct task_struct *p, struct lb_env *env)
 
 	if (sysctl_sched_migration_cost == -1)
 		return 1;
+
+	/*
+	 * Don't migrate task if the task's cookie does not match
+	 * with the destination CPU's core cookie.
+	 */
+	if (!sched_core_cookie_match(cpu_rq(env->dst_cpu), p))
+		return 1;
+
 	if (sysctl_sched_migration_cost == 0)
 		return 0;
 
@@ -8857,6 +8870,10 @@ find_idlest_group(struct sched_domain *sd, struct task_struct *p, int this_cpu)
 					p->cpus_ptr))
 			continue;
 
+		/* Skip over this group if no cookie matched */
+		if (!sched_group_cookie_match(cpu_rq(this_cpu), p, group))
+			continue;
+
 		local_group = cpumask_test_cpu(this_cpu,
 					       sched_group_span(group));
 
kernel/sched/sched.h +73 -0

92 lines not on this page · raw message

author maintainer
#67 lore raw

PATCH 16/19 sched: Trivial core scheduling cookie management

In reply to Peter Zijlstra

In order to not have to use pid_struct, create a new, smaller, structure to manage task cookies for core scheduling.

---
 include/linux/sched.h     |    6 ++
 kernel/fork.c             |    1 
 kernel/sched/Makefile     |    1 
 kernel/sched/core.c       |    7 +-
 kernel/sched/core_sched.c |  109 ++++++++++++++++++++++++++++++++++++++++++++++
 kernel/sched/sched.h      |   16 ++++++
 6 files changed, 137 insertions(+), 3 deletions(-)
include/linux/sched.h +6 -0
@@ -2170,4 +2170,10 @@ int sched_trace_rq_nr_running(struct rq
 
 const struct cpumask *sched_trace_rd_span(struct root_domain *rd);
 
+#ifdef CONFIG_SCHED_CORE
+extern void sched_core_free(struct task_struct *tsk);
+#else
+static inline void sched_core_free(struct task_struct *tsk) { }
+#endif
+
 #endif
kernel/fork.c +1 -0
@@ -737,6 +737,7 @@ void __put_task_struct(struct task_struc
 	exit_creds(tsk);
 	delayacct_tsk_free(tsk);
 	put_signal_struct(tsk->signal);
+	sched_core_free(tsk);
 
 	if (!profile_handoff_task(tsk))
 		free_task(tsk);
kernel/sched/Makefile +1 -0
@@ -36,3 +36,4 @@ obj-$(CONFIG_CPU_FREQ_GOV_SCHEDUTIL) +=
 obj-$(CONFIG_MEMBARRIER) += membarrier.o
 obj-$(CONFIG_CPU_ISOLATION) += isolation.o
 obj-$(CONFIG_PSI) += psi.o
+obj-$(CONFIG_SCHED_CORE) += core_sched.o
kernel/sched/core.c +4 -3
@@ -157,7 +157,7 @@ static inline int rb_sched_core_cmp(cons
 	return 0;
 }
 
-static void sched_core_enqueue(struct rq *rq, struct task_struct *p)
+void sched_core_enqueue(struct rq *rq, struct task_struct *p)
 {
 	rq->core->core_task_seq++;
 
@@ -167,14 +167,15 @@ static void sched_core_enqueue(struct rq
 	rb_add(&p->core_node, &rq->core_tree, rb_sched_core_less);
 }
 
-static void sched_core_dequeue(struct rq *rq, struct task_struct *p)
+void sched_core_dequeue(struct rq *rq, struct task_struct *p)
 {
 	rq->core->core_task_seq++;
 
-	if (!p->core_cookie)
+	if (!sched_core_enqueued(p))
 		return;
 
 	rb_erase(&p->core_node, &rq->core_tree);
+	RB_CLEAR_NODE(&p->core_node);
 }
 
 /*
kernel/sched/core_sched.c new file +109 -0

109 lines not on this page · raw message

kernel/sched/sched.h +16 -0
@@ -1222,6 +1222,22 @@ static inline bool sched_group_cookie_ma
 
 extern void queue_core_balance(struct rq *rq);
 
+static inline bool sched_core_enqueued(struct task_struct *p)
+{
+	return !RB_EMPTY_NODE(&p->core_node);
+}
+
+extern void sched_core_enqueue(struct rq *rq, struct task_struct *p);
+extern void sched_core_dequeue(struct rq *rq, struct task_struct *p);
+
+extern void sched_core_get(void);
+extern void sched_core_put(void);
+
+extern unsigned long sched_core_alloc_cookie(void);
+extern void sched_core_put_cookie(unsigned long cookie);
+extern unsigned long sched_core_get_cookie(unsigned long cookie);
+extern unsigned long sched_core_update_cookie(struct task_struct *p, unsigned long cookie);
+
 #else /* !CONFIG_SCHED_CORE */
 
 static inline bool sched_core_enabled(struct rq *rq)
merge bot Applied
#68 lore raw

[tip: sched/core] sched: Trivial core scheduling cookie management

The following commit has been merged into the sched/core branch of tip:

The following commit has been merged into the sched/core branch of tip:

Commit-ID:     6e33cad0af49336952e5541464bd02f5b5fd433e
Gitweb:        https://git.kernel.org/tip/6e33cad0af49336952e5541464bd02f5b5fd433e
Author:        Peter Zijlstra <[email protected]>
AuthorDate:    Fri, 26 Mar 2021 18:55:06 +01:00
Committer:     Peter Zijlstra <[email protected]>
CommitterDate: Wed, 12 May 2021 11:43:31 +02:00

sched: Trivial core scheduling cookie management

In order to not have to use pid_struct, create a new, smaller, structure to manage task cookies for core scheduling.

---
 include/linux/sched.h     |   6 ++-
 kernel/fork.c             |   1 +-
 kernel/sched/Makefile     |   1 +-
 kernel/sched/core.c       |   7 +-
 kernel/sched/core_sched.c | 109 +++++++++++++++++++++++++++++++++++++-
 kernel/sched/sched.h      |  16 +++++-
 6 files changed, 137 insertions(+), 3 deletions(-)
 create mode 100644 kernel/sched/core_sched.c
include/linux/sched.h +6 -0
index 9b822e3..eab3f7c 100644
@@ -2179,4 +2179,10 @@ int sched_trace_rq_nr_running(struct rq *rq);
 
 const struct cpumask *sched_trace_rd_span(struct root_domain *rd);
 
+#ifdef CONFIG_SCHED_CORE
+extern void sched_core_free(struct task_struct *tsk);
+#else
+static inline void sched_core_free(struct task_struct *tsk) { }
+#endif
+
 #endif
kernel/fork.c +1 -0
index dc06afd..d16c60c 100644
@@ -742,6 +742,7 @@ void __put_task_struct(struct task_struct *tsk)
 	exit_creds(tsk);
 	delayacct_tsk_free(tsk);
 	put_signal_struct(tsk->signal);
+	sched_core_free(tsk);
 
 	if (!profile_handoff_task(tsk))
 		free_task(tsk);
kernel/sched/Makefile +1 -0
index 5fc9c9b..978fcfc 100644
@@ -36,3 +36,4 @@ obj-$(CONFIG_CPU_FREQ_GOV_SCHEDUTIL) += cpufreq_schedutil.o
 obj-$(CONFIG_MEMBARRIER) += membarrier.o
 obj-$(CONFIG_CPU_ISOLATION) += isolation.o
 obj-$(CONFIG_PSI) += psi.o
+obj-$(CONFIG_SCHED_CORE) += core_sched.o
kernel/sched/core.c +4 -3
index b498888..55b2d93 100644
@@ -167,7 +167,7 @@ static inline int rb_sched_core_cmp(const void *key, const struct rb_node *node)
 	return 0;
 }
 
-static void sched_core_enqueue(struct rq *rq, struct task_struct *p)
+void sched_core_enqueue(struct rq *rq, struct task_struct *p)
 {
 	rq->core->core_task_seq++;
 
@@ -177,14 +177,15 @@ static void sched_core_enqueue(struct rq *rq, struct task_struct *p)
 	rb_add(&p->core_node, &rq->core_tree, rb_sched_core_less);
 }
 
-static void sched_core_dequeue(struct rq *rq, struct task_struct *p)
+void sched_core_dequeue(struct rq *rq, struct task_struct *p)
 {
 	rq->core->core_task_seq++;
 
-	if (!p->core_cookie)
+	if (!sched_core_enqueued(p))
 		return;
 
 	rb_erase(&p->core_node, &rq->core_tree);
+	RB_CLEAR_NODE(&p->core_node);
 }
 
 /*
kernel/sched/core_sched.c new file +109 -0

109 lines not on this page · raw message

kernel/sched/sched.h +16 -0
index 3878386..904c52b 100644
@@ -1229,6 +1229,22 @@ static inline bool sched_group_cookie_match(struct rq *rq,
 
 extern void queue_core_balance(struct rq *rq);
 
+static inline bool sched_core_enqueued(struct task_struct *p)
+{
+	return !RB_EMPTY_NODE(&p->core_node);
+}
+
+extern void sched_core_enqueue(struct rq *rq, struct task_struct *p);
+extern void sched_core_dequeue(struct rq *rq, struct task_struct *p);
+
+extern void sched_core_get(void);
+extern void sched_core_put(void);
+
+extern unsigned long sched_core_alloc_cookie(void);
+extern void sched_core_put_cookie(unsigned long cookie);
+extern unsigned long sched_core_get_cookie(unsigned long cookie);
+extern unsigned long sched_core_update_cookie(struct task_struct *p, unsigned long cookie);
+
 #else /* !CONFIG_SCHED_CORE */
 
 static inline bool sched_core_enabled(struct rq *rq)
author maintainer
#69 lore raw

PATCH 17/19 sched: Inherit task cookie on fork()

In reply to Peter Zijlstra

Note that sched_core_fork() is called from under tasklist_lock, and not from sched_fork() earlier. This avoids a few races later.

---
 include/linux/sched.h     |    2 ++
 kernel/fork.c             |    3 +++
 kernel/sched/core_sched.c |    6 ++++++
 3 files changed, 11 insertions(+)
include/linux/sched.h +2 -0
@@ -2172,8 +2172,10 @@ const struct cpumask *sched_trace_rd_spa
 
 #ifdef CONFIG_SCHED_CORE
 extern void sched_core_free(struct task_struct *tsk);
+extern void sched_core_fork(struct task_struct *p);
 #else
 static inline void sched_core_free(struct task_struct *tsk) { }
+static inline void sched_core_fork(struct task_struct *p) { }
 #endif
 
 #endif
kernel/fork.c +3 -0
@@ -2249,6 +2249,8 @@ static __latent_entropy struct task_stru
 
 	klp_copy_process(p);
 
+	sched_core_fork(p);
+
 	spin_lock(&current->sighand->siglock);
 
 	/*
@@ -2336,6 +2338,7 @@ static __latent_entropy struct task_stru
 	return p;
 
 bad_fork_cancel_cgroup:
+	sched_core_free(p);
 	spin_unlock(&current->sighand->siglock);
 	write_unlock_irq(&tasklist_lock);
 	cgroup_cancel_fork(p, args);
kernel/sched/core_sched.c +6 -0
@@ -100,6 +100,12 @@ static unsigned long sched_core_clone_co
 	return cookie;
 }
 
+void sched_core_fork(struct task_struct *p)
+{
+	RB_CLEAR_NODE(&p->core_node);
+	p->core_cookie = sched_core_clone_cookie(current);
+}
+
 void sched_core_free(struct task_struct *p)
 {
 	sched_core_put_cookie(p->core_cookie);
#70 lore raw

Re: [PATCH 17/19] sched: Inherit task cookie on fork()

Hi Peter,

Quoted 46 lines · Peter Zijlstra

Note that sched_core_fork() is called from under tasklist_lock, and not from sched_fork() earlier. This avoids a few races later.

---
 include/linux/sched.h     |    2 ++
 kernel/fork.c             |    3 +++
 kernel/sched/core_sched.c |    6 ++++++
 3 files changed, 11 insertions(+)
include/linux/sched.h +2 -0
@@ -2172,8 +2172,10 @@ const struct cpumask *sched_trace_rd_spa
 
 #ifdef CONFIG_SCHED_CORE
 extern void sched_core_free(struct task_struct *tsk);
+extern void sched_core_fork(struct task_struct *p);
 #else
 static inline void sched_core_free(struct task_struct *tsk) { }
+static inline void sched_core_fork(struct task_struct *p) { }
 #endif
 
 #endif
kernel/fork.c +3 -0
@@ -2249,6 +2249,8 @@ static __latent_entropy struct task_stru
 
        klp_copy_process(p);
 
+       sched_core_fork(p);
+
        spin_lock(&current->sighand->siglock);
 
        /*
@@ -2336,6 +2338,7 @@ static __latent_entropy struct task_stru
        return p;
 
 bad_fork_cancel_cgroup:
+       sched_core_free(p);
        spin_unlock(&current->sighand->siglock);
        write_unlock_irq(&tasklist_lock);
        cgroup_cancel_fork(p, args);

kernel/sched/core_sched.c static unsigned long sched_core_clone_co · line 106

Quoted 7 lines
kernel/sched/core_sched.c +4 -0
@@ -100,6 +100,12 @@ static unsigned long sched_core_clone_co
        return cookie;
 }
 
+void sched_core_fork(struct task_struct *p)
+{
+       RB_CLEAR_NODE(&p->core_node);
+       p->core_cookie = sched_core_clone_cookie(current);

Does this make sense also for !CLONE_THREAD forks?

With earlier versions of core scheduling, we have done the following on ChromeOS. Basically, if it is a "thread clone", share the cookie since memory is shared within a process (same address space within a process). Otherwise, set the cookie to a new unique cookie so that the new process does not share core with parent initially (since their address space will be different).

Example Psedu-ocode in sched_fork():

        if (current->core_cookie && (clone_flags & CLONE_THREAD)) {
                    p->core_cookie = clone_cookie(current);
        } else {
                     p->core_cookie = create_new_cookie();
       }

In your version though, I don't see that it always clones the cookie whether it is a CLONE_THREAD clone or not. Is that correct? I feel that's a security issue.

-Joel

#71 lore raw

Re: [PATCH 17/19] sched: Inherit task cookie on fork()

Quoted 52 lines · Joel Fernandes

Hi Peter,

Quoted 49 lines · Peter Zijlstra

Note that sched_core_fork() is called from under tasklist_lock, and not from sched_fork() earlier. This avoids a few races later.

---
  include/linux/sched.h     |    2 ++
  kernel/fork.c             |    3 +++
  kernel/sched/core_sched.c |    6 ++++++
  3 files changed, 11 insertions(+)
include/linux/sched.h +2 -0
@@ -2172,8 +2172,10 @@ const struct cpumask *sched_trace_rd_spa
 
  #ifdef CONFIG_SCHED_CORE
  extern void sched_core_free(struct task_struct *tsk);
+extern void sched_core_fork(struct task_struct *p);
  #else
  static inline void sched_core_free(struct task_struct *tsk) { }
+static inline void sched_core_fork(struct task_struct *p) { }
  #endif
 
  #endif
kernel/fork.c +3 -0
@@ -2249,6 +2249,8 @@ static __latent_entropy struct task_stru
 
         klp_copy_process(p);
 
+       sched_core_fork(p);
+
         spin_lock(&current->sighand->siglock);
 
         /*
@@ -2336,6 +2338,7 @@ static __latent_entropy struct task_stru
         return p;
 
  bad_fork_cancel_cgroup:
+       sched_core_free(p);
         spin_unlock(&current->sighand->siglock);
         write_unlock_irq(&tasklist_lock);
         cgroup_cancel_fork(p, args);
kernel/sched/core_sched.c +0 -0
@@ -100,6 +100,12 @@ static unsigned long sched_core_clone_co
         return cookie;
  }
 
Quoted 6 lines

kernel/sched/core_sched.c static unsigned long sched_core_clone_co · line 106

Quoted 4 lines
kernel/sched/core_sched.c +4 -0
+void sched_core_fork(struct task_struct *p)
+{
+       RB_CLEAR_NODE(&p->core_node);
+       p->core_cookie = sched_core_clone_cookie(current);

Does this make sense also for !CLONE_THREAD forks?

Yes. Given the absence of a cgroup interface, fork inheritance (clone the cookie) is the best way to create shared cookie hierarchies. The security issue you mentioned was handled in my original code by setting a unique cookie on 'exec', but Peter took that out for the reason mentioned above. It was part of the "lets get this in compromise" effort.

-chrish

#72 lore raw

Re: [PATCH 17/19] sched: Inherit task cookie on fork()

Quoted 61 lines · Chris Hyser
Quoted 52 lines · Joel Fernandes

Hi Peter,

Quoted 49 lines · Peter Zijlstra

Note that sched_core_fork() is called from under tasklist_lock, and not from sched_fork() earlier. This avoids a few races later.

---
  include/linux/sched.h     |    2 ++
  kernel/fork.c             |    3 +++
  kernel/sched/core_sched.c |    6 ++++++
  3 files changed, 11 insertions(+)
include/linux/sched.h +2 -0
@@ -2172,8 +2172,10 @@ const struct cpumask *sched_trace_rd_spa
 
  #ifdef CONFIG_SCHED_CORE
  extern void sched_core_free(struct task_struct *tsk);
+extern void sched_core_fork(struct task_struct *p);
  #else
  static inline void sched_core_free(struct task_struct *tsk) { }
+static inline void sched_core_fork(struct task_struct *p) { }
  #endif
 
  #endif
kernel/fork.c +3 -0
@@ -2249,6 +2249,8 @@ static __latent_entropy struct task_stru
 
         klp_copy_process(p);
 
+       sched_core_fork(p);
+
         spin_lock(&current->sighand->siglock);
 
         /*
@@ -2336,6 +2338,7 @@ static __latent_entropy struct task_stru
         return p;
 
  bad_fork_cancel_cgroup:
+       sched_core_free(p);
         spin_unlock(&current->sighand->siglock);
         write_unlock_irq(&tasklist_lock);
         cgroup_cancel_fork(p, args);
kernel/sched/core_sched.c +0 -0
@@ -100,6 +100,12 @@ static unsigned long sched_core_clone_co
         return cookie;
  }
 
Quoted 6 lines

kernel/sched/core_sched.c static unsigned long sched_core_clone_co · line 106

Quoted 4 lines
kernel/sched/core_sched.c +4 -0
+void sched_core_fork(struct task_struct *p)
+{
+       RB_CLEAR_NODE(&p->core_node);
+       p->core_cookie = sched_core_clone_cookie(current);

Does this make sense also for !CLONE_THREAD forks?

Quoted 3 lines

Yes. Given the absence of a cgroup interface, fork inheritance (clone the cookie) is the best way to create shared cookie hierarchies. The security issue you mentioned was handled in my original code by setting a unique cookie on 'exec', but Peter took that out for the reason mentioned above. It was part of the "lets get this in compromise" effort.

Thanks for sharing the history of it. I guess one can argue that this policy is better to be hardcoded in userspace since core-scheduling can be used for non-security usecases as well. Maybe one could simply call the prctl(2) from userspace if they so desire, before calling exec() ?

- Joel

#73 lore raw

Re: [PATCH 17/19] sched: Inherit task cookie on fork()

Quoted 13 lines · Joel Fernandes
Quoted 10 lines · Chris Hyser
Quoted 6 lines

kernel/sched/core_sched.c static unsigned long sched_core_clone_co · line 106

Quoted 4 lines
kernel/sched/core_sched.c +4 -0
+void sched_core_fork(struct task_struct *p)
+{
+       RB_CLEAR_NODE(&p->core_node);
+       p->core_cookie = sched_core_clone_cookie(current);

Does this make sense also for !CLONE_THREAD forks?

Yes. Given the absence of a cgroup interface, fork inheritance (clone the cookie) is the best way to create shared cookie hierarchies. The security issue you mentioned was handled in my original code by setting a unique cookie on 'exec', but Peter took that out for the reason mentioned above. It was part of the "lets get this in compromise" effort.

Quoted 5 lines

Thanks for sharing the history of it. I guess one can argue that this policy is better to be hardcoded in userspace since core-scheduling can be used for non-security usecases as well. Maybe one could simply call the prctl(2) from userspace if they so desire, before calling exec() ?

I think the defining use case is a container's init. If the cookie is set for it by the container creator and without any other user code knowing about core_sched, every descendant spawned will have the same cookie and be in the same core_sched group much like the cgroup interface had provided. If we create a unique cookie in the kernel either on fork or exec, we are secure, but we will now have 1000's of core sched groups.

CLEAR was also removed (temporarily, I hope) because a core_sched knowledgeable program in the example core_sched container group should not be able to remove itself from _all_ core sched groups. It can modify it's cookie, but that is no different than the normal case.

Both of these beg for a kernel policy, but that discussion was TBD.

-chrish

author maintainer
#74 lore raw

Re: [PATCH 17/19] sched: Inherit task cookie on fork()

Quoted 10 lines · Chris Hyser
Quoted 9 lines · Joel Fernandes
Quoted 7 lines · Chris Hyser
Quoted 6 lines

kernel/sched/core_sched.c static unsigned long sched_core_clone_co · line 106

Quoted 4 lines
kernel/sched/core_sched.c +4 -0
+void sched_core_fork(struct task_struct *p)
+{
+       RB_CLEAR_NODE(&p->core_node);
+       p->core_cookie = sched_core_clone_cookie(current);

Does this make sense also for !CLONE_THREAD forks?

Quoted 6 lines
Quoted 6 lines
Quoted 6 lines

Yes. Given the absence of a cgroup interface, fork inheritance (clone the cookie) is the best way to create shared cookie hierarchies. The security issue you mentioned was handled in my original code by setting a unique cookie on 'exec', but Peter took that out for the reason mentioned above. It was part of the "lets get this in compromise" effort.

Right, not only that, given all this is moot when parent and child have the same PTRACE permissions, since if they do, they can inspect one another's innards anyway, exec()/fork() just fundamentally isn't a magical boundary we should care about.

The only special case there is SUID exec(), because in that case the actual credentials change and the PTRACE permissions do actually change.

I sorta had a patch to do that, but it's yuck because that cred change happens after the point of no return and we need an allocation for the new cookie. Now, we could rely on the fact that a task context allocation (GFP_KERNEL) for something as small as our cookie will never fail and hence we shouldn't be bothered by it, we should do the error path and yuck.

Quoted 13 lines
Quoted 5 lines

Thanks for sharing the history of it. I guess one can argue that this policy is better to be hardcoded in userspace since core-scheduling can be used for non-security usecases as well. Maybe one could simply call the prctl(2) from userspace if they so desire, before calling exec() ?

I think the defining use case is a container's init. If the cookie is set for it by the container creator and without any other user code knowing about core_sched, every descendant spawned will have the same cookie and be in the same core_sched group much like the cgroup interface had provided. If we create a unique cookie in the kernel either on fork or exec, we are secure, but we will now have 1000's of core sched groups.

Quoted 4 lines

CLEAR was also removed (temporarily, I hope) because a core_sched knowledgeable program in the example core_sched container group should not be able to remove itself from _all_ core sched groups. It can modify it's cookie, but that is no different than the normal case.

Note that much of clear is possible by using SHARE_FROM on your parent to reset the cookie.

Quoted 1 line

Both of these beg for a kernel policy, but that discussion was TBD.

Right, I need a Champion that actually cares about cgroups and has use-cases to go argue with TJ on this. I've proposed code that I think has sane semantics, but I'm not in a position to argue for it, given I think a world without cgroups is a better world :-)))

#75 lore raw

Re: [PATCH 17/19] sched: Inherit task cookie on fork()

Quoted 4 lines · Peter Zijlstra

Right, I need a Champion that actually cares about cgroups and has use-cases to go argue with TJ on this. I've proposed code that I think has sane semantics, but I'm not in a position to argue for it, given I think a world without cgroups is a better world :-)))

Not sure if Tejun has any thoughts on
http://lkml.kernel.org/r/CABk29NtahuW6UERvRdK5v8My_MfPsoESDKXUjGdvaQcHOJEMvg@mail.gmail.com.

We're looking at using the prctl interface with one of our main internal users of core scheduling. As an example, suppose we have a management process that wants to make tasks A and B share a cookie:
- Spawn a new thread m, which then does the following, and exits.
- PR_SCHED_CORE_CREATE for just its own PID
- PR_SCHED_CORE_SHARE_TO A
- PR_SCHED_CORE_SHARE_TO B

That seems to work ok; I'll follow up if there are any pain points that aren't easily addressed with the prctl interface.

#76 lore raw

Re: [PATCH 17/19] sched: Inherit task cookie on fork()

Quoted 18 lines · Josh Don
Quoted 4 lines · Peter Zijlstra

Right, I need a Champion that actually cares about cgroups and has use-cases to go argue with TJ on this. I've proposed code that I think has sane semantics, but I'm not in a position to argue for it, given I think a world without cgroups is a better world :-)))

Not sure if Tejun has any thoughts on
http://lkml.kernel.org/r/CABk29NtahuW6UERvRdK5v8My_MfPsoESDKXUjGdvaQcHOJEMvg@mail.gmail.com.

We're looking at using the prctl interface with one of our main internal users of core scheduling. As an example, suppose we have a management process that wants to make tasks A and B share a cookie:
- Spawn a new thread m, which then does the following, and exits.
- PR_SCHED_CORE_CREATE for just its own PID
- PR_SCHED_CORE_SHARE_TO A
- PR_SCHED_CORE_SHARE_TO B

Quoted 2 lines

That seems to work ok; I'll follow up if there are any pain points that aren't easily addressed with the prctl interface.

This is exactly what I'm doing to tag all the processes for qemu. I have as shell script that creates a cookie for $BASHPID and then SHARE_TO all the process and it works just great. :)

don

merge bot Applied
#77 lore raw

[tip: sched/core] sched: Inherit task cookie on fork()

In reply to Peter Zijlstra
The following commit has been merged into the sched/core branch of tip:

The following commit has been merged into the sched/core branch of tip:

Commit-ID:     85dd3f61203c5cfa72b308ff327b5fbf3fc1ce5e
Gitweb:        https://git.kernel.org/tip/85dd3f61203c5cfa72b308ff327b5fbf3fc1ce5e
Author:        Peter Zijlstra <[email protected]>
AuthorDate:    Mon, 29 Mar 2021 15:18:35 +02:00
Committer:     Peter Zijlstra <[email protected]>
CommitterDate: Wed, 12 May 2021 11:43:31 +02:00

sched: Inherit task cookie on fork()

Note that sched_core_fork() is called from under tasklist_lock, and not from sched_fork() earlier. This avoids a few races later.

---
 include/linux/sched.h     | 2 ++
 kernel/fork.c             | 3 +++
 kernel/sched/core_sched.c | 6 ++++++
 3 files changed, 11 insertions(+)
include/linux/sched.h +2 -0
index eab3f7c..fba47e5 100644
@@ -2181,8 +2181,10 @@ const struct cpumask *sched_trace_rd_span(struct root_domain *rd);
 
 #ifdef CONFIG_SCHED_CORE
 extern void sched_core_free(struct task_struct *tsk);
+extern void sched_core_fork(struct task_struct *p);
 #else
 static inline void sched_core_free(struct task_struct *tsk) { }
+static inline void sched_core_fork(struct task_struct *p) { }
 #endif
 
 #endif
kernel/fork.c +3 -0
index d16c60c..e7fd928 100644
@@ -2251,6 +2251,8 @@ static __latent_entropy struct task_struct *copy_process(
 
 	klp_copy_process(p);
 
+	sched_core_fork(p);
+
 	spin_lock(&current->sighand->siglock);
 
 	/*
@@ -2338,6 +2340,7 @@ static __latent_entropy struct task_struct *copy_process(
 	return p;
 
 bad_fork_cancel_cgroup:
+	sched_core_free(p);
 	spin_unlock(&current->sighand->siglock);
 	write_unlock_irq(&tasklist_lock);
 	cgroup_cancel_fork(p, args);
kernel/sched/core_sched.c +6 -0
index 8d0869a..dcbbeae 100644
@@ -103,6 +103,12 @@ static unsigned long sched_core_clone_cookie(struct task_struct *p)
 	return cookie;
 }
 
+void sched_core_fork(struct task_struct *p)
+{
+	RB_CLEAR_NODE(&p->core_node);
+	p->core_cookie = sched_core_clone_cookie(current);
+}
+
 void sched_core_free(struct task_struct *p)
 {
 	sched_core_put_cookie(p->core_cookie);
author maintainer
#78 lore raw

PATCH 18/19 sched: prctl() core-scheduling interface

In reply to Peter Zijlstra

From: Chris Hyser <[email protected]>

This patch provides support for setting and copying core scheduling 'task cookies' between threads (PID), processes (TGID), and process groups (PGID).

The value of core scheduling isn't that tasks don't share a core, 'nosmt' can do that. The value lies in exploiting all the sharing opportunities that exist to recover possible lost performance and that requires a degree of flexibility in the API.

Quoted 1 line

From a security perspective (and there are others), the thread,

process and process group distinction is an existent hierarchal categorization of tasks that reflects many of the security concerns about 'data sharing'. For example, protecting against cache-snooping by a thread that can just read the memory directly isn't all that useful.

With this in mind, subcommands to CREATE/SHARE (TO/FROM) provide a mechanism to create and share cookies. CREATE/SHARE_TO specify a target pid with enum pidtype used to specify the scope of the targeted tasks. For example, PIDTYPE_TGID will share the cookie with the process and all of it's threads as typically desired in a security scenario.

API:

prctl(PR_SCHED_CORE, PR_SCHED_CORE_GET, tgtpid, pidtype, &cookie) prctl(PR_SCHED_CORE, PR_SCHED_CORE_CREATE, tgtpid, pidtype, NULL) prctl(PR_SCHED_CORE, PR_SCHED_CORE_SHARE_TO, tgtpid, pidtype, NULL) prctl(PR_SCHED_CORE, PR_SCHED_CORE_SHARE_FROM, srcpid, pidtype, NULL)

where 'tgtpid/srcpid == 0' implies the current process and pidtype is kernel enum pid_type {PIDTYPE_PID, PIDTYPE_TGID, PIDTYPE_PGID, ...}.

For return values, EINVAL, ENOMEM are what they say. ESRCH means the tgtpid/srcpid was not found. EPERM indicates lack of PTRACE permission access to tgtpid/srcpid. ENODEV indicates your machines lacks SMT.

[peterz: complete rewrite]
---
 include/linux/sched.h            |    2 
 include/uapi/linux/prctl.h       |    8 ++
 kernel/sched/core_sched.c        |  114 +++++++++++++++++++++++++++++++++++++++
 kernel/sys.c                     |    5 +
 tools/include/uapi/linux/prctl.h |    8 ++
 5 files changed, 137 insertions(+)
include/linux/sched.h +2 -0
@@ -2173,6 +2173,8 @@ const struct cpumask *sched_trace_rd_spa
 #ifdef CONFIG_SCHED_CORE
 extern void sched_core_free(struct task_struct *tsk);
 extern void sched_core_fork(struct task_struct *p);
+extern int sched_core_share_pid(unsigned int cmd, pid_t pid, enum pid_type type,
+				unsigned long uaddr);
 #else
 static inline void sched_core_free(struct task_struct *tsk) { }
 static inline void sched_core_fork(struct task_struct *p) { }
include/uapi/linux/prctl.h +8 -0
@@ -255,4 +255,12 @@ struct prctl_mm_map {
 # define SYSCALL_DISPATCH_FILTER_ALLOW	0
 # define SYSCALL_DISPATCH_FILTER_BLOCK	1
 
+/* Request the scheduler to share a core */
+#define PR_SCHED_CORE			60
+# define PR_SCHED_CORE_GET		0
+# define PR_SCHED_CORE_CREATE		1 /* create unique core_sched cookie */
+# define PR_SCHED_CORE_SHARE_TO		2 /* push core_sched cookie to pid */
+# define PR_SCHED_CORE_SHARE_FROM	3 /* pull core_sched cookie to pid */
+# define PR_SCHED_CORE_MAX		4
+
 #endif /* _LINUX_PRCTL_H */
kernel/sched/core_sched.c +114 -0

122 lines not on this page · raw message

kernel/sys.c +5 -0
@@ -2534,6 +2534,11 @@ SYSCALL_DEFINE5(prctl, int, option, unsi
 		error = set_syscall_user_dispatch(arg2, arg3, arg4,
 						  (char __user *) arg5);
 		break;
+#ifdef CONFIG_SCHED_CORE
+	case PR_SCHED_CORE:
+		error = sched_core_share_pid(arg2, arg3, arg4, arg5);
+		break;
+#endif
 	default:
 		error = -EINVAL;
 		break;
tools/include/uapi/linux/prctl.h +8 -0
@@ -255,4 +255,12 @@ struct prctl_mm_map {
 # define SYSCALL_DISPATCH_FILTER_ALLOW	0
 # define SYSCALL_DISPATCH_FILTER_BLOCK	1
 
+/* Request the scheduler to share a core */
+#define PR_SCHED_CORE			60
+# define PR_SCHED_CORE_GET		0
+# define PR_SCHED_CORE_CREATE		1 /* create unique core_sched cookie */
+# define PR_SCHED_CORE_SHARE_TO		2 /* push core_sched cookie to pid */
+# define PR_SCHED_CORE_SHARE_FROM	3 /* pull core_sched cookie to pid */
+# define PR_SCHED_CORE_MAX		4
+
 #endif /* _LINUX_PRCTL_H */
merge bot Applied
#79 lore raw

[tip: sched/core] sched: prctl() core-scheduling interface

The following commit has been merged into the sched/core branch of tip:

The following commit has been merged into the sched/core branch of tip:

Commit-ID:     7ac592aa35a684ff1858fb9ec282886b9e3575ac
Gitweb:        https://git.kernel.org/tip/7ac592aa35a684ff1858fb9ec282886b9e3575ac
Author:        Chris Hyser <[email protected]>
AuthorDate:    Wed, 24 Mar 2021 17:40:15 -04:00
Committer:     Peter Zijlstra <[email protected]>
CommitterDate: Wed, 12 May 2021 11:43:31 +02:00

sched: prctl() core-scheduling interface

This patch provides support for setting and copying core scheduling 'task cookies' between threads (PID), processes (TGID), and process groups (PGID).

The value of core scheduling isn't that tasks don't share a core, 'nosmt' can do that. The value lies in exploiting all the sharing opportunities that exist to recover possible lost performance and that requires a degree of flexibility in the API.

Quoted 1 line

From a security perspective (and there are others), the thread,

process and process group distinction is an existent hierarchal categorization of tasks that reflects many of the security concerns about 'data sharing'. For example, protecting against cache-snooping by a thread that can just read the memory directly isn't all that useful.

With this in mind, subcommands to CREATE/SHARE (TO/FROM) provide a mechanism to create and share cookies. CREATE/SHARE_TO specify a target pid with enum pidtype used to specify the scope of the targeted tasks. For example, PIDTYPE_TGID will share the cookie with the process and all of it's threads as typically desired in a security scenario.

API:

prctl(PR_SCHED_CORE, PR_SCHED_CORE_GET, tgtpid, pidtype, &cookie) prctl(PR_SCHED_CORE, PR_SCHED_CORE_CREATE, tgtpid, pidtype, NULL) prctl(PR_SCHED_CORE, PR_SCHED_CORE_SHARE_TO, tgtpid, pidtype, NULL) prctl(PR_SCHED_CORE, PR_SCHED_CORE_SHARE_FROM, srcpid, pidtype, NULL)

where 'tgtpid/srcpid == 0' implies the current process and pidtype is kernel enum pid_type {PIDTYPE_PID, PIDTYPE_TGID, PIDTYPE_PGID, ...}.

For return values, EINVAL, ENOMEM are what they say. ESRCH means the tgtpid/srcpid was not found. EPERM indicates lack of PTRACE permission access to tgtpid/srcpid. ENODEV indicates your machines lacks SMT.

---
 include/linux/sched.h            |   2 +-
 include/uapi/linux/prctl.h       |   8 ++-
 kernel/sched/core_sched.c        | 114 ++++++++++++++++++++++++++++++-
 kernel/sys.c                     |   5 +-
 tools/include/uapi/linux/prctl.h |   8 ++-
 5 files changed, 137 insertions(+)
include/linux/sched.h +2 -0
index fba47e5..c7e7d50 100644
@@ -2182,6 +2182,8 @@ const struct cpumask *sched_trace_rd_span(struct root_domain *rd);
 #ifdef CONFIG_SCHED_CORE
 extern void sched_core_free(struct task_struct *tsk);
 extern void sched_core_fork(struct task_struct *p);
+extern int sched_core_share_pid(unsigned int cmd, pid_t pid, enum pid_type type,
+				unsigned long uaddr);
 #else
 static inline void sched_core_free(struct task_struct *tsk) { }
 static inline void sched_core_fork(struct task_struct *p) { }
include/uapi/linux/prctl.h +8 -0
index 18a9f59..967d9c5 100644
@@ -259,4 +259,12 @@ struct prctl_mm_map {
 #define PR_PAC_SET_ENABLED_KEYS		60
 #define PR_PAC_GET_ENABLED_KEYS		61
 
+/* Request the scheduler to share a core */
+#define PR_SCHED_CORE			62
+# define PR_SCHED_CORE_GET		0
+# define PR_SCHED_CORE_CREATE		1 /* create unique core_sched cookie */
+# define PR_SCHED_CORE_SHARE_TO		2 /* push core_sched cookie to pid */
+# define PR_SCHED_CORE_SHARE_FROM	3 /* pull core_sched cookie to pid */
+# define PR_SCHED_CORE_MAX		4
+
 #endif /* _LINUX_PRCTL_H */
kernel/sched/core_sched.c +114 -0

122 lines not on this page · raw message

kernel/sys.c +5 -0
index 3a583a2..9de46a4 100644
@@ -2550,6 +2550,11 @@ SYSCALL_DEFINE5(prctl, int, option, unsigned long, arg2, unsigned long, arg3,
 		error = set_syscall_user_dispatch(arg2, arg3, arg4,
 						  (char __user *) arg5);
 		break;
+#ifdef CONFIG_SCHED_CORE
+	case PR_SCHED_CORE:
+		error = sched_core_share_pid(arg2, arg3, arg4, arg5);
+		break;
+#endif
 	default:
 		error = -EINVAL;
 		break;
tools/include/uapi/linux/prctl.h +8 -0
index 18a9f59..967d9c5 100644
@@ -259,4 +259,12 @@ struct prctl_mm_map {
 #define PR_PAC_SET_ENABLED_KEYS		60
 #define PR_PAC_GET_ENABLED_KEYS		61
 
+/* Request the scheduler to share a core */
+#define PR_SCHED_CORE			62
+# define PR_SCHED_CORE_GET		0
+# define PR_SCHED_CORE_CREATE		1 /* create unique core_sched cookie */
+# define PR_SCHED_CORE_SHARE_TO		2 /* push core_sched cookie to pid */
+# define PR_SCHED_CORE_SHARE_FROM	3 /* pull core_sched cookie to pid */
+# define PR_SCHED_CORE_MAX		4
+
 #endif /* _LINUX_PRCTL_H */
#80 lore raw

Re: [PATCH 18/19] sched: prctl() core-scheduling interface

In reply to Peter Zijlstra
Quoted 6 lines · Peter Zijlstra

From: Chris Hyser <[email protected]>

This patch provides support for setting and copying core scheduling 'task cookies' between threads (PID), processes (TGID), and process groups (PGID).

[snip]

Internally, we have lots of trusted processes that don't have a security need for coresched cookies. However, these processes could still decide to create cookies for themselves, which will degrade machine capacity and performance for other jobs on the machine.

Any thoughts on whether it would be desirable to have the ability to restrict use of SCHED_CORE_CREATE? Perhaps a new SCHED_CORE capability would be appropriate?

- Josh

#81 lore raw

Re: [PATCH 18/19] sched: prctl() core-scheduling interface

Quoted 11 lines · Josh Don
Quoted 6 lines · Peter Zijlstra

From: Chris Hyser <[email protected]>

This patch provides support for setting and copying core scheduling 'task cookies' between threads (PID), processes (TGID), and process groups (PGID).

[snip]

Quoted 8 lines

Internally, we have lots of trusted processes that don't have a security need for coresched cookies. However, these processes could still decide to create cookies for themselves, which will degrade machine capacity and performance for other jobs on the machine.

Any thoughts on whether it would be desirable to have the ability to restrict use of SCHED_CORE_CREATE? Perhaps a new SCHED_CORE capability would be appropriate?

Hi,
Maybe a capability may not work because then other users who don't care for the issue you mention will be required to manage/assign the capability as well?

How about you use seccomp to filter the prctl based on the PID, and CREATE command?

-Joel

#82 lore raw

Re: [PATCH 18/19] sched: prctl() core-scheduling interface

In reply to Peter Zijlstra
Quoted 9 lines · Peter Zijlstra

API:

prctl(PR_SCHED_CORE, PR_SCHED_CORE_GET, tgtpid, pidtype, &cookie) prctl(PR_SCHED_CORE, PR_SCHED_CORE_CREATE, tgtpid, pidtype, NULL) prctl(PR_SCHED_CORE, PR_SCHED_CORE_SHARE_TO, tgtpid, pidtype, NULL) prctl(PR_SCHED_CORE, PR_SCHED_CORE_SHARE_FROM, srcpid, pidtype, NULL)

where 'tgtpid/srcpid == 0' implies the current process and pidtype is kernel enum pid_type {PIDTYPE_PID, PIDTYPE_TGID, PIDTYPE_PGID, ...}.

It means that enum pid_tipe has to be a part of UAPI now. Would you like to address it, or rather I'd send a patch?

author maintainer
#83 lore raw

Re: [PATCH 18/19] sched: prctl() core-scheduling interface

Quoted 8 lines · Eugene Syromiatnikov
Quoted 7 lines · Peter Zijlstra

API:

prctl(PR_SCHED_CORE, PR_SCHED_CORE_GET, tgtpid, pidtype, &cookie) prctl(PR_SCHED_CORE, PR_SCHED_CORE_CREATE, tgtpid, pidtype, NULL) prctl(PR_SCHED_CORE, PR_SCHED_CORE_SHARE_TO, tgtpid, pidtype, NULL) prctl(PR_SCHED_CORE, PR_SCHED_CORE_SHARE_FROM, srcpid, pidtype, NULL)

Quoted 5 lines
Quoted 2 lines

where 'tgtpid/srcpid == 0' implies the current process and pidtype is kernel enum pid_type {PIDTYPE_PID, PIDTYPE_TGID, PIDTYPE_PGID, ...}.

It means that enum pid_tipe has to be a part of UAPI now. Would you like to address it, or rather I'd send a patch?

Please send a patch; I'm more sparse than usual atm.

#84 lore raw

Re: [PATCH 18/19] sched: prctl() core-scheduling interface

In reply to Peter Zijlstra
Quoted 5 lines · Peter Zijlstra

From: Chris Hyser <[email protected]>

This patch provides support for setting and copying core scheduling 'task cookies' between threads (PID), processes (TGID), and process groups (PGID).

Hello.

It seems that there is some issue within the scheduler code that can be triggered via this interface:

    # gcc -std=gnu99 -Wextra -Werror prctl-sched-core-oops-repro.c -o prctl-sched-core-oops-repro
    # ../src/strace -fvq -eprctl,clone,setsid ./prctl-sched-core-oops-repro
    clone(child_stack=NULL, flags=CLONE_CHILD_CLEARTID|CLONE_CHILD_SETTID|SIGCHLD, child_tidptr=0x7f271f875890) = 239820
    [pid 239820] setsid()                   = 239820
    [pid 239820] +++ exited with 0 +++
    --- SIGCHLD {si_signo=SIGCHLD, si_code=CLD_EXITED, si_pid=239820, si_uid=0, si_status=0, si_utime=0, si_stime=0} ---
    Iteration 0 status: 0
    prctl(PR_SCHED_CORE, PR_SCHED_CORE_CREATE, 239816, 0x2 /* PIDTYPE_PGID */, NULL) = 0
    clone(child_stack=NULL, flags=CLONE_CHILD_CLEARTID|CLONE_CHILD_SETTID|SIGCHLD, child_tidptr=0x7f271f875890) = 239821
    [pid 239821] setsid()                   = ?
    [pid 239821] +++ killed by SIGKILL +++
    --- SIGCHLD {si_signo=SIGCHLD, si_code=CLD_KILLED, si_pid=239821, si_uid=0, si_status=SIGKILL, si_utime=0, si_stime=0} ---
    Iteration 1 status: 0x9
    prctl(PR_SCHED_CORE, PR_SCHED_CORE_CREATE, 239816, 0x2 /* PIDTYPE_PGID */, NULL) = 0
    +++ exited with 0 +++

kmsg indicates that a NULL pointer dereference has occurred:

    [76195.611570] BUG: kernel NULL pointer dereference, address: 0000000000000000
    ...
    [76195.621771] RIP: 0010:do_raw_spin_trylock+0x5/0x40
    ...
    [76195.640144] Call Trace:
    [76195.640706]  _raw_spin_lock_nested+0x37/0x80
    [76195.641645]  ? raw_spin_rq_lock_nested+0x4b/0x80
    [76195.642693]  raw_spin_rq_lock_nested+0x4b/0x80
    [76195.643669]  online_fair_sched_group+0x39/0x240
    [76195.644663]  sched_autogroup_create_attach+0x9d/0x170
    [76195.645765]  ksys_setsid+0xe6/0x110
    [76195.646533]  __do_sys_setsid+0xa/0x10
    [76195.647358]  do_syscall_64+0x3b/0x90
    [76195.648219]  entry_SYSCALL_64_after_hwframe+0x44/0xae

The full kmsg excerpt and the reproducer code are attached.

There's also additional "BUG: sleeping function called from invalid context at include/linux/percpu-rwsem.h:49" message is produced (see the full log in the attached file "prctl-sched-core-oops-bug-dmesg.log") when the full test case[1] is run, but I haven't been successful so far in producing a minimal reproduced for it.

[1] https://github.com/strace/strace/commit/a90a5a56d2b76ba3ebd417472a02f40d3d6599d8
Run with ./bootstrap && ./configure CFLAGS='-g -Og' --enable-gcc-Werror && make check TESTS=prctl-sched-core--pidns-translation.gen

#include <stdio.h>
#include <linux/prctl.h>
#include <sys/prctl.h>
#include <sys/wait.h>
#include <unistd.h>

int main()

{
	int pid;
	int pgid = getpgid(0);

	for (size_t i = 0; i < 2; i++) {
		if ((pid = fork())) {
			int status;

			wait(&status);
			printf("Iteration %d status: %#x\n", i, status);
		} else {
			setsid();

			return 0;
		}

		prctl(PR_SCHED_CORE, PR_SCHED_CORE_CREATE, pgid, 0x2 /* PIDTYPE_PGID */, NULL);
	}

	return 0;
}
[76195.611570] BUG: kernel NULL pointer dereference, address: 0000000000000000
[76195.613059] #PF: supervisor read access in kernel mode
[76195.614174] #PF: error_code(0x0000) - not-present page
[76195.615329] PGD 800000005f27e067 P4D 800000005f27e067 PUD 3f7a3067 PMD 0 
[76195.616801] Oops: 0000 [#67] SMP PTI
[76195.617586] CPU: 2 PID: 239821 Comm: prctl-sched-cor Tainted: G      D W        --------- ---  5.14.0-0.rc5.20210813gitf8e6dfc64f61.46.fc36.x86_64 #1
[76195.620374] Hardware name: HP ProLiant BL480c G1, BIOS I14 10/04/2007
[76195.621771] RIP: 0010:do_raw_spin_trylock+0x5/0x40
[76195.622821] Code: c6 a4 12 5f 9f 48 89 ef e8 c8 fe ff ff eb a9 89 c6 48 89 ef e8 0c f5 ff ff 66 90 eb a9 0f 1f 84 00 00 00 00 00 0f 1f 44 00 00 <8b> 07 85 c0 75 28 ba 01 00 00 00 f0 0f b1 17 75 1d 65 8b 05 fb 98
[76195.626797] RSP: 0018:ffffa366014abe58 EFLAGS: 00010086
[76195.627936] RAX: 0000000000000001 RBX: 0000000000000004 RCX: 0000000000000000
[76195.629470] RDX: 0000000000000000 RSI: 0000000000000000 RDI: 0000000000000000
[76195.631048] RBP: 0000000000000000 R08: 0000000000000001 R09: 0000000000000000
[76195.632585] R10: 0000000000000000 R11: ffff98292b21ad48 R12: 0000000000000018
[76195.634078] R13: 0000000000000000 R14: ffff98292b7ef940 R15: ffff982813938e00
[76195.635621] FS:  00007f271f8755c0(0000) GS:ffff98292b200000(0000) knlGS:0000000000000000
[76195.637354] CS:  0010 DS: 0000 ES: 0000 CR0: 0000000080050033
[76195.638606] CR2: 0000000000000000 CR3: 000000000fdd0000 CR4: 00000000000006e0
[76195.640144] Call Trace:
[76195.640706]  _raw_spin_lock_nested+0x37/0x80
[76195.641645]  ? raw_spin_rq_lock_nested+0x4b/0x80
[76195.642693]  raw_spin_rq_lock_nested+0x4b/0x80
[76195.643669]  online_fair_sched_group+0x39/0x240
[76195.644663]  sched_autogroup_create_attach+0x9d/0x170
[76195.645765]  ksys_setsid+0xe6/0x110
[76195.646533]  __do_sys_setsid+0xa/0x10
[76195.647358]  do_syscall_64+0x3b/0x90
[76195.648219]  entry_SYSCALL_64_after_hwframe+0x44/0xae
[76195.649401] RIP: 0033:0x7f271f7496cb
[76195.650241] Code: 73 01 c3 48 8b 0d 5d a7 11 00 f7 d8 64 89 01 48 83 c8 ff c3 66 2e 0f 1f 84 00 00 00 00 00 90 f3 0f 1e fa b8 70 00 00 00 0f 05 <48> 3d 01 f0 ff ff 73 01 c3 48 8b 0d 2d a7 11 00 f7 d8 64 89 01 48
[76195.654522] RSP: 002b:00007ffed2ea9888 EFLAGS: 00000206 ORIG_RAX: 0000000000000070
[76195.656275] RAX: ffffffffffffffda RBX: 0000000000000000 RCX: 00007f271f7496cb
[76195.657934] RDX: 0000000000000001 RSI: 0000000000000000 RDI: 0000000000000001
[76195.659566] RBP: 00007ffed2ea98b0 R08: 0000000000000000 R09: 00007f271f8755c0
[76195.661227] R10: 00007f271f67ca60 R11: 0000000000000206 R12: 00007ffed2ea99e8
[76195.662869] R13: 0000000000401176 R14: 00007f271f8adc00 R15: 0000000000403e18
[76195.664510] Modules linked in: rfkill iTCO_wdt intel_pmc_bxt iTCO_vendor_support kvm_intel gpio_ich kvm snd_pcsp snd_pcm snd_timer ipmi_ssif irqbypass snd soundcore lpc_ich acpi_ipmi e1000e tg3 hpilo ipmi_si bnx2 i5000_edac ipmi_devintf i5k_amb ipmi_msghandler fuse zram ip_tables xfs radeon i2c_algo_bit drm_ttm_helper ttm drm_kms_helper cec drm hpsa serio_raw hpwdt scsi_transport_sas
[76195.672185] CR2: 0000000000000000
[76195.672973] ---[ end trace ab4b5b9489802ac5 ]---
[76195.674050] RIP: 0010:__lock_acquire+0x5c1/0x1e00
[76195.675146] Code: 00 00 83 f8 2f 0f 87 59 05 00 00 3b 05 cc 46 f6 02 41 bd 01 00 00 00 0f 86 03 01 00 00 89 05 ba 46 f6 02 e9 f8 00 00 00 31 c9 <48> 81 3f 20 8e cc a0 41 0f 45 c8 83 fe 01 0f 87 6f fa ff ff 89 f0
[76195.679386] RSP: 0018:ffffa36602d5fd40 EFLAGS: 00010046
[76195.680606] RAX: 0000000000000046 RBX: 0000000000000001 RCX: 0000000000000000
[76195.682239] RDX: 0000000000000000 RSI: 0000000000000000 RDI: 0000000000000018
[76195.683881] RBP: 0000000000000018 R08: 0000000000000001 R09: 0000000000000001
[76195.685554] R10: 0000000000000001 R11: 0000000000000000 R12: ffff98285fad33c0
[76195.687197] R13: 0000000000000000 R14: 0000000000000000 R15: 0000000000000000
[76195.688840] FS:  00007f271f8755c0(0000) GS:ffff98292b200000(0000) knlGS:0000000000000000
[76195.690690] CS:  0010 DS: 0000 ES: 0000 CR0: 0000000080050033
[76195.692034] CR2: 0000000000000000 CR3: 000000000fdd0000 CR4: 00000000000006e0
[76195.693677] note: prctl-sched-cor[239821] exited with preempt_count 2
[31148.057893] BUG: kernel NULL pointer dereference, address: 0000000000000000
[31148.059390] #PF: supervisor read access in kernel mode
[31148.060530] #PF: error_code(0x0000) - not-present page
[31148.061601] PGD 800000005f693067 P4D 800000005f693067 PUD 114fe067 PMD 0 
[31148.063082] Oops: 0000 [#21] SMP PTI
[31148.063880] CPU: 0 PID: 214019 Comm: prctl-sched-cor Tainted: G      D W        --------- ---  5.14.0-0.rc5.20210813gitf8e6dfc64f61.46.fc36.x86_64 #1
[31148.067019] Hardware name: HP ProLiant BL480c G1, BIOS I14 10/04/2007
[31148.068536] RIP: 0010:do_raw_spin_trylock+0x5/0x40
[31148.069629] Code: c6 a4 12 5f 9f 48 89 ef e8 c8 fe ff ff eb a9 89 c6 48 89 ef e8 0c f5 ff ff 66 90 eb a9 0f 1f 84 00 00 00 00 00 0f 1f 44 00 00 <8b> 07 85 c0 75 28 ba 01 00 00 00 f0 0f b1 17 75 1d 65 8b 05 fb 98
[31148.073601] RSP: 0018:ffffa366030ffe58 EFLAGS: 00010086
[31148.074740] RAX: 0000000000000001 RBX: 0000000000000004 RCX: 0000000000000000
[31148.076235] RDX: 0000000000000000 RSI: 0000000000000000 RDI: 0000000000000000
[31148.077770] RBP: 0000000000000000 R08: 0000000000000001 R09: 0000000000000000
[31148.079264] R10: 0000000000000000 R11: 000000000121ee36 R12: 0000000000000018
[31148.080752] R13: 0000000000000000 R14: ffff98292b7ef940 R15: ffff98280d75da00
[31148.082295] FS:  00007f1aefadc600(0000) GS:ffff98292ae00000(0000) knlGS:0000000000000000
[31148.084029] CS:  0010 DS: 0000 ES: 0000 CR0: 0000000080050033
[31148.085282] CR2: 0000000000000000 CR3: 00000000013be000 CR4: 00000000000006f0
[31148.086770] Call Trace:
[31148.087286]  _raw_spin_lock_nested+0x37/0x80
[31148.088212]  ? raw_spin_rq_lock_nested+0x4b/0x80
[31148.089180]  raw_spin_rq_lock_nested+0x4b/0x80
[31148.090189]  online_fair_sched_group+0x39/0x240
[31148.091185]  sched_autogroup_create_attach+0x9d/0x170
[31148.092310]  ksys_setsid+0xe6/0x110
[31148.093073]  __do_sys_setsid+0xa/0x10
[31148.093881]  do_syscall_64+0x3b/0x90
[31148.094678]  entry_SYSCALL_64_after_hwframe+0x44/0xae
[31148.095783] RIP: 0033:0x7f1aef9b06cb
[31148.096579] Code: 73 01 c3 48 8b 0d 5d a7 11 00 f7 d8 64 89 01 48 83 c8 ff c3 66 2e 0f 1f 84 00 00 00 00 00 90 f3 0f 1e fa b8 70 00 00 00 0f 05 <48> 3d 01 f0 ff ff 73 01 c3 48 8b 0d 2d a7 11 00 f7 d8 64 89 01 48
[31148.100758] RSP: 002b:00007fff4b7eef78 EFLAGS: 00000206 ORIG_RAX: 0000000000000070
[31148.102535] RAX: ffffffffffffffda RBX: 0000000000000000 RCX: 00007f1aef9b06cb
[31148.104188] RDX: 0000000000000010 RSI: 0000000000406230 RDI: 0000000000000003
[31148.105859] RBP: 00000000ffffffff R08: 0000000000000000 R09: 00007f1aefadc600
[31148.107523] R10: 00007f1aef8e3a60 R11: 0000000000000206 R12: 0000000000000001
[31148.109155] R13: 0000000000401326 R14: 00007f1aefb14c00 R15: 0000000000405e18
[31148.110819] Modules linked in: rfkill iTCO_wdt intel_pmc_bxt iTCO_vendor_support kvm_intel gpio_ich kvm snd_pcsp snd_pcm snd_timer ipmi_ssif irqbypass snd soundcore lpc_ich acpi_ipmi e1000e tg3 hpilo ipmi_si bnx2 i5000_edac ipmi_devintf i5k_amb ipmi_msghandler fuse zram ip_tables xfs radeon i2c_algo_bit drm_ttm_helper ttm drm_kms_helper cec drm hpsa serio_raw hpwdt scsi_transport_sas
[31148.118454] CR2: 0000000000000000
[31148.119246] ---[ end trace ab4b5b9489802a97 ]---
[31148.120379] RIP: 0010:__lock_acquire+0x5c1/0x1e00
[31148.121475] Code: 00 00 83 f8 2f 0f 87 59 05 00 00 3b 05 cc 46 f6 02 41 bd 01 00 00 00 0f 86 03 01 00 00 89 05 ba 46 f6 02 e9 f8 00 00 00 31 c9 <48> 81 3f 20 8e cc a0 41 0f 45 c8 83 fe 01 0f 87 6f fa ff ff 89 f0
[31148.125694] RSP: 0018:ffffa36602d5fd40 EFLAGS: 00010046
[31148.126907] RAX: 0000000000000046 RBX: 0000000000000001 RCX: 0000000000000000
[31148.128546] RDX: 0000000000000000 RSI: 0000000000000000 RDI: 0000000000000018
[31148.130195] RBP: 0000000000000018 R08: 0000000000000001 R09: 0000000000000001
[31148.131853] R10: 0000000000000001 R11: 0000000000000000 R12: ffff98285fad33c0
[31148.133493] R13: 0000000000000000 R14: 0000000000000000 R15: 0000000000000000
[31148.135168] FS:  00007f1aefadc600(0000) GS:ffff98292ae00000(0000) knlGS:0000000000000000
[31148.137018] CS:  0010 DS: 0000 ES: 0000 CR0: 0000000080050033
[31148.138352] CR2: 0000000000000000 CR3: 00000000013be000 CR4: 00000000000006f0
[31148.139984] note: prctl-sched-cor[214019] exited with preempt_count 2
[31148.141470] BUG: sleeping function called from invalid context at include/linux/percpu-rwsem.h:49
[31148.143494] in_atomic(): 0, irqs_disabled(): 1, non_block: 0, pid: 214019, name: prctl-sched-cor
[31148.145497] INFO: lockdep is turned off.
[31148.146422] irq event stamp: 0
[31148.147195] hardirqs last  enabled at (0): [<0000000000000000>] 0x0
[31148.148640] hardirqs last disabled at (0): [<ffffffff9e0e4e9f>] copy_process+0x7cf/0x1fb0
[31148.150511] softirqs last  enabled at (0): [<ffffffff9e0e4e9f>] copy_process+0x7cf/0x1fb0
[31148.152384] softirqs last disabled at (0): [<0000000000000000>] 0x0
[31148.153825] CPU: 0 PID: 214019 Comm: prctl-sched-cor Tainted: G      D W        --------- ---  5.14.0-0.rc5.20210813gitf8e6dfc64f61.46.fc36.x86_64 #1
[31148.156852] Hardware name: HP ProLiant BL480c G1, BIOS I14 10/04/2007
[31148.158338] Call Trace:
[31148.158945]  dump_stack_lvl+0x57/0x72
[31148.159859]  ___might_sleep.cold+0xb6/0xc6
[31148.160822]  exit_signals+0x1c/0x2d0
[31148.161668]  do_exit+0xbf/0xc10
[31148.162419]  ? ksys_setsid+0xe6/0x110
[31148.163228]  rewind_stack_do_exit+0x17/0x20
[31148.164203] RIP: 0033:0x7f1aef9b06cb
[31148.165043] Code: 73 01 c3 48 8b 0d 5d a7 11 00 f7 d8 64 89 01 48 83 c8 ff c3 66 2e 0f 1f 84 00 00 00 00 00 90 f3 0f 1e fa b8 70 00 00 00 0f 05 <48> 3d 01 f0 ff ff 73 01 c3 48 8b 0d 2d a7 11 00 f7 d8 64 89 01 48
[31148.169275] RSP: 002b:00007fff4b7eef78 EFLAGS: 00000206 ORIG_RAX: 0000000000000070
[31148.171019] RAX: ffffffffffffffda RBX: 0000000000000000 RCX: 00007f1aef9b06cb
[31148.172700] RDX: 0000000000000010 RSI: 0000000000406230 RDI: 0000000000000003
[31148.174343] RBP: 00000000ffffffff R08: 0000000000000000 R09: 00007f1aefadc600
[31148.175994] R10: 00007f1aef8e3a60 R11: 0000000000000206 R12: 0000000000000001
[31148.177638] R13: 0000000000401326 R14: 00007f1aefb14c00 R15: 0000000000405e18
author maintainer
#85 lore raw

Re: [PATCH 18/19] sched: prctl() core-scheduling interface

Quoted 15 lines · Eugene Syromiatnikov
[76195.611570] BUG: kernel NULL pointer dereference, address: 0000000000000000
[76195.613059] #PF: supervisor read access in kernel mode
[76195.614174] #PF: error_code(0x0000) - not-present page
[76195.615329] PGD 800000005f27e067 P4D 800000005f27e067 PUD 3f7a3067 PMD 0 
[76195.616801] Oops: 0000 [#67] SMP PTI
[76195.617586] CPU: 2 PID: 239821 Comm: prctl-sched-cor Tainted: G      D W        --------- ---  5.14.0-0.rc5.20210813gitf8e6dfc64f61.46.fc36.x86_64 #1
[76195.620374] Hardware name: HP ProLiant BL480c G1, BIOS I14 10/04/2007
[76195.621771] RIP: 0010:do_raw_spin_trylock+0x5/0x40
[76195.622821] Code: c6 a4 12 5f 9f 48 89 ef e8 c8 fe ff ff eb a9 89 c6 48 89 ef e8 0c f5 ff ff 66 90 eb a9 0f 1f 84 00 00 00 00 00 0f 1f 44 00 00 <8b> 07 85 c0 75 28 ba 01 00 00 00 f0 0f b1 17 75 1d 65 8b 05 fb 98
[76195.626797] RSP: 0018:ffffa366014abe58 EFLAGS: 00010086
[76195.627936] RAX: 0000000000000001 RBX: 0000000000000004 RCX: 0000000000000000
[76195.629470] RDX: 0000000000000000 RSI: 0000000000000000 RDI: 0000000000000000
[76195.631048] RBP: 0000000000000000 R08: 0000000000000001 R09: 0000000000000000
[76195.632585] R10: 0000000000000000 R11: ffff98292b21ad48 R12: 0000000000000018
[76195.634078] R13: 0000000000000000 R14: ffff98292b7ef940 R15: ffff982813938e00
Quoted 8 lines
[76195.635621] FS:  00007f271f8755c0(0000) GS:ffff98292b200000(0000) knlGS:0000000000000000
[76195.637354] CS:  0010 DS: 0000 ES: 0000 CR0: 0000000080050033
[76195.638606] CR2: 0000000000000000 CR3: 000000000fdd0000 CR4: 00000000000006e0
[76195.640144] Call Trace:
[76195.640706]  _raw_spin_lock_nested+0x37/0x80
[76195.641645]  ? raw_spin_rq_lock_nested+0x4b/0x80
[76195.642693]  raw_spin_rq_lock_nested+0x4b/0x80
[76195.643669]  online_fair_sched_group+0x39/0x240

Urgh... lemme guess, your HP BIOS is funny and reports more possible CPUs than you actually have resulting in cpu_possible_mask != cpu_online_mask. Alternatively, you booted with nr_cpus= or something daft like that.

That code does for_each_possible_cpus(i) { rq_lock_irq(cpu_rq(i)); }, which, because of core-sched, needs rq->core set-up, but because these CPUs have never been online, that's not done and *BOOM*.

Or something like that.. I'll try and have a look tomorrow, I'm in dire need of sleep.

#86 lore raw

Re: [PATCH 18/19] sched: prctl() core-scheduling interface

Quoted 4 lines · Peter Zijlstra

Urgh... lemme guess, your HP BIOS is funny and reports more possible CPUs than you actually have resulting in cpu_possible_mask != cpu_online_mask. Alternatively, you booted with nr_cpus= or something daft like that.

Yep, it seems to be the case:

    # cat /sys/devices/system/cpu/possible
    0-7
    # cat /sys/devices/system/cpu/online
    0-3
Quoted 8 lines

That code does for_each_possible_cpus(i) { rq_lock_irq(cpu_rq(i)); }, which, because of core-sched, needs rq->core set-up, but because these CPUs have never been online, that's not done and *BOOM*.

Or something like that.. I'll try and have a look tomorrow, I'm in dire need of sleep.

author maintainer
#87 lore raw

PATCH 1/1 sched: Fix Core-wide rq->lock for uninitialized CPUs

Quoted 6 lines · Eugene Syromiatnikov
Quoted 4 lines · Peter Zijlstra

Urgh... lemme guess, your HP BIOS is funny and reports more possible CPUs than you actually have resulting in cpu_possible_mask != cpu_online_mask. Alternatively, you booted with nr_cpus= or something daft like that.

Quoted 7 lines

Yep, it seems to be the case:

    # cat /sys/devices/system/cpu/possible
    0-7
    # cat /sys/devices/system/cpu/online
    0-3

I think the below should work... can you please verify?

---
Subject: sched: Fix Core-wide rq->lock for uninitialized CPUs

Eugene tripped over the case where rq_lock(), as called in a for_each_possible_cpu() loop came apart because rq->core hadn't been setup yet.

This is a somewhat unusual, but valid case.

Rework things such that rq->core is initialized to point at itself. IOW initialize each CPU as a single threaded Core. CPU online will then join the new CPU (thread) to an existing Core where needed.

For completeness sake, have CPU offline fully undo the state so as to not presume the topology will match the next time it comes online.

Fixes 9edeaea1bc45 ("sched: Core-wide rq->lock")
---
 kernel/sched/core.c  | 143 +++++++++++++++++++++++++++++++++++++++++----------
 kernel/sched/sched.h |   2 +-
 2 files changed, 118 insertions(+), 27 deletions(-)
kernel/sched/core.c +117 -26
index 0f14e09a4f99..21d633971fcf 100644
@@ -237,9 +237,30 @@ static DEFINE_MUTEX(sched_core_mutex);
 static atomic_t sched_core_count;
 static struct cpumask sched_core_mask;
 
+static void sched_core_lock(int cpu, unsigned long *flags)
+{
+	const struct cpumask *smt_mask = cpu_smt_mask(cpu);
+	int t, i = 0;
+
+	local_irq_save(*flags);
+	for_each_cpu(t, smt_mask)
+		raw_spin_lock_nested(&cpu_rq(t)->__lock, i++);
+}
+
+static void sched_core_unlock(int cpu, unsigned long *flags)
+{
+	const struct cpumask *smt_mask = cpu_smt_mask(cpu);
+	int t;
+
+	for_each_cpu(t, smt_mask)
+		raw_spin_unlock(&cpu_rq(t)->__lock);
+	local_irq_restore(*flags);
+}
+
 static void __sched_core_flip(bool enabled)
 {
-	int cpu, t, i;
+	unsigned long flags;
+	int cpu, t;
 
 	cpus_read_lock();
 
@@ -250,19 +271,12 @@ static void __sched_core_flip(bool enabled)
 	for_each_cpu(cpu, &sched_core_mask) {
 		const struct cpumask *smt_mask = cpu_smt_mask(cpu);
 
-		i = 0;
-		local_irq_disable();
-		for_each_cpu(t, smt_mask) {
-			/* supports up to SMT8 */
-			raw_spin_lock_nested(&cpu_rq(t)->__lock, i++);
-		}
+		sched_core_lock(cpu, &flags);
 
 		for_each_cpu(t, smt_mask)
 			cpu_rq(t)->core_enabled = enabled;
 
-		for_each_cpu(t, smt_mask)
-			raw_spin_unlock(&cpu_rq(t)->__lock);
-		local_irq_enable();
+		sched_core_unlock(cpu, &flags);
 
 		cpumask_andnot(&sched_core_mask, &sched_core_mask, smt_mask);
 	}
@@ -5979,35 +5993,109 @@ void queue_core_balance(struct rq *rq)
 	queue_balance_callback(rq, &per_cpu(core_balance_head, rq->cpu), sched_core_balance);
 }
 
-static inline void sched_core_cpu_starting(unsigned int cpu)
+static void sched_core_cpu_starting(unsigned int cpu)
 {
 	const struct cpumask *smt_mask = cpu_smt_mask(cpu);
-	struct rq *rq, *core_rq = NULL;
-	int i;
+	struct rq *rq = cpu_rq(cpu), *core_rq = NULL;
+	unsigned long flags;
+	int t;
 
-	core_rq = cpu_rq(cpu)->core;
+	sched_core_lock(cpu, &flags);
 
-	if (!core_rq) {
-		for_each_cpu(i, smt_mask) {
-			rq = cpu_rq(i);
-			if (rq->core && rq->core == rq)
-				core_rq = rq;
+	WARN_ON_ONCE(rq->core != rq);
+
+	/* if we're the first, we'll be our own leader */
+	if (cpumask_weight(smt_mask) == 1)
+		goto unlock;
+
+	/* find the leader */
+	for_each_cpu(t, smt_mask) {
+		if (t == cpu)
+			continue;
+		rq = cpu_rq(t);
+		if (rq->core == rq) {
+			core_rq = rq;
+			break;
 		}
+	}
 
-		if (!core_rq)
-			core_rq = cpu_rq(cpu);
+	if (WARN_ON_ONCE(!core_rq)) /* whoopsie */
+		goto unlock;
 
-		for_each_cpu(i, smt_mask) {
-			rq = cpu_rq(i);
+	/* install and validate core_rq */
+	for_each_cpu(t, smt_mask) {
+		rq = cpu_rq(t);
 
-			WARN_ON_ONCE(rq->core && rq->core != core_rq);
+		if (t == cpu)
 			rq->core = core_rq;
-		}
+
+		WARN_ON_ONCE(rq->core != core_rq);
 	}
+
+unlock:
+	sched_core_unlock(cpu, &flags);
 }
+
+static void sched_core_cpu_deactivate(unsigned int cpu)
+{
+	const struct cpumask *smt_mask = cpu_smt_mask(cpu);
+	struct rq *rq = cpu_rq(cpu), *core_rq = NULL;
+	unsigned long flags;
+	int t;
+
+	sched_core_lock(cpu, &flags);
+
+	/* if we're the last man standing, nothing to do */
+	if (cpumask_weight(smt_mask) == 1) {
+		WARN_ON_ONCE(rq->core != rq);
+		goto unlock;
+	}
+
+	/* if we're not the leader, nothing to do */
+	if (rq->core != rq)
+		goto unlock;
+
+	/* find a new leader */
+	for_each_cpu(t, smt_mask) {
+		if (t == cpu)
+			continue;
+		core_rq = cpu_rq(t);
+		break;
+	}
+
+	if (WARN_ON_ONCE(!core_rq)) /* impossible */
+		goto unlock;
+
+	/* copy the shared state to the new leader */
+	core_rq->core_task_seq      = rq->core_task_seq;
+	core_rq->core_pick_seq      = rq->core_pick_seq;
+	core_rq->core_cookie        = rq->core_cookie;
+	core_rq->core_forceidle     = rq->core_forceidle;
+	core_rq->core_forceidle_seq = rq->core_forceidle_seq;
+
+	/* install new leader */
+	for_each_cpu(t, smt_mask) {
+		rq = cpu_rq(t);
+		rq->core = core_rq;
+	}
+
+unlock:
+	sched_core_unlock(cpu, &flags);
+}
+
+static inline void sched_core_cpu_dying(unsigned int cpu)
+{
+	struct rq *rq = cpu_rq(cpu);
+
+	if (rq->core != rq)
+		rq->core = rq;
+}
+
 #else /* !CONFIG_SCHED_CORE */
 
 static inline void sched_core_cpu_starting(unsigned int cpu) {}
+static inline void sched_core_cpu_deactivate(unsigned int cpu) {}
+static inline void sched_core_cpu_dying(unsigned int cpu) {}
 
 static struct task_struct *
 pick_next_task(struct rq *rq, struct task_struct *prev, struct rq_flags *rf)
@@ -9009,6 +9097,8 @@ int sched_cpu_deactivate(unsigned int cpu)
 	 */
 	if (cpumask_weight(cpu_smt_mask(cpu)) == 2)
 		static_branch_dec_cpuslocked(&sched_smt_present);
+
+	sched_core_cpu_deactivate(cpu);
 #endif
 
 	if (!sched_smp_initialized)
@@ -9113,6 +9203,7 @@ int sched_cpu_dying(unsigned int cpu)
 	calc_load_migrate(rq);
 	update_max_interval();
 	hrtick_clear(rq);
+	sched_core_cpu_dying(cpu);
 	return 0;
 }
 #endif
@@ -9324,7 +9415,7 @@ void __init sched_init(void)
 		atomic_set(&rq->nr_iowait, 0);
 
 #ifdef CONFIG_SCHED_CORE
-		rq->core = NULL;
+		rq->core = rq;
 		rq->core_pick = NULL;
 		rq->core_enabled = 0;
 		rq->core_tree = RB_ROOT;
kernel/sched/sched.h +1 -1
index a9a660c6e08a..e6347c88c467 100644
@@ -1103,7 +1103,7 @@ struct rq {
 	unsigned int		core_sched_seq;
 	struct rb_root		core_tree;
 
-	/* shared state */
+	/* shared state -- careful with sched_core_cpu_deactivate() */
 	unsigned int		core_task_seq;
 	unsigned int		core_pick_seq;
 	unsigned long		core_cookie;
#88 lore raw

Re: [PATCH] sched: Fix Core-wide rq->lock for uninitialized CPUs

Quoted 146 lines · Peter Zijlstra
Quoted 6 lines · Eugene Syromiatnikov
Quoted 4 lines · Peter Zijlstra

Urgh... lemme guess, your HP BIOS is funny and reports more possible CPUs than you actually have resulting in cpu_possible_mask != cpu_online_mask. Alternatively, you booted with nr_cpus= or something daft like that.

Quoted 7 lines

Yep, it seems to be the case:

    # cat /sys/devices/system/cpu/possible
    0-7
    # cat /sys/devices/system/cpu/online
    0-3

I think the below should work... can you please verify?

---
Subject: sched: Fix Core-wide rq->lock for uninitialized CPUs

Eugene tripped over the case where rq_lock(), as called in a for_each_possible_cpu() loop came apart because rq->core hadn't been setup yet.

This is a somewhat unusual, but valid case.

Rework things such that rq->core is initialized to point at itself. IOW initialize each CPU as a single threaded Core. CPU online will then join the new CPU (thread) to an existing Core where needed.

For completeness sake, have CPU offline fully undo the state so as to not presume the topology will match the next time it comes online.

Fixes 9edeaea1bc45 ("sched: Core-wide rq->lock")
---
 kernel/sched/core.c  | 143 +++++++++++++++++++++++++++++++++++++++++----------
 kernel/sched/sched.h |   2 +-
 2 files changed, 118 insertions(+), 27 deletions(-)
kernel/sched/core.c +47 -23
index 0f14e09a4f99..21d633971fcf 100644
@@ -237,9 +237,30 @@ static DEFINE_MUTEX(sched_core_mutex);
 static atomic_t sched_core_count;
 static struct cpumask sched_core_mask;
 
+static void sched_core_lock(int cpu, unsigned long *flags)
+{
+	const struct cpumask *smt_mask = cpu_smt_mask(cpu);
+	int t, i = 0;
+
+	local_irq_save(*flags);
+	for_each_cpu(t, smt_mask)
+		raw_spin_lock_nested(&cpu_rq(t)->__lock, i++);
+}
+
+static void sched_core_unlock(int cpu, unsigned long *flags)
+{
+	const struct cpumask *smt_mask = cpu_smt_mask(cpu);
+	int t;
+
+	for_each_cpu(t, smt_mask)
+		raw_spin_unlock(&cpu_rq(t)->__lock);
+	local_irq_restore(*flags);
+}
+
 static void __sched_core_flip(bool enabled)
 {
-	int cpu, t, i;
+	unsigned long flags;
+	int cpu, t;
 
 	cpus_read_lock();
 
@@ -250,19 +271,12 @@ static void __sched_core_flip(bool enabled)
 	for_each_cpu(cpu, &sched_core_mask) {
 		const struct cpumask *smt_mask = cpu_smt_mask(cpu);
 
-		i = 0;
-		local_irq_disable();
-		for_each_cpu(t, smt_mask) {
-			/* supports up to SMT8 */
-			raw_spin_lock_nested(&cpu_rq(t)->__lock, i++);
-		}
+		sched_core_lock(cpu, &flags);
 
 		for_each_cpu(t, smt_mask)
 			cpu_rq(t)->core_enabled = enabled;
 
-		for_each_cpu(t, smt_mask)
-			raw_spin_unlock(&cpu_rq(t)->__lock);
-		local_irq_enable();
+		sched_core_unlock(cpu, &flags);
 
 		cpumask_andnot(&sched_core_mask, &sched_core_mask, smt_mask);
 	}
@@ -5979,35 +5993,109 @@ void queue_core_balance(struct rq *rq)
 	queue_balance_callback(rq, &per_cpu(core_balance_head, rq->cpu), sched_core_balance);
 }
 
-static inline void sched_core_cpu_starting(unsigned int cpu)
+static void sched_core_cpu_starting(unsigned int cpu)
 {
 	const struct cpumask *smt_mask = cpu_smt_mask(cpu);
-	struct rq *rq, *core_rq = NULL;
-	int i;
+	struct rq *rq = cpu_rq(cpu), *core_rq = NULL;
+	unsigned long flags;
+	int t;
 
-	core_rq = cpu_rq(cpu)->core;
+	sched_core_lock(cpu, &flags);
 
-	if (!core_rq) {
-		for_each_cpu(i, smt_mask) {
-			rq = cpu_rq(i);
-			if (rq->core && rq->core == rq)
-				core_rq = rq;
+	WARN_ON_ONCE(rq->core != rq);
+
+	/* if we're the first, we'll be our own leader */
+	if (cpumask_weight(smt_mask) == 1)
+		goto unlock;
+
+	/* find the leader */
+	for_each_cpu(t, smt_mask) {
+		if (t == cpu)
+			continue;
+		rq = cpu_rq(t);
+		if (rq->core == rq) {
+			core_rq = rq;
+			break;
 		}
+	}
 
-		if (!core_rq)
-			core_rq = cpu_rq(cpu);
+	if (WARN_ON_ONCE(!core_rq)) /* whoopsie */
+		goto unlock;
 
-		for_each_cpu(i, smt_mask) {
-			rq = cpu_rq(i);
+	/* install and validate core_rq */

kernel/sched/core.c void queue_core_balance(struct rq *rq) · line 6033

Quoted 10 lines
kernel/sched/core.c +5 -2
+	for_each_cpu(t, smt_mask) {
+		rq = cpu_rq(t);
 
-			WARN_ON_ONCE(rq->core && rq->core != core_rq);
+		if (t == cpu)
 			rq->core = core_rq;
-		}
+
+		WARN_ON_ONCE(rq->core != core_rq);
 	}

Yoh, Peter!

Read not more than ten times but stay here long enough until I realize something that I am not sure. You are not only just fix the problem, you change the rq's core selection method(manner.). Original source now set the core to one rq, here you distribute the core_rq to not just only one rq.

For example(SMT3): rq1 --> rq0, rq0 --> rq2, rq2 --> rq2(/* whoopsie */).

And the core can change when deactivate.
Not sure I'm getting all the things right, but this is more performance efficient because the lock contention is avoided when pick task from my imagination.
Again, I'm not know much about everything, just put some words.

Quoted 99 lines
kernel/sched/core.c +65 -1
+
+unlock:
+	sched_core_unlock(cpu, &flags);
 }
+
+static void sched_core_cpu_deactivate(unsigned int cpu)
+{
+	const struct cpumask *smt_mask = cpu_smt_mask(cpu);
+	struct rq *rq = cpu_rq(cpu), *core_rq = NULL;
+	unsigned long flags;
+	int t;
+
+	sched_core_lock(cpu, &flags);
+
+	/* if we're the last man standing, nothing to do */
+	if (cpumask_weight(smt_mask) == 1) {
+		WARN_ON_ONCE(rq->core != rq);
+		goto unlock;
+	}
+
+	/* if we're not the leader, nothing to do */
+	if (rq->core != rq)
+		goto unlock;
+
+	/* find a new leader */
+	for_each_cpu(t, smt_mask) {
+		if (t == cpu)
+			continue;
+		core_rq = cpu_rq(t);
+		break;
+	}
+
+	if (WARN_ON_ONCE(!core_rq)) /* impossible */
+		goto unlock;
+
+	/* copy the shared state to the new leader */
+	core_rq->core_task_seq      = rq->core_task_seq;
+	core_rq->core_pick_seq      = rq->core_pick_seq;
+	core_rq->core_cookie        = rq->core_cookie;
+	core_rq->core_forceidle     = rq->core_forceidle;
+	core_rq->core_forceidle_seq = rq->core_forceidle_seq;
+
+	/* install new leader */
+	for_each_cpu(t, smt_mask) {
+		rq = cpu_rq(t);
+		rq->core = core_rq;
+	}
+
+unlock:
+	sched_core_unlock(cpu, &flags);
+}
+
+static inline void sched_core_cpu_dying(unsigned int cpu)
+{
+	struct rq *rq = cpu_rq(cpu);
+
+	if (rq->core != rq)
+		rq->core = rq;
+}
+
 #else /* !CONFIG_SCHED_CORE */
 
 static inline void sched_core_cpu_starting(unsigned int cpu) {}
+static inline void sched_core_cpu_deactivate(unsigned int cpu) {}
+static inline void sched_core_cpu_dying(unsigned int cpu) {}
 
 static struct task_struct *
 pick_next_task(struct rq *rq, struct task_struct *prev, struct rq_flags *rf)
@@ -9009,6 +9097,8 @@ int sched_cpu_deactivate(unsigned int cpu)
 	 */
 	if (cpumask_weight(cpu_smt_mask(cpu)) == 2)
 		static_branch_dec_cpuslocked(&sched_smt_present);
+
+	sched_core_cpu_deactivate(cpu);
 #endif
 
 	if (!sched_smp_initialized)
@@ -9113,6 +9203,7 @@ int sched_cpu_dying(unsigned int cpu)
 	calc_load_migrate(rq);
 	update_max_interval();
 	hrtick_clear(rq);
+	sched_core_cpu_dying(cpu);
 	return 0;
 }
 #endif
@@ -9324,7 +9415,7 @@ void __init sched_init(void)
 		atomic_set(&rq->nr_iowait, 0);
 
 #ifdef CONFIG_SCHED_CORE
-		rq->core = NULL;
+		rq->core = rq;
 		rq->core_pick = NULL;
 		rq->core_enabled = 0;
 		rq->core_tree = RB_ROOT;
kernel/sched/sched.h +0 -0
index a9a660c6e08a..e6347c88c467 100644

kernel/sched/sched.h struct rq { · line 1109

Quoted 8 lines
kernel/sched/sched.h +1 -1
@@ -1103,7 +1103,7 @@ struct rq {
 	unsigned int		core_sched_seq;
 	struct rb_root		core_tree;
 
-	/* shared state */
+	/* shared state -- careful with sched_core_cpu_deactivate() */
 	unsigned int		core_task_seq;
 	unsigned int		core_pick_seq;
 	unsigned long		core_cookie;

Thanks,
Tao

Tested-by on v1 1/1
#89 lore raw

Re: [PATCH] sched: Fix Core-wide rq->lock for uninitialized CPUs

In reply to Peter Zijlstra
Quoted 9 lines · Peter Zijlstra
Quoted 8 lines · Eugene Syromiatnikov
Quoted 4 lines · Peter Zijlstra

Urgh... lemme guess, your HP BIOS is funny and reports more possible CPUs than you actually have resulting in cpu_possible_mask != cpu_online_mask. Alternatively, you booted with nr_cpus= or something daft like that.

Yep, it seems to be the case:

Quoted 7 lines
Quoted 5 lines
    # cat /sys/devices/system/cpu/possible
    0-7
    # cat /sys/devices/system/cpu/online
    0-3

I think the below should work... can you please verify?

Yes, it no longer oops'es now, thank you!

    # cat /sys/devices/system/cpu/possible
    0-7
    # cat /sys/devices/system/cpu/online
    0-3
    # ./prctl-sched-core-oops-repro
    Iteration 0 status: 0
    Iteration 1 status: 0
    # ../src/strace -fvq -eprctl,clone,setsid -esignal=none ./prctl-sched-core-oops-repro
    clone(child_stack=NULL, flags=CLONE_CHILD_CLEARTID|CLONE_CHILD_SETTID|SIGCHLD, child_tidptr=0x7f510b1c7890) = 108328
    [pid 108328] setsid()                   = 108328
    [pid 108328] +++ exited with 0 +++
    Iteration 0 status: 0
    prctl(PR_SCHED_CORE, PR_SCHED_CORE_CREATE, 108324, 0x2 /* PIDTYPE_PGID */, NULL) = 0
    clone(child_stack=NULL, flags=CLONE_CHILD_CLEARTID|CLONE_CHILD_SETTID|SIGCHLD, child_tidptr=0x7f510b1c7890) = 108329
    [pid 108329] setsid()                   = 108329
    [pid 108329] +++ exited with 0 +++
    Iteration 1 status: 0
    prctl(PR_SCHED_CORE, PR_SCHED_CORE_CREATE, 108324, 0x2 /* PIDTYPE_PGID */, NULL) = 0
    +++ exited with 0 +++
Quoted 13 lines

---
Subject: sched: Fix Core-wide rq->lock for uninitialized CPUs

Eugene tripped over the case where rq_lock(), as called in a for_each_possible_cpu() loop came apart because rq->core hadn't been setup yet.

This is a somewhat unusual, but valid case.

Rework things such that rq->core is initialized to point at itself. IOW initialize each CPU as a single threaded Core. CPU online will then join the new CPU (thread) to an existing Core where needed.

Quoted 6 lines

For completeness sake, have CPU offline fully undo the state so as to not presume the topology will match the next time it comes online.

Fixes 9edeaea1bc45 ("sched: Core-wide rq->lock")
Reviewed-by on v1 1/1
#90 lore raw

Re: [PATCH] sched: Fix Core-wide rq->lock for uninitialized CPUs

In reply to Peter Zijlstra

Hi Peter,

kernel/sched/sched.h struct rq { · line 1106

Quoted 3 lines · Peter Zijlstra
kernel/sched/sched.h +1 -1
-       /* shared state */
+       /* shared state -- careful with sched_core_cpu_deactivate() */

Could also throw these fields into a wrapping struct. Either way seems fine.

author maintainer
#91 lore raw

Re: [PATCH] sched: Fix Core-wide rq->lock for uninitialized CPUs

Quoted 10 lines · Josh Don

Hi Peter,

kernel/sched/sched.h struct rq { · line 1106

Quoted 3 lines · Peter Zijlstra
kernel/sched/sched.h +1 -1
-       /* shared state */
+       /* shared state -- careful with sched_core_cpu_deactivate() */

Could also throw these fields into a wrapping struct. Either way seems fine.

Yes, I considered that, but didn't want the churn at this time.

merge bot Applied
#92 lore raw

[tip: sched/urgent] sched: Fix Core-wide rq->lock for uninitialized CPUs

In reply to Peter Zijlstra
The following commit has been merged into the sched/urgent branch of tip:

The following commit has been merged into the sched/urgent branch of tip:

Commit-ID:     3c474b3239f12fe0b00d7e82481f36a1f31e79ab
Gitweb:        https://git.kernel.org/tip/3c474b3239f12fe0b00d7e82481f36a1f31e79ab
Author:        Peter Zijlstra <[email protected]>
AuthorDate:    Thu, 19 Aug 2021 13:09:17 +02:00
Committer:     Peter Zijlstra <[email protected]>
CommitterDate: Fri, 20 Aug 2021 12:32:53 +02:00

sched: Fix Core-wide rq->lock for uninitialized CPUs

Eugene tripped over the case where rq_lock(), as called in a for_each_possible_cpu() loop came apart because rq->core hadn't been setup yet.

This is a somewhat unusual, but valid case.

Rework things such that rq->core is initialized to point at itself. IOW initialize each CPU as a single threaded Core. CPU online will then join the new CPU (thread) to an existing Core where needed.

For completeness sake, have CPU offline fully undo the state so as to not presume the topology will match the next time it comes online.

---
 kernel/sched/core.c  | 143 ++++++++++++++++++++++++++++++++++--------
 kernel/sched/sched.h |   2 +-
 2 files changed, 118 insertions(+), 27 deletions(-)
kernel/sched/core.c +117 -26
index 20ffcc0..f3b27c6 100644
@@ -237,9 +237,30 @@ static DEFINE_MUTEX(sched_core_mutex);
 static atomic_t sched_core_count;
 static struct cpumask sched_core_mask;
 
+static void sched_core_lock(int cpu, unsigned long *flags)
+{
+	const struct cpumask *smt_mask = cpu_smt_mask(cpu);
+	int t, i = 0;
+
+	local_irq_save(*flags);
+	for_each_cpu(t, smt_mask)
+		raw_spin_lock_nested(&cpu_rq(t)->__lock, i++);
+}
+
+static void sched_core_unlock(int cpu, unsigned long *flags)
+{
+	const struct cpumask *smt_mask = cpu_smt_mask(cpu);
+	int t;
+
+	for_each_cpu(t, smt_mask)
+		raw_spin_unlock(&cpu_rq(t)->__lock);
+	local_irq_restore(*flags);
+}
+
 static void __sched_core_flip(bool enabled)
 {
-	int cpu, t, i;
+	unsigned long flags;
+	int cpu, t;
 
 	cpus_read_lock();
 
@@ -250,19 +271,12 @@ static void __sched_core_flip(bool enabled)
 	for_each_cpu(cpu, &sched_core_mask) {
 		const struct cpumask *smt_mask = cpu_smt_mask(cpu);
 
-		i = 0;
-		local_irq_disable();
-		for_each_cpu(t, smt_mask) {
-			/* supports up to SMT8 */
-			raw_spin_lock_nested(&cpu_rq(t)->__lock, i++);
-		}
+		sched_core_lock(cpu, &flags);
 
 		for_each_cpu(t, smt_mask)
 			cpu_rq(t)->core_enabled = enabled;
 
-		for_each_cpu(t, smt_mask)
-			raw_spin_unlock(&cpu_rq(t)->__lock);
-		local_irq_enable();
+		sched_core_unlock(cpu, &flags);
 
 		cpumask_andnot(&sched_core_mask, &sched_core_mask, smt_mask);
 	}
@@ -5736,35 +5750,109 @@ void queue_core_balance(struct rq *rq)
 	queue_balance_callback(rq, &per_cpu(core_balance_head, rq->cpu), sched_core_balance);
 }
 
-static inline void sched_core_cpu_starting(unsigned int cpu)
+static void sched_core_cpu_starting(unsigned int cpu)
 {
 	const struct cpumask *smt_mask = cpu_smt_mask(cpu);
-	struct rq *rq, *core_rq = NULL;
-	int i;
+	struct rq *rq = cpu_rq(cpu), *core_rq = NULL;
+	unsigned long flags;
+	int t;
 
-	core_rq = cpu_rq(cpu)->core;
+	sched_core_lock(cpu, &flags);
 
-	if (!core_rq) {
-		for_each_cpu(i, smt_mask) {
-			rq = cpu_rq(i);
-			if (rq->core && rq->core == rq)
-				core_rq = rq;
+	WARN_ON_ONCE(rq->core != rq);
+
+	/* if we're the first, we'll be our own leader */
+	if (cpumask_weight(smt_mask) == 1)
+		goto unlock;
+
+	/* find the leader */
+	for_each_cpu(t, smt_mask) {
+		if (t == cpu)
+			continue;
+		rq = cpu_rq(t);
+		if (rq->core == rq) {
+			core_rq = rq;
+			break;
 		}
+	}
 
-		if (!core_rq)
-			core_rq = cpu_rq(cpu);
+	if (WARN_ON_ONCE(!core_rq)) /* whoopsie */
+		goto unlock;
 
-		for_each_cpu(i, smt_mask) {
-			rq = cpu_rq(i);
+	/* install and validate core_rq */
+	for_each_cpu(t, smt_mask) {
+		rq = cpu_rq(t);
 
-			WARN_ON_ONCE(rq->core && rq->core != core_rq);
+		if (t == cpu)
 			rq->core = core_rq;
-		}
+
+		WARN_ON_ONCE(rq->core != core_rq);
 	}
+
+unlock:
+	sched_core_unlock(cpu, &flags);
 }
+
+static void sched_core_cpu_deactivate(unsigned int cpu)
+{
+	const struct cpumask *smt_mask = cpu_smt_mask(cpu);
+	struct rq *rq = cpu_rq(cpu), *core_rq = NULL;
+	unsigned long flags;
+	int t;
+
+	sched_core_lock(cpu, &flags);
+
+	/* if we're the last man standing, nothing to do */
+	if (cpumask_weight(smt_mask) == 1) {
+		WARN_ON_ONCE(rq->core != rq);
+		goto unlock;
+	}
+
+	/* if we're not the leader, nothing to do */
+	if (rq->core != rq)
+		goto unlock;
+
+	/* find a new leader */
+	for_each_cpu(t, smt_mask) {
+		if (t == cpu)
+			continue;
+		core_rq = cpu_rq(t);
+		break;
+	}
+
+	if (WARN_ON_ONCE(!core_rq)) /* impossible */
+		goto unlock;
+
+	/* copy the shared state to the new leader */
+	core_rq->core_task_seq      = rq->core_task_seq;
+	core_rq->core_pick_seq      = rq->core_pick_seq;
+	core_rq->core_cookie        = rq->core_cookie;
+	core_rq->core_forceidle     = rq->core_forceidle;
+	core_rq->core_forceidle_seq = rq->core_forceidle_seq;
+
+	/* install new leader */
+	for_each_cpu(t, smt_mask) {
+		rq = cpu_rq(t);
+		rq->core = core_rq;
+	}
+
+unlock:
+	sched_core_unlock(cpu, &flags);
+}
+
+static inline void sched_core_cpu_dying(unsigned int cpu)
+{
+	struct rq *rq = cpu_rq(cpu);
+
+	if (rq->core != rq)
+		rq->core = rq;
+}
+
 #else /* !CONFIG_SCHED_CORE */
 
 static inline void sched_core_cpu_starting(unsigned int cpu) {}
+static inline void sched_core_cpu_deactivate(unsigned int cpu) {}
+static inline void sched_core_cpu_dying(unsigned int cpu) {}
 
 static struct task_struct *
 pick_next_task(struct rq *rq, struct task_struct *prev, struct rq_flags *rf)
@@ -8707,6 +8795,8 @@ int sched_cpu_deactivate(unsigned int cpu)
 	 */
 	if (cpumask_weight(cpu_smt_mask(cpu)) == 2)
 		static_branch_dec_cpuslocked(&sched_smt_present);
+
+	sched_core_cpu_deactivate(cpu);
 #endif
 
 	if (!sched_smp_initialized)
@@ -8811,6 +8901,7 @@ int sched_cpu_dying(unsigned int cpu)
 	calc_load_migrate(rq);
 	update_max_interval();
 	hrtick_clear(rq);
+	sched_core_cpu_dying(cpu);
 	return 0;
 }
 #endif
@@ -9022,7 +9113,7 @@ void __init sched_init(void)
 		atomic_set(&rq->nr_iowait, 0);
 
 #ifdef CONFIG_SCHED_CORE
-		rq->core = NULL;
+		rq->core = rq;
 		rq->core_pick = NULL;
 		rq->core_enabled = 0;
 		rq->core_tree = RB_ROOT;
kernel/sched/sched.h +1 -1
index 14a41a2..da4295f 100644
@@ -1093,7 +1093,7 @@ struct rq {
 	unsigned int		core_sched_seq;
 	struct rb_root		core_tree;
 
-	/* shared state */
+	/* shared state -- careful with sched_core_cpu_deactivate() */
 	unsigned int		core_task_seq;
 	unsigned int		core_pick_seq;
 	unsigned long		core_cookie;
author maintainer
#93 lore raw

PATCH 19/19 kselftest: Add test for core sched prctl interface

In reply to Peter Zijlstra

From: Chris Hyser <[email protected]>

Provides a selftest and examples of using the interface.

[peterz: updated to not use sched_debug]
---
 tools/testing/selftests/sched/.gitignore      |    1 
 tools/testing/selftests/sched/Makefile        |   14 +
 tools/testing/selftests/sched/config          |    1 
 tools/testing/selftests/sched/cs_prctl_test.c |  338 ++++++++++++++++++++++++++
 4 files changed, 354 insertions(+)
 create mode 100644 tools/testing/selftests/sched/.gitignore
 create mode 100644 tools/testing/selftests/sched/Makefile
 create mode 100644 tools/testing/selftests/sched/config
 create mode 100644 tools/testing/selftests/sched/cs_prctl_test.c
tools/testing/selftests/sched/.gitignore new file +1 -0
@@ -0,0 +1 @@
+cs_prctl_test
tools/testing/selftests/sched/Makefile new file +14 -0
@@ -0,0 +1,14 @@
+# SPDX-License-Identifier: GPL-2.0+
+
+ifneq ($(shell $(CC) --version 2>&1 | head -n 1 | grep clang),)
+CLANG_FLAGS += -no-integrated-as
+endif
+
+CFLAGS += -O2 -Wall -g -I./ -I../../../../usr/include/  -Wl,-rpath=./ \
+	  $(CLANG_FLAGS)
+LDLIBS += -lpthread
+
+TEST_GEN_FILES := cs_prctl_test
+TEST_PROGS := cs_prctl_test
+
+include ../lib.mk
tools/testing/selftests/sched/config new file +1 -0
@@ -0,0 +1 @@
+CONFIG_SCHED_DEBUG=y
tools/testing/selftests/sched/cs_prctl_test.c new file +338 -0

338 lines not on this page · raw message

merge bot Applied
#94 lore raw

[tip: sched/core] kselftest: Add test for core sched prctl interface

The following commit has been merged into the sched/core branch of tip:

The following commit has been merged into the sched/core branch of tip:

Commit-ID:     9f26990074931bbf797373e53104216059b300b1
Gitweb:        https://git.kernel.org/tip/9f26990074931bbf797373e53104216059b300b1
Author:        Chris Hyser <[email protected]>
AuthorDate:    Wed, 24 Mar 2021 17:40:16 -04:00
Committer:     Peter Zijlstra <[email protected]>
CommitterDate: Wed, 12 May 2021 11:43:32 +02:00

kselftest: Add test for core sched prctl interface

Provides a selftest and examples of using the interface.

---
 tools/testing/selftests/sched/.gitignore      |   1 +-
 tools/testing/selftests/sched/Makefile        |  14 +-
 tools/testing/selftests/sched/config          |   1 +-
 tools/testing/selftests/sched/cs_prctl_test.c | 338 +++++++++++++++++-
 4 files changed, 354 insertions(+)
 create mode 100644 tools/testing/selftests/sched/.gitignore
 create mode 100644 tools/testing/selftests/sched/Makefile
 create mode 100644 tools/testing/selftests/sched/config
 create mode 100644 tools/testing/selftests/sched/cs_prctl_test.c
tools/testing/selftests/sched/.gitignore new file +1 -0
new file mode 100644
index 0000000..6996d46
@@ -0,0 +1 @@
+cs_prctl_test
tools/testing/selftests/sched/Makefile new file +14 -0
new file mode 100644
index 0000000..10c72f1
@@ -0,0 +1,14 @@
+# SPDX-License-Identifier: GPL-2.0+
+
+ifneq ($(shell $(CC) --version 2>&1 | head -n 1 | grep clang),)
+CLANG_FLAGS += -no-integrated-as
+endif
+
+CFLAGS += -O2 -Wall -g -I./ -I../../../../usr/include/  -Wl,-rpath=./ \
+	  $(CLANG_FLAGS)
+LDLIBS += -lpthread
+
+TEST_GEN_FILES := cs_prctl_test
+TEST_PROGS := cs_prctl_test
+
+include ../lib.mk
tools/testing/selftests/sched/config new file +1 -0
new file mode 100644
index 0000000..e8b09aa
@@ -0,0 +1 @@
+CONFIG_SCHED_DEBUG=y
tools/testing/selftests/sched/cs_prctl_test.c new file +338 -0

338 lines not on this page · raw message

#95 lore raw
In reply to Peter Zijlstra
Quoted 11 lines · Peter Zijlstra

Hai,

This is an agressive fold of all the core-scheduling work so far. I've stripped a whole bunch of tags along the way (hopefully not too many, please yell if you feel I made a mistake), including tested-by. Please retest.

Changes since the last partial post is dropping all the cgroup stuff and PR_SCHED_CORE_CLEAR as well as that exec() behaviour in order to later resolve the cgroup issue.

Hi Peter,

Is there a reason that PR_SCHED_CORE_CLEAR got removed? It's handy to be able to clear
cookies.

Thanks,

Don

Quoted 33 lines

Since we're really rather late for the coming merge window, my plan was to merge the lot right after the merge window.

Again, please test.

These patches should shortly be available in my queue.git.

---
 b/kernel/sched/core_sched.c                     |  229 ++++++
 b/tools/testing/selftests/sched/.gitignore      |    1
 b/tools/testing/selftests/sched/Makefile        |   14
 b/tools/testing/selftests/sched/config          |    1
 b/tools/testing/selftests/sched/cs_prctl_test.c |  338 +++++++++
 include/linux/sched.h                           |   19
 include/uapi/linux/prctl.h                      |    8
 kernel/Kconfig.preempt                          |    6
 kernel/fork.c                                   |    4
 kernel/sched/Makefile                           |    1
 kernel/sched/core.c                             |  858 ++++++++++++++++++++++--
 kernel/sched/cpuacct.c                          |   12
 kernel/sched/deadline.c                         |   38 -
 kernel/sched/debug.c                            |    4
 kernel/sched/fair.c                             |  276 +++++--
 kernel/sched/idle.c                             |   13
 kernel/sched/pelt.h                             |    2
 kernel/sched/rt.c                               |   31
 kernel/sched/sched.h                            |  393 ++++++++--
 kernel/sched/stop_task.c                        |   14
 kernel/sched/topology.c                         |    4
 kernel/sys.c                                    |    5
 tools/include/uapi/linux/prctl.h                |    8
 23 files changed, 2057 insertions(+), 222 deletions(-)
author maintainer
#96 lore raw
Quoted 8 lines · Don Hiatt
Quoted 7 lines · Peter Zijlstra

Hai,

This is an agressive fold of all the core-scheduling work so far. I've stripped a whole bunch of tags along the way (hopefully not too many, please yell if you feel I made a mistake), including tested-by. Please retest.

Quoted 8 lines
Quoted 4 lines

Changes since the last partial post is dropping all the cgroup stuff and PR_SCHED_CORE_CLEAR as well as that exec() behaviour in order to later resolve the cgroup issue.

Hi Peter,

Is there a reason that PR_SCHED_CORE_CLEAR got removed? It's handy to be able to clear cookies.

I agree, but if we're forced to do cgroup with cookie inherit, when CLEAR allows a trivial escape.

Until that whole cgroup thing is sorted, it is easiest to not have CLEAR, lest we have to change/augment CLEAR semantics in unfortunate ways, which is always harder once an API is out there.

Also, see the entire previous discussion here:

https://lore.kernel.org/lkml/[email protected]/

#97 lore raw
In reply to Peter Zijlstra
Quoted 34 lines · Peter Zijlstra

Hai,

This is an agressive fold of all the core-scheduling work so far. I've stripped a whole bunch of tags along the way (hopefully not too many, please yell if you feel I made a mistake), including tested-by. Please retest.

Changes since the last partial post is dropping all the cgroup stuff and PR_SCHED_CORE_CLEAR as well as that exec() behaviour in order to later resolve the cgroup issue.

Since we're really rather late for the coming merge window, my plan was to merge the lot right after the merge window.

Again, please test.

These patches should shortly be available in my queue.git.

---
 b/kernel/sched/core_sched.c                     |  229 ++++++
 b/tools/testing/selftests/sched/.gitignore      |    1 
 b/tools/testing/selftests/sched/Makefile        |   14 
 b/tools/testing/selftests/sched/config          |    1 
 b/tools/testing/selftests/sched/cs_prctl_test.c |  338 +++++++++
 include/linux/sched.h                           |   19 
 include/uapi/linux/prctl.h                      |    8 
 kernel/Kconfig.preempt                          |    6 
 kernel/fork.c                                   |    4 
 kernel/sched/Makefile                           |    1 
 kernel/sched/core.c                             |  858 ++++++++++++++++++++++--
 kernel/sched/cpuacct.c                          |   12 
 kernel/sched/deadline.c                         |   38 -
 kernel/sched/debug.c                            |    4 
 kernel/sched/fair.c                             |  276 +++++--
 kernel/sched/idle.c                             |   13 
Quoted 9 lines
 kernel/sched/pelt.h                             |    2 
 kernel/sched/rt.c                               |   31 
 kernel/sched/sched.h                            |  393 ++++++++--
 kernel/sched/stop_task.c                        |   14 
 kernel/sched/topology.c                         |    4 
 kernel/sys.c                                    |    5 
 tools/include/uapi/linux/prctl.h                |    8 
 23 files changed, 2057 insertions(+), 222 deletions(-)

Adding sysbench/uperf/wis performance results for reference:

- kernel under test:

	-- above patchset of core-scheduling + local fix for softlockup issue: https://lore.kernel.org/lkml/[email protected]/
	-- coresched_v10 kernel source: https://github.com/digitalocean/linux-coresched/commits/coresched/v10-v5.10.y

- workloads:
-- A. sysbench cpu (192 threads) + sysbench cpu (192 threads) -- B. sysbench cpu (192 threads) + sysbench mysql (192 threads) -- C. uperf netperf.xml (192 threads over TCP or UDP protocol separately) -- D. will-it-scale context_switch via pipe (192 threads)

- test machine setup:
CPU(s): 192
On-line CPU(s) list: 0-191
Thread(s) per core: 2
Core(s) per socket: 48
Socket(s): 2
NUMA node(s): 4

- performance change key info:
--workload B: coresched (cs_on), sysbench mysql performance drop around 20% vs coresched_v10
--workload C, coresched (cs_on), uperf performance increased almost double vs coresched_v10
--workload C, default (cs_off), uperf performance drop over 20% vs coresched_v10, same issue seen on v5.12-rc8 base (w/o coresched patchset) --workload D, coresched (cs_on), wis performance increased almost double vs coresched_v10

- performance info of workloads, normalized based on coresched_v10 results --workload A:
Note:
* no performance change compared to coresched_v10

+---------------------------------------+------+----------------------------------------------+------------------------------------------------+-------+-------------------------------+---------------------------------+
|                                       | **   | coresched_peterz_aubrey_fix_base_v5.12-rc8   | coresched_peterz_aubrey_fix_base_v5.12-rc8     | ***   | coresched_v10_base_v5.10.11   | coresched_v10_base_v5.10.11     |
+=======================================+======+==============================================+================================================+=======+===============================+=================================+
| workload                              | **   | sysbench cpu * 192                           | sysbench cpu * 192                             | ***   | sysbench cpu * 192            | sysbench cpu * 192              |
+---------------------------------------+------+----------------------------------------------+------------------------------------------------+-------+-------------------------------+---------------------------------+
| prctl/cgroup                          | **   | prctl on workload cpu_0                      | prctl on workload cpu_1                        | ***   | cg_sysbench_cpu_0             | cg_sysbench_cpu_1               |
+---------------------------------------+------+----------------------------------------------+------------------------------------------------+-------+-------------------------------+---------------------------------+
| record_item                           | **   | Tput_avg (events/s)                          | Tput_avg (events/s)                            | ***   | Tput_avg (events/s)           | Tput_avg (events/s)             |
+---------------------------------------+------+----------------------------------------------+------------------------------------------------+-------+-------------------------------+---------------------------------+
| coresched normalized vs coresched_v10 | **   | 0.99                                         | 1.01                                           | ***   | 1                             | 1                               |
+---------------------------------------+------+----------------------------------------------+------------------------------------------------+-------+-------------------------------+---------------------------------+
| default normalized vs coresched_v10   | **   | 1.03                                         | 0.98                                           | ***   | 1                             | 1                               |
+---------------------------------------+------+----------------------------------------------+------------------------------------------------+-------+-------------------------------+---------------------------------+
| smtoff normalized vs coresched_v10    | **   | 1.01                                         | 0.99                                           | ***   | 1                             | 1                               |
+---------------------------------------+------+----------------------------------------------+------------------------------------------------+-------+-------------------------------+---------------------------------+

--workload B:
Note:
* coresched (cs_on), sysbench mysql performance drop around 20% vs coresched_v10

+---------------------------------------+------+----------------------------------------------+------------------------------------------------+-------+-------------------------------+---------------------------------+
|                                       | **   | coresched_peterz_aubrey_fix_base_v5.12-rc8   | coresched_peterz_aubrey_fix_base_v5.12-rc8     | ***   | coresched_v10_base_v5.10.11   | coresched_v10_base_v5.10.11     |
+=======================================+======+==============================================+================================================+=======+===============================+=================================+
| workload                              | **   | sysbench cpu * 192                           | sysbench mysql * 192                           | ***   | sysbench cpu * 192            | sysbench mysql * 192            |
+---------------------------------------+------+----------------------------------------------+------------------------------------------------+-------+-------------------------------+---------------------------------+
| prctl/cgroup                          | **   | prctl on workload cpu_0                      | prctl on workload mysql_0                      | ***   | cg_sysbench_cpu_0             | cg_sysbench_mysql_0             |
+---------------------------------------+------+----------------------------------------------+------------------------------------------------+-------+-------------------------------+---------------------------------+
| record_item                           | **   | Tput_avg (events/s)                          | Tput_avg (events/s)                            | ***   | Tput_avg (events/s)           | Tput_avg (events/s)             |
+---------------------------------------+------+----------------------------------------------+------------------------------------------------+-------+-------------------------------+---------------------------------+
| coresched normalized vs coresched_v10 | **   | 1.03                                         | 0.77                                           | ***   | 1                             | 1                               |
+---------------------------------------+------+----------------------------------------------+------------------------------------------------+-------+-------------------------------+---------------------------------+
| default normalized vs coresched_v10   | **   | 1.02                                         | 0.9                                            | ***   | 1                             | 1                               |
+---------------------------------------+------+----------------------------------------------+------------------------------------------------+-------+-------------------------------+---------------------------------+
| smtoff normalized vs coresched_v10    | **   | 0.94                                         | 1.14                                           | ***   | 1                             | 1                               |
+---------------------------------------+------+----------------------------------------------+------------------------------------------------+-------+-------------------------------+---------------------------------+

--workload C:
Note:
* coresched (cs_on), uperf performance increased almost double vs coresched_v10
* default (cs_off), uperf performance drop over 20% vs coresched_v10, same issue seen on v5.12-rc8 base (w/o coresched patchset)

+---------------------------------------+------+----------------------------------------------+------------------------------------------------+-------+-------------------------------+---------------------------------+
|                                       | **   | coresched_peterz_aubrey_fix_base_v5.12-rc8   | coresched_peterz_aubrey_fix_base_v5.12-rc8     | ***   | coresched_v10_base_v5.10.11   | coresched_v10_base_v5.10.11     |
+=======================================+======+==============================================+================================================+=======+===============================+=================================+
| workload                              | **   | uperf netperf TCP * 192                      | uperf netperf UDP * 192                        | ***   | uperf netperf TCP * 192       | uperf netperf UDP * 192         |
+---------------------------------------+------+----------------------------------------------+------------------------------------------------+-------+-------------------------------+---------------------------------+
| prctl/cgroup                          | **   | prctl on workload uperf                      | prctl on workload uperf                        | ***   | cg_uperf                      | cg_uperf                        |
+---------------------------------------+------+----------------------------------------------+------------------------------------------------+-------+-------------------------------+---------------------------------+
| record_item                           | **   | Tput_avg (Gb/s)                              | Tput_avg (Gb/s)                                | ***   | Tput_avg (Gb/s)               | Tput_avg (Gb/s)                 |
+---------------------------------------+------+----------------------------------------------+------------------------------------------------+-------+-------------------------------+---------------------------------+
| coresched normalized vs coresched_v10 | **   | 1.87                                         | 1.99                                           | ***   | 1                             | 1                               |
+---------------------------------------+------+----------------------------------------------+------------------------------------------------+-------+-------------------------------+---------------------------------+
| default normalized vs coresched_v10   | **   | 0.78                                         | 0.74                                           | ***   | 1                             | 1                               |
+---------------------------------------+------+----------------------------------------------+------------------------------------------------+-------+-------------------------------+---------------------------------+
| smtoff normalized vs coresched_v10    | **   | 0.87                                         | 0.95                                           | ***   | 1                             | 1                               |
+---------------------------------------+------+----------------------------------------------+------------------------------------------------+-------+-------------------------------+---------------------------------+

--workload D:
Note:
* coresched (cs_on), wis performance increased almost double vs coresched_v10

+---------------------------------------+------+----------------------------------------------+-------+-------------------------------+
|                                       | **   | coresched_peterz_aubrey_fix_base_v5.12-rc8   | ***   | coresched_v10_base_v5.10.11   |
+=======================================+======+==============================================+=======+===============================+
| workload                              | **   | will-it-scale  * 192                         | ***   | will-it-scale  * 192          |
|                                       |      | (pipe based context_switch)                  |       | (pipe based context_switch)   |
+---------------------------------------+------+----------------------------------------------+-------+-------------------------------+
| prctl/cgroup                          | **   | prctl on workload wis                        | ***   | cg_wis                        |
+---------------------------------------+------+----------------------------------------------+-------+-------------------------------+
| record_item                           | **   | threads_avg                                  | ***   | threads_avg                   |
+---------------------------------------+------+----------------------------------------------+-------+-------------------------------+
| coresched normalized vs coresched_v10 | **   | 1.98                                         | ***   | 1                             |
+---------------------------------------+------+----------------------------------------------+-------+-------------------------------+
| default normalized vs coresched_v10   | **   | 1.13                                         | ***   | 1                             |
+---------------------------------------+------+----------------------------------------------+-------+-------------------------------+
| smtoff normalized vs coresched_v10    | **   | 1.32                                         | ***   | 1                             |
+---------------------------------------+------+----------------------------------------------+-------+-------------------------------+

-- notes on record_item:
* coresched normalized vs coresched_v10: smton, cs enabled, test result normalized by result of coresched_v10 under same config
* default normalized vs coresched_v10: smton, cs disabled, test result normalized by result of coresched_v10 under same config
* smtoff normalized vs coresched_v10: smtoff, test result normalized by result of coresched_v10 under same config

Hongyu

#99 lore raw
Quoted 7 lines · Peter Zijlstra
Quoted 5 lines · Hongyu

Adding sysbench/uperf/wis performance results for reference:

- kernel under test:

	-- above patchset of core-scheduling + local fix for softlockup issue: https://lore.kernel.org/lkml/[email protected]/
	-- coresched_v10 kernel source: https://github.com/digitalocean/linux-coresched/commits/coresched/v10-v5.10.y
Quoted 6 lines

Shall I summarize all this as:

?

Yes, that would be great.

Regards,
Hongyu

#100 lore raw
In reply to Peter Zijlstra
Quoted 7 lines · Peter Zijlstra

Hai,

This is an agressive fold of all the core-scheduling work so far. I've stripped a whole bunch of tags along the way (hopefully not too many, please yell if you feel I made a mistake), including tested-by. Please retest.

Quoted 8 lines

Changes since the last partial post is dropping all the cgroup stuff and PR_SCHED_CORE_CLEAR as well as that exec() behaviour in order to later resolve the cgroup issue.

Since we're really rather late for the coming merge window, my plan was to merge the lot right after the merge window.

Again, please test.

Thanks Peter, I uploaded this set to our ChromeOS 5.10 kernel tree with minor changes to make it apply, to:
https://chromium-review.googlesource.com/q/topic:cs510
Let me know if anything looks weird, but it does build. I will be testing it further in the coming days.

Of course product kernels slightly lag upstream but such is life <:-)

- Joel

Quoted 29 lines

These patches should shortly be available in my queue.git.

---
 b/kernel/sched/core_sched.c                     |  229 ++++++
 b/tools/testing/selftests/sched/.gitignore      |    1
 b/tools/testing/selftests/sched/Makefile        |   14
 b/tools/testing/selftests/sched/config          |    1
 b/tools/testing/selftests/sched/cs_prctl_test.c |  338 +++++++++
 include/linux/sched.h                           |   19
 include/uapi/linux/prctl.h                      |    8
 kernel/Kconfig.preempt                          |    6
 kernel/fork.c                                   |    4
 kernel/sched/Makefile                           |    1
 kernel/sched/core.c                             |  858 ++++++++++++++++++++++--
 kernel/sched/cpuacct.c                          |   12
 kernel/sched/deadline.c                         |   38 -
 kernel/sched/debug.c                            |    4
 kernel/sched/fair.c                             |  276 +++++--
 kernel/sched/idle.c                             |   13
 kernel/sched/pelt.h                             |    2
 kernel/sched/rt.c                               |   31
 kernel/sched/sched.h                            |  393 ++++++++--
 kernel/sched/stop_task.c                        |   14
 kernel/sched/topology.c                         |    4
 kernel/sys.c                                    |    5
 tools/include/uapi/linux/prctl.h                |    8
 23 files changed, 2057 insertions(+), 222 deletions(-)
maintainer
#101 lore raw
In reply to Peter Zijlstra

Hi Peter,

Quoted 7 lines · Peter Zijlstra

Hai,

This is an agressive fold of all the core-scheduling work so far. I've stripped a whole bunch of tags along the way (hopefully not too many, please yell if you feel I made a mistake), including tested-by. Please retest.

Quoted 8 lines

Changes since the last partial post is dropping all the cgroup stuff and PR_SCHED_CORE_CLEAR as well as that exec() behaviour in order to later resolve the cgroup issue.

Since we're really rather late for the coming merge window, my plan was to merge the lot right after the merge window.

Again, please test.

I have run various tests with your sched/core-sched branch on my arm64
machine: 2 numa * 28 cores * SMT4

List of tests:
perf sched pipe
hackbench process and threads with 1,4,16,32,64,128,256 groups tbench with 1,4,16,32,64,128,256 groups
Unixbench shell and exec
reaim

core-sched was compiled but not used: CONFIG_SCHED_CORE=y

I haven't seen any regressions or problems; it's a mix of +/- 1-2%. I have only seen one significant improvement for hackbench -g 1 (+4.92%) that I don't really explain. Will dig a bit one this later

So Tested-by: Vincent Guittot <[email protected]>

Quoted 29 lines

These patches should shortly be available in my queue.git.

---
 b/kernel/sched/core_sched.c                     |  229 ++++++
 b/tools/testing/selftests/sched/.gitignore      |    1
 b/tools/testing/selftests/sched/Makefile        |   14
 b/tools/testing/selftests/sched/config          |    1
 b/tools/testing/selftests/sched/cs_prctl_test.c |  338 +++++++++
 include/linux/sched.h                           |   19
 include/uapi/linux/prctl.h                      |    8
 kernel/Kconfig.preempt                          |    6
 kernel/fork.c                                   |    4
 kernel/sched/Makefile                           |    1
 kernel/sched/core.c                             |  858 ++++++++++++++++++++++--
 kernel/sched/cpuacct.c                          |   12
 kernel/sched/deadline.c                         |   38 -
 kernel/sched/debug.c                            |    4
 kernel/sched/fair.c                             |  276 +++++--
 kernel/sched/idle.c                             |   13
 kernel/sched/pelt.h                             |    2
 kernel/sched/rt.c                               |   31
 kernel/sched/sched.h                            |  393 ++++++++--
 kernel/sched/stop_task.c                        |   14
 kernel/sched/topology.c                         |    4
 kernel/sys.c                                    |    5
 tools/include/uapi/linux/prctl.h                |    8
 23 files changed, 2057 insertions(+), 222 deletions(-)
maintainer Tested-by
#102 lore raw
Quoted 30 lines · Vincent Guittot

Hi Peter,

Quoted 7 lines · Peter Zijlstra

Hai,

This is an agressive fold of all the core-scheduling work so far. I've stripped a whole bunch of tags along the way (hopefully not too many, please yell if you feel I made a mistake), including tested-by. Please retest.

Quoted 8 lines

Changes since the last partial post is dropping all the cgroup stuff and PR_SCHED_CORE_CLEAR as well as that exec() behaviour in order to later resolve the cgroup issue.

Since we're really rather late for the coming merge window, my plan was to merge the lot right after the merge window.

Again, please test.

I have run various tests with your sched/core-sched branch on my arm64
machine: 2 numa * 28 cores * SMT4

List of tests:
perf sched pipe
hackbench process and threads with 1,4,16,32,64,128,256 groups tbench with 1,4,16,32,64,128,256 groups
Unixbench shell and exec
reaim

Quoted 7 lines

core-sched was compiled but not used: CONFIG_SCHED_CORE=y

I haven't seen any regressions or problems; it's a mix of +/- 1-2%. I have only seen one significant improvement for hackbench -g 1 (+4.92%) that I don't really explain. Will dig a bit one this later

So Tested-by: Vincent Guittot <[email protected]>

With the correct email address

Quoted 32 lines
Quoted 32 lines

These patches should shortly be available in my queue.git.

---
 b/kernel/sched/core_sched.c                     |  229 ++++++
 b/tools/testing/selftests/sched/.gitignore      |    1
 b/tools/testing/selftests/sched/Makefile        |   14
 b/tools/testing/selftests/sched/config          |    1
 b/tools/testing/selftests/sched/cs_prctl_test.c |  338 +++++++++
 include/linux/sched.h                           |   19
 include/uapi/linux/prctl.h                      |    8
 kernel/Kconfig.preempt                          |    6
 kernel/fork.c                                   |    4
 kernel/sched/Makefile                           |    1
 kernel/sched/core.c                             |  858 ++++++++++++++++++++++--
 kernel/sched/cpuacct.c                          |   12
 kernel/sched/deadline.c                         |   38 -
 kernel/sched/debug.c                            |    4
 kernel/sched/fair.c                             |  276 +++++--
 kernel/sched/idle.c                             |   13
 kernel/sched/pelt.h                             |    2
 kernel/sched/rt.c                               |   31
 kernel/sched/sched.h                            |  393 ++++++++--
 kernel/sched/stop_task.c                        |   14
 kernel/sched/topology.c                         |    4
 kernel/sys.c                                    |    5
 tools/include/uapi/linux/prctl.h                |    8
 23 files changed, 2057 insertions(+), 222 deletions(-)