sched: Core Scheduling
Subsystems (5)
[tip: sched/core] sched/fair: Snapshot the min_vruntime of CPUs on force idle
In reply to Peter ZijlstraThe following commit has been merged into the sched/core branch of tip:
The following commit has been merged into the sched/core branch of tip:
Commit-ID: c6047c2e3af68dae23ad884249e0d42ff28d2d1b Gitweb: https://git.kernel.org/tip/c6047c2e3af68dae23ad884249e0d42ff28d2d1b Author: Joel Fernandes (Google) <[email protected]> AuthorDate: Tue, 17 Nov 2020 18:19:39 -05:00 Committer: Peter Zijlstra <[email protected]> CommitterDate: Wed, 12 May 2021 11:43:29 +02:00
sched/fair: Snapshot the min_vruntime of CPUs on force idle
During force-idle, we end up doing cross-cpu comparison of vruntimes during pick_next_task. If we simply compare (vruntime-min_vruntime) across CPUs, and if the CPUs only have 1 task each, we will always end up comparing 0 with 0 and pick just one of the tasks all the time. This starves the task that was not picked. To fix this, take a snapshot of the min_vruntime when entering force idle and use it for comparison. This min_vruntime snapshot will only be used for cross-CPU vruntime comparison, and nothing else.
A note about the min_vruntime snapshot and force idling:
During selection:
When we're not fi, we need to update snapshot.
when we're fi and we were not fi, we must update snapshot. When we're fi and we were already fi, we must not update snapshot.
Which gives:
fib fi update 0 0 1 0 1 1 1 0 1 1 1 0
Where:
fi: force-idled now
fib: force-idled before
So the min_vruntime snapshot needs to be updated when: !(fib && fi).
Also, the cfs_prio_less() function needs to be aware of whether the core is in force idle or not, since it will be use this information to know whether to advance a cfs_rq's min_vruntime_fi in the hierarchy. So pass this information along via pick_task() -> prio_less().
--- kernel/sched/core.c | 59 +++++++++++++++++++--------------- kernel/sched/fair.c | 75 +++++++++++++++++++++++++++++++++++++++++++- kernel/sched/sched.h | 8 +++++- 3 files changed, 117 insertions(+), 25 deletions(-)
kernel/sched/core.c +34 -25
@@ -111,7 +111,7 @@ static inline int __task_prio(struct task_struct *p)*/ /* real prio, less is less */ -static inline bool prio_less (struct task_struct *a,struct task_struct *b) +static inline bool prio_less (struct task_struct *a,struct task_struct *b,bool in_fi) {int pa =__task_prio (a), pb =__task_prio (b);
@@ -125,19 +125,8 @@ static inline bool prio_less(struct task_struct *a, struct task_struct *b)if (pa ==-1 )/* dl_prio() doesn't work because of stop_class above */ return !dl_time_before (a->dl.deadline, b->dl.deadline); -if (pa ==MAX_RT_PRIO +MAX_NICE ) {/* fair */ -u64 vruntime = b->se.vruntime; - -/* -* Normalize the vruntime if tasks are in different cpus. -*/ -if (task_cpu (a) !=task_cpu (b)) { - vruntime -=task_cfs_rq (b)->min_vruntime; - vruntime +=task_cfs_rq (a)->min_vruntime; - } - -return !((s64)(a->se.vruntime - vruntime) <=0 ); - } +if (pa ==MAX_RT_PRIO +MAX_NICE )/* fair */ +return cfs_prio_less (a, b, in_fi);return false; }
@@ -151,7 +140,7 @@ static inline bool __sched_core_less(struct task_struct *a, struct task_struct *return false;/* flip prio, so high prio is leftmost */ -if (prio_less (b, a)) +if (prio_less (b, a,task_rq (a)->core->core_forceidle))return true;return false;
@@ -5350,7 +5339,7 @@ static inline bool cookie_match(struct task_struct *a, struct task_struct *b)* - Else returns idle_task. */ static struct task_struct * -pick_task (struct rq *rq,const struct sched_class *class,struct task_struct *max) +pick_task (struct rq *rq,const struct sched_class *class,struct task_struct *max,bool in_fi) {struct task_struct *class_pick, *cookie_pick;unsigned long cookie = rq->core->core_cookie;
@@ -5365,7 +5354,7 @@ pick_task(struct rq *rq, const struct sched_class *class, struct task_struct *ma* higher priority than max. */ if (max && class_pick->core_cookie && -prio_less (class_pick, max)) +prio_less (class_pick, max, in_fi))return idle_sched_class.pick_task (rq);return class_pick;
@@ -5384,19 +5373,22 @@ pick_task(struct rq *rq, const struct sched_class *class, struct task_struct *ma* the core (so far) and it must be selected, otherwise we must go with * the cookie pick in order to satisfy the constraint. */ -if (prio_less (cookie_pick, class_pick) && - (!max ||prio_less (max, class_pick))) +if (prio_less (cookie_pick, class_pick, in_fi) && + (!max ||prio_less (max, class_pick, in_fi)))return class_pick;return cookie_pick; } +extern void task_vruntime_update (struct rq *rq,struct task_struct *p,bool in_fi); +static struct task_struct *pick_next_task (struct rq *rq,struct task_struct *prev,struct rq_flags *rf) {struct task_struct *next, *max =NULL ;const struct sched_class *class;const struct cpumask *smt_mask; +bool fi_before = false;bool need_sync;int i, j, cpu;
@@ -5478,9 +5470,14 @@ pick_next_task(struct rq *rq, struct task_struct *prev, struct rq_flags *rf)if (!next->core_cookie) { rq->core_pick =NULL ; +/* +* For robustness, update the min_vruntime_fi for +* unconstrained picks as well. +*/ +WARN_ON_ONCE (fi_before); +task_vruntime_update (rq, next, false); goto done; } - need_sync = true; }for_each_cpu (i, smt_mask) {
@@ -5511,11 +5508,16 @@ again:* highest priority task already selected for this * core. */ - p =pick_task (rq_i, class, max); + p =pick_task (rq_i, class, max, fi_before);if (!p)continue ; rq_i->core_pick = p; +if (rq_i->idle == p && rq_i->nr_running) { + rq->core->core_forceidle = true; +if (!fi_before) + rq->core->core_forceidle_seq++; + }/* * If this new candidate is of higher priority than the
@@ -5534,6 +5536,7 @@ again: max = p;if (old_max) { + rq->core->core_forceidle = false;for_each_cpu (j, smt_mask) {if (j == i)continue ;
@@ -5574,10 +5577,16 @@ again:if (!rq_i->core_pick)continue ; -if (is_task_rq_idle (rq_i->core_pick) && rq_i->nr_running && - !rq_i->core->core_forceidle) { - rq_i->core->core_forceidle = true; - } +/* +* Update for new !FI->FI transitions, or if continuing to be in !FI: +* fi_before fi update? +* 0 0 1 +* 0 1 1 +* 1 0 1 +* 1 1 0 +*/ +if (!(fi_before && rq->core->core_forceidle)) +task_vruntime_update (rq_i, rq_i->core_pick, rq->core->core_forceidle);if (i == cpu) { rq_i->core_pick =NULL ;
kernel/sched/fair.c +75 -0
81 lines not on this page · raw message
kernel/sched/sched.h +8 -0
@@ -526,6 +526,11 @@ struct cfs_rq {u64 exec_clock;u64 min_vruntime; +#ifdef CONFIG_SCHED_CORE +unsigned int forceidle_seq; +u64 min_vruntime_fi; +#endif +#ifndef CONFIG_64BIT u64 min_vruntime_copy;#endif
@@ -1089,6 +1094,7 @@ struct rq {unsigned int core_pick_seq;unsigned long core_cookie;unsigned char core_forceidle; +unsigned int core_forceidle_seq;#endif };
@@ -1162,6 +1168,8 @@ static inline raw_spinlock_t *__rq_lockp(struct rq *rq) return &rq->__lock; } +bool cfs_prio_less (struct task_struct *a,struct task_struct *b,bool fi); +#else /* !CONFIG_SCHED_CORE */ static inline bool sched_core_enabled (struct rq *rq)
Peter Zijlstra
maintainerPATCH 14/19 sched: Trivial forced-newidle balancer
In reply to Peter ZijlstraWhen a sibling is forced-idle to match the core-cookie; search for matching tasks to fill the core.
rcu_read_unlock() can incur an infrequent deadlock in sched_core_balance(). Fix this by using the RCU-sched flavor instead.
--- include/linux/sched.h | 1 kernel/sched/core.c | 130 +++++++++++++++++++++++++++++++++++++++++++++++++- kernel/sched/idle.c | 1 kernel/sched/sched.h | 6 ++ 4 files changed, 137 insertions(+), 1 deletion(-)
include/linux/sched.h +1 -0
@@ -704,6 +704,7 @@ struct task_struct {#ifdef CONFIG_SCHED_CORE struct rb_node core_node;unsigned long core_cookie; +unsigned int core_occupation;#endif #ifdef CONFIG_CGROUP_SCHED
kernel/sched/core.c +129 -1
167 lines not on this page · raw message
kernel/sched/idle.c +1 -0
@@ -437,6 +437,7 @@ static void set_next_task_idle(struct rq {update_idle_core (rq);schedstat_inc (rq->sched_goidle); +queue_core_balance (rq); }#ifdef CONFIG_SMP
kernel/sched/sched.h +6 -0
@@ -1170,6 +1170,8 @@ static inline raw_spinlock_t *__rq_lockpbool cfs_prio_less (struct task_struct *a,struct task_struct *b,bool fi); +extern void queue_core_balance (struct rq *rq); +#else /* !CONFIG_SCHED_CORE */ static inline bool sched_core_enabled (struct rq *rq)
@@ -1187,6 +1189,10 @@ static inline raw_spinlock_t *rq_lockp(s return &rq->__lock; } +static inline void queue_core_balance (struct rq *rq) +{ +} +#endif /* CONFIG_SCHED_CORE */ static inline void lockdep_assert_rq_held (struct rq *rq)
[tip: sched/core] sched: Trivial forced-newidle balancer
The following commit has been merged into the sched/core branch of tip:
The following commit has been merged into the sched/core branch of tip:
Commit-ID: d2dfa17bc7de67e99685c4d6557837bf801a102c Gitweb: https://git.kernel.org/tip/d2dfa17bc7de67e99685c4d6557837bf801a102c Author: Peter Zijlstra <[email protected]> AuthorDate: Tue, 17 Nov 2020 18:19:43 -05:00 Committer: Peter Zijlstra <[email protected]> CommitterDate: Wed, 12 May 2021 11:43:30 +02:00
sched: Trivial forced-newidle balancer
When a sibling is forced-idle to match the core-cookie; search for matching tasks to fill the core.
rcu_read_unlock() can incur an infrequent deadlock in sched_core_balance(). Fix this by using the RCU-sched flavor instead.
--- include/linux/sched.h | 1 +- kernel/sched/core.c | 130 ++++++++++++++++++++++++++++++++++++++++- kernel/sched/idle.c | 1 +- kernel/sched/sched.h | 6 ++- 4 files changed, 137 insertions(+), 1 deletion(-)
include/linux/sched.h +1 -0
@@ -705,6 +705,7 @@ struct task_struct {#ifdef CONFIG_SCHED_CORE struct rb_node core_node;unsigned long core_cookie; +unsigned int core_occupation;#endif #ifdef CONFIG_CGROUP_SCHED
kernel/sched/core.c +129 -1
167 lines not on this page · raw message
kernel/sched/idle.c +1 -0
@@ -437,6 +437,7 @@ static void set_next_task_idle(struct rq *rq, struct task_struct *next, bool fir {update_idle_core (rq);schedstat_inc (rq->sched_goidle); +queue_core_balance (rq); }#ifdef CONFIG_SMP
kernel/sched/sched.h +6 -0
@@ -1170,6 +1170,8 @@ static inline raw_spinlock_t *__rq_lockp(struct rq *rq)bool cfs_prio_less (struct task_struct *a,struct task_struct *b,bool fi); +extern void queue_core_balance (struct rq *rq); +#else /* !CONFIG_SCHED_CORE */ static inline bool sched_core_enabled (struct rq *rq)
@@ -1192,6 +1194,10 @@ static inline raw_spinlock_t *__rq_lockp(struct rq *rq) return &rq->__lock; } +static inline void queue_core_balance (struct rq *rq) +{ +} +#endif /* CONFIG_SCHED_CORE */ static inline void lockdep_assert_rq_held (struct rq *rq)
Peter Zijlstra
maintainerPATCH 15/19 sched: Migration changes for core scheduling
In reply to Peter ZijlstraFrom: Aubrey Li <[email protected]>
- Don't migrate if there is a cookie mismatch
Load balance tries to move task from busiest CPU to the destination CPU. When core scheduling is enabled, if the task's cookie does not match with the destination CPU's core cookie, this task may be skipped by this CPU. This mitigates the forced idle time on the destination CPU.
- Select cookie matched idle CPU
In the fast path of task wakeup, select the first cookie matched idle CPU instead of the first idle CPU.
- Find cookie matched idlest CPU
In the slow path of task wakeup, find the idlest CPU whose core cookie matches with task's cookie
--- kernel/sched/fair.c | 29 ++++++++++++++++---- kernel/sched/sched.h | 73 +++++++++++++++++++++++++++++++++++++++++++++++++++ 2 files changed, 96 insertions(+), 6 deletions(-)
kernel/sched/fair.c +23 -6
@@ -5889,11 +5889,15 @@ find_idlest_group_cpu(struct sched_group/* Traverse only the allowed CPUs */ for_each_cpu_and (i,sched_group_span (group), p->cpus_ptr) { +struct rq *rq =cpu_rq (i); + +if (!sched_core_cookie_match (rq, p)) +continue ; +if (sched_idle_cpu (i))return i;if (available_idle_cpu (i)) { -struct rq *rq =cpu_rq (i);struct cpuidle_state *idle =idle_get_state (rq);if (idle && idle->exit_latency < min_exit_latency) {/*
@@ -5979,9 +5983,10 @@ static inline int find_idlest_cpu(structreturn new_cpu; } -static inline int __select_idle_cpu (int cpu) +static inline int __select_idle_cpu (int cpu,struct task_struct *p) { -if (available_idle_cpu (cpu) ||sched_idle_cpu (cpu)) +if ((available_idle_cpu (cpu) ||sched_idle_cpu (cpu)) && +sched_cpu_cookie_match (cpu_rq (cpu), p))return cpu;return -1 ;
@@ -6051,7 +6056,7 @@ static int select_idle_core(struct task_int cpu;if (!static_branch_likely (&sched_smt_present)) -return __select_idle_cpu (core); +return __select_idle_cpu (core, p);for_each_cpu (cpu,cpu_smt_mask (core)) {if (!available_idle_cpu (cpu)) {
@@ -6107,7 +6112,7 @@ static inline bool test_idle_cores(int cstatic inline int select_idle_core (struct task_struct *p,int core,struct cpumask *cpus,int *idle_cpu) { -return __select_idle_cpu (core); +return __select_idle_cpu (core, p); }static inline int select_idle_smt (struct task_struct *p,struct sched_domain *sd,int target)
@@ -6164,7 +6169,7 @@ static int select_idle_cpu(struct task_s }else {if (!--nr)return -1 ; - idle_cpu =__select_idle_cpu (cpu); + idle_cpu =__select_idle_cpu (cpu, p);if ((unsigned int )idle_cpu < nr_cpumask_bits)break ; }
@@ -7517,6 +7522,14 @@ static int task_hot(struct task_struct *if (sysctl_sched_migration_cost ==-1 )return 1 ; + +/* +* Don't migrate task if the task's cookie does not match +* with the destination CPU's core cookie. +*/ +if (!sched_core_cookie_match (cpu_rq (env->dst_cpu), p)) +return 1 ; +if (sysctl_sched_migration_cost ==0 )return 0 ;
@@ -8847,6 +8860,10 @@ find_idlest_group(struct sched_domain *s p->cpus_ptr))continue ; +/* Skip over this group if no cookie matched */ +if (!sched_group_cookie_match (cpu_rq (this_cpu), p, group)) +continue ; + local_group =cpumask_test_cpu (this_cpu,sched_group_span (group));
kernel/sched/sched.h +73 -0
92 lines not on this page · raw message
[tip: sched/core] sched: Migration changes for core scheduling
The following commit has been merged into the sched/core branch of tip:
The following commit has been merged into the sched/core branch of tip:
Commit-ID: 97886d9dcd86820bdbc1fa73b455982809cbc8c2 Gitweb: https://git.kernel.org/tip/97886d9dcd86820bdbc1fa73b455982809cbc8c2 Author: Aubrey Li <[email protected]> AuthorDate: Wed, 24 Mar 2021 17:40:13 -04:00 Committer: Peter Zijlstra <[email protected]> CommitterDate: Wed, 12 May 2021 11:43:30 +02:00
sched: Migration changes for core scheduling
- Don't migrate if there is a cookie mismatch
Load balance tries to move task from busiest CPU to the destination CPU. When core scheduling is enabled, if the task's cookie does not match with the destination CPU's core cookie, this task may be skipped by this CPU. This mitigates the forced idle time on the destination CPU.
- Select cookie matched idle CPU
In the fast path of task wakeup, select the first cookie matched idle CPU instead of the first idle CPU.
- Find cookie matched idlest CPU
In the slow path of task wakeup, find the idlest CPU whose core cookie matches with task's cookie
--- kernel/sched/fair.c | 29 +++++++++++++---- kernel/sched/sched.h | 73 +++++++++++++++++++++++++++++++++++++++++++- 2 files changed, 96 insertions(+), 6 deletions(-)
kernel/sched/fair.c +23 -6
@@ -5889,11 +5889,15 @@ find_idlest_group_cpu(struct sched_group *group, struct task_struct *p, int this/* Traverse only the allowed CPUs */ for_each_cpu_and (i,sched_group_span (group), p->cpus_ptr) { +struct rq *rq =cpu_rq (i); + +if (!sched_core_cookie_match (rq, p)) +continue ; +if (sched_idle_cpu (i))return i;if (available_idle_cpu (i)) { -struct rq *rq =cpu_rq (i);struct cpuidle_state *idle =idle_get_state (rq);if (idle && idle->exit_latency < min_exit_latency) {/*
@@ -5979,9 +5983,10 @@ static inline int find_idlest_cpu(struct sched_domain *sd, struct task_struct *preturn new_cpu; } -static inline int __select_idle_cpu (int cpu) +static inline int __select_idle_cpu (int cpu,struct task_struct *p) { -if (available_idle_cpu (cpu) ||sched_idle_cpu (cpu)) +if ((available_idle_cpu (cpu) ||sched_idle_cpu (cpu)) && +sched_cpu_cookie_match (cpu_rq (cpu), p))return cpu;return -1 ;
@@ -6051,7 +6056,7 @@ static int select_idle_core(struct task_struct *p, int core, struct cpumask *cpuint cpu;if (!static_branch_likely (&sched_smt_present)) -return __select_idle_cpu (core); +return __select_idle_cpu (core, p);for_each_cpu (cpu,cpu_smt_mask (core)) {if (!available_idle_cpu (cpu)) {
@@ -6107,7 +6112,7 @@ static inline bool test_idle_cores(int cpu, bool def)static inline int select_idle_core (struct task_struct *p,int core,struct cpumask *cpus,int *idle_cpu) { -return __select_idle_cpu (core); +return __select_idle_cpu (core, p); }static inline int select_idle_smt (struct task_struct *p,struct sched_domain *sd,int target)
@@ -6164,7 +6169,7 @@ static int select_idle_cpu(struct task_struct *p, struct sched_domain *sd, bool }else {if (!--nr)return -1 ; - idle_cpu =__select_idle_cpu (cpu); + idle_cpu =__select_idle_cpu (cpu, p);if ((unsigned int )idle_cpu < nr_cpumask_bits)break ; }
@@ -7527,6 +7532,14 @@ static int task_hot(struct task_struct *p, struct lb_env *env)if (sysctl_sched_migration_cost ==-1 )return 1 ; + +/* +* Don't migrate task if the task's cookie does not match +* with the destination CPU's core cookie. +*/ +if (!sched_core_cookie_match (cpu_rq (env->dst_cpu), p)) +return 1 ; +if (sysctl_sched_migration_cost ==0 )return 0 ;
@@ -8857,6 +8870,10 @@ find_idlest_group(struct sched_domain *sd, struct task_struct *p, int this_cpu) p->cpus_ptr))continue ; +/* Skip over this group if no cookie matched */ +if (!sched_group_cookie_match (cpu_rq (this_cpu), p, group)) +continue ; + local_group =cpumask_test_cpu (this_cpu,sched_group_span (group));
kernel/sched/sched.h +73 -0
92 lines not on this page · raw message
Peter Zijlstra
maintainerPATCH 16/19 sched: Trivial core scheduling cookie management
In reply to Peter ZijlstraIn order to not have to use pid_struct, create a new, smaller, structure to manage task cookies for core scheduling.
--- include/linux/sched.h | 6 ++ kernel/fork.c | 1 kernel/sched/Makefile | 1 kernel/sched/core.c | 7 +- kernel/sched/core_sched.c | 109 ++++++++++++++++++++++++++++++++++++++++++++++ kernel/sched/sched.h | 16 ++++++ 6 files changed, 137 insertions(+), 3 deletions(-)
include/linux/sched.h +6 -0
@@ -2170,4 +2170,10 @@ int sched_trace_rq_nr_running(struct rqconst struct cpumask *sched_trace_rd_span (struct root_domain *rd); +#ifdef CONFIG_SCHED_CORE +extern void sched_core_free (struct task_struct *tsk); +#else +static inline void sched_core_free (struct task_struct *tsk) { } +#endif +#endif
kernel/fork.c +1 -0
@@ -737,6 +737,7 @@ void __put_task_struct(struct task_strucexit_creds (tsk);delayacct_tsk_free (tsk);put_signal_struct (tsk->signal); +sched_core_free (tsk);if (!profile_handoff_task (tsk))free_task (tsk);
kernel/sched/Makefile +1 -0
@@ -36,3 +36,4 @@ obj-$(CONFIG_CPU_FREQ_GOV_SCHEDUTIL) += obj-$(CONFIG_MEMBARRIER) += membarrier.o obj-$(CONFIG_CPU_ISOLATION) += isolation.o obj-$(CONFIG_PSI) += psi.o +obj-$(CONFIG_SCHED_CORE) += core_sched.o
kernel/sched/core.c +4 -3
@@ -157,7 +157,7 @@ static inline int rb_sched_core_cmp(consreturn 0 ; } -static void sched_core_enqueue (struct rq *rq,struct task_struct *p) +void sched_core_enqueue (struct rq *rq,struct task_struct *p) { rq->core->core_task_seq++;
@@ -167,14 +167,15 @@ static void sched_core_enqueue(struct rqrb_add (&p->core_node, &rq->core_tree, rb_sched_core_less); } -static void sched_core_dequeue (struct rq *rq,struct task_struct *p) +void sched_core_dequeue (struct rq *rq,struct task_struct *p) { rq->core->core_task_seq++; -if (!p->core_cookie) +if (!sched_core_enqueued (p))return ;rb_erase (&p->core_node, &rq->core_tree); +RB_CLEAR_NODE (&p->core_node); }/*
kernel/sched/core_sched.c new file +109 -0
109 lines not on this page · raw message
kernel/sched/sched.h +16 -0
@@ -1222,6 +1222,22 @@ static inline bool sched_group_cookie_maextern void queue_core_balance (struct rq *rq); +static inline bool sched_core_enqueued (struct task_struct *p) +{ +return !RB_EMPTY_NODE (&p->core_node); +} + +extern void sched_core_enqueue (struct rq *rq,struct task_struct *p); +extern void sched_core_dequeue (struct rq *rq,struct task_struct *p); + +extern void sched_core_get (void ); +extern void sched_core_put (void ); + +extern unsigned long sched_core_alloc_cookie (void ); +extern void sched_core_put_cookie (unsigned long cookie); +extern unsigned long sched_core_get_cookie (unsigned long cookie); +extern unsigned long sched_core_update_cookie (struct task_struct *p,unsigned long cookie); +#else /* !CONFIG_SCHED_CORE */ static inline bool sched_core_enabled (struct rq *rq)
[tip: sched/core] sched: Trivial core scheduling cookie management
The following commit has been merged into the sched/core branch of tip:
The following commit has been merged into the sched/core branch of tip:
Commit-ID: 6e33cad0af49336952e5541464bd02f5b5fd433e Gitweb: https://git.kernel.org/tip/6e33cad0af49336952e5541464bd02f5b5fd433e Author: Peter Zijlstra <[email protected]> AuthorDate: Fri, 26 Mar 2021 18:55:06 +01:00 Committer: Peter Zijlstra <[email protected]> CommitterDate: Wed, 12 May 2021 11:43:31 +02:00
sched: Trivial core scheduling cookie management
In order to not have to use pid_struct, create a new, smaller, structure to manage task cookies for core scheduling.
--- include/linux/sched.h | 6 ++- kernel/fork.c | 1 +- kernel/sched/Makefile | 1 +- kernel/sched/core.c | 7 +- kernel/sched/core_sched.c | 109 +++++++++++++++++++++++++++++++++++++- kernel/sched/sched.h | 16 +++++- 6 files changed, 137 insertions(+), 3 deletions(-) create mode 100644 kernel/sched/core_sched.c
include/linux/sched.h +6 -0
@@ -2179,4 +2179,10 @@ int sched_trace_rq_nr_running(struct rq *rq);const struct cpumask *sched_trace_rd_span (struct root_domain *rd); +#ifdef CONFIG_SCHED_CORE +extern void sched_core_free (struct task_struct *tsk); +#else +static inline void sched_core_free (struct task_struct *tsk) { } +#endif +#endif
kernel/fork.c +1 -0
@@ -742,6 +742,7 @@ void __put_task_struct(struct task_struct *tsk)exit_creds (tsk);delayacct_tsk_free (tsk);put_signal_struct (tsk->signal); +sched_core_free (tsk);if (!profile_handoff_task (tsk))free_task (tsk);
kernel/sched/Makefile +1 -0
@@ -36,3 +36,4 @@ obj-$(CONFIG_CPU_FREQ_GOV_SCHEDUTIL) += cpufreq_schedutil.o obj-$(CONFIG_MEMBARRIER) += membarrier.o obj-$(CONFIG_CPU_ISOLATION) += isolation.o obj-$(CONFIG_PSI) += psi.o +obj-$(CONFIG_SCHED_CORE) += core_sched.o
kernel/sched/core.c +4 -3
@@ -167,7 +167,7 @@ static inline int rb_sched_core_cmp(const void *key, const struct rb_node *node)return 0 ; } -static void sched_core_enqueue (struct rq *rq,struct task_struct *p) +void sched_core_enqueue (struct rq *rq,struct task_struct *p) { rq->core->core_task_seq++;
@@ -177,14 +177,15 @@ static void sched_core_enqueue(struct rq *rq, struct task_struct *p)rb_add (&p->core_node, &rq->core_tree, rb_sched_core_less); } -static void sched_core_dequeue (struct rq *rq,struct task_struct *p) +void sched_core_dequeue (struct rq *rq,struct task_struct *p) { rq->core->core_task_seq++; -if (!p->core_cookie) +if (!sched_core_enqueued (p))return ;rb_erase (&p->core_node, &rq->core_tree); +RB_CLEAR_NODE (&p->core_node); }/*
kernel/sched/core_sched.c new file +109 -0
109 lines not on this page · raw message
kernel/sched/sched.h +16 -0
@@ -1229,6 +1229,22 @@ static inline bool sched_group_cookie_match(struct rq *rq,extern void queue_core_balance (struct rq *rq); +static inline bool sched_core_enqueued (struct task_struct *p) +{ +return !RB_EMPTY_NODE (&p->core_node); +} + +extern void sched_core_enqueue (struct rq *rq,struct task_struct *p); +extern void sched_core_dequeue (struct rq *rq,struct task_struct *p); + +extern void sched_core_get (void ); +extern void sched_core_put (void ); + +extern unsigned long sched_core_alloc_cookie (void ); +extern void sched_core_put_cookie (unsigned long cookie); +extern unsigned long sched_core_get_cookie (unsigned long cookie); +extern unsigned long sched_core_update_cookie (struct task_struct *p,unsigned long cookie); +#else /* !CONFIG_SCHED_CORE */ static inline bool sched_core_enabled (struct rq *rq)
Peter Zijlstra
maintainerPATCH 17/19 sched: Inherit task cookie on fork()
In reply to Peter ZijlstraNote that sched_core_fork() is called from under tasklist_lock, and not from sched_fork() earlier. This avoids a few races later.
--- include/linux/sched.h | 2 ++ kernel/fork.c | 3 +++ kernel/sched/core_sched.c | 6 ++++++ 3 files changed, 11 insertions(+)
include/linux/sched.h +2 -0
@@ -2172,8 +2172,10 @@ const struct cpumask *sched_trace_rd_spa#ifdef CONFIG_SCHED_CORE extern void sched_core_free (struct task_struct *tsk); +extern void sched_core_fork (struct task_struct *p);#else static inline void sched_core_free (struct task_struct *tsk) { } +static inline void sched_core_fork (struct task_struct *p) { }#endif #endif
kernel/fork.c +3 -0
@@ -2249,6 +2249,8 @@ static __latent_entropy struct task_struklp_copy_process (p); +sched_core_fork (p); +spin_lock (¤t->sighand->siglock);/*
@@ -2336,6 +2338,7 @@ static __latent_entropy struct task_strureturn p; bad_fork_cancel_cgroup: +sched_core_free (p);spin_unlock (¤t->sighand->siglock);write_unlock_irq (&tasklist_lock);cgroup_cancel_fork (p, args);
kernel/sched/core_sched.c +6 -0
@@ -100,6 +100,12 @@ static unsigned long sched_core_clone_coreturn cookie; } +void sched_core_fork (struct task_struct *p) +{ +RB_CLEAR_NODE (&p->core_node); + p->core_cookie =sched_core_clone_cookie (current); +} +void sched_core_free (struct task_struct *p) {sched_core_put_cookie (p->core_cookie);
Re: [PATCH 17/19] sched: Inherit task cookie on fork()
Hi Peter,
Quoted 46 lines · Peter Zijlstra
Note that
sched_core_fork()is called from undertasklist_lock, and not fromsched_fork()earlier. This avoids a few races later.Signed-off-by Peter Zijlstra (Intel) <[email protected]>--- include/linux/sched.h | 2 ++ kernel/fork.c | 3 +++ kernel/sched/core_sched.c | 6 ++++++ 3 files changed, 11 insertions(+)include/linux/sched.h +2
-0@@ -2172,8 +2172,10 @@ const struct cpumask *sched_trace_rd_spa#ifdef CONFIG_SCHED_CORE extern void sched_core_free (struct task_struct *tsk); +extern void sched_core_fork (struct task_struct *p);#else static inline void sched_core_free (struct task_struct *tsk) { } +static inline void sched_core_fork (struct task_struct *p) { }#endif #endif kernel/fork.c +3
-0@@ -2249,6 +2249,8 @@ static __latent_entropy struct task_struklp_copy_process (p); +sched_core_fork (p); +spin_lock (¤t->sighand->siglock);/* @@ -2336,6 +2338,7 @@ static __latent_entropy struct task_strureturn p; bad_fork_cancel_cgroup: +sched_core_free (p);spin_unlock (¤t->sighand->siglock);write_unlock_irq (&tasklist_lock);cgroup_cancel_fork (p, args);
kernel/sched/core_sched.c static unsigned long sched_core_clone_co · line 106
Quoted 7 lines
kernel/sched/core_sched.c +4
-0@@ -100,6 +100,12 @@ static unsigned long sched_core_clone_coreturn cookie; } +void sched_core_fork (struct task_struct *p) +{ +RB_CLEAR_NODE (&p->core_node); + p->core_cookie =sched_core_clone_cookie (current);
Does this make sense also for !CLONE_THREAD forks?
With earlier versions of core scheduling, we have done the following on ChromeOS. Basically, if it is a "thread clone", share the cookie since memory is shared within a process (same address space within a process). Otherwise, set the cookie to a new unique cookie so that the new process does not share core with parent initially (since their address space will be different).
Example Psedu-ocode in sched_fork():
if (current->core_cookie && (clone_flags & CLONE_THREAD)) {
p->core_cookie = clone_cookie(current);
} else {
p->core_cookie = create_new_cookie();
}In your version though, I don't see that it always clones the cookie whether it is a CLONE_THREAD clone or not. Is that correct? I feel that's a security issue.
-Joel
Re: [PATCH 17/19] sched: Inherit task cookie on fork()
Quoted 52 lines · Joel Fernandes
Hi Peter,
Quoted 49 lines · Peter Zijlstra
Note that
sched_core_fork()is called from undertasklist_lock, and not fromsched_fork()earlier. This avoids a few races later.Signed-off-by Peter Zijlstra (Intel) <[email protected]>--- include/linux/sched.h | 2 ++ kernel/fork.c | 3 +++ kernel/sched/core_sched.c | 6 ++++++ 3 files changed, 11 insertions(+)include/linux/sched.h +2
-0@@ -2172,8 +2172,10 @@ const struct cpumask *sched_trace_rd_spa#ifdef CONFIG_SCHED_CORE extern void sched_core_free (struct task_struct *tsk); +extern void sched_core_fork (struct task_struct *p);#else static inline void sched_core_free (struct task_struct *tsk) { } +static inline void sched_core_fork (struct task_struct *p) { }#endif #endif kernel/fork.c +3
-0@@ -2249,6 +2249,8 @@ static __latent_entropy struct task_struklp_copy_process (p); +sched_core_fork (p); +spin_lock (¤t->sighand->siglock);/* @@ -2336,6 +2338,7 @@ static __latent_entropy struct task_strureturn p; bad_fork_cancel_cgroup: +sched_core_free (p);spin_unlock (¤t->sighand->siglock);write_unlock_irq (&tasklist_lock);cgroup_cancel_fork (p, args);kernel/sched/core_sched.c +0
-0@@ -100,6 +100,12 @@ static unsigned long sched_core_clone_coreturn cookie; }
Quoted 6 lines
kernel/sched/core_sched.c static unsigned long sched_core_clone_co · line 106
Quoted 4 lines
kernel/sched/core_sched.c +4
-0+void sched_core_fork (struct task_struct *p) +{ +RB_CLEAR_NODE (&p->core_node); + p->core_cookie =sched_core_clone_cookie (current);Does this make sense also for !CLONE_THREAD forks?
Yes. Given the absence of a cgroup interface, fork inheritance (clone the cookie) is the best way to create shared cookie hierarchies. The security issue you mentioned was handled in my original code by setting a unique cookie on 'exec', but Peter took that out for the reason mentioned above. It was part of the "lets get this in compromise" effort.
-chrish
Re: [PATCH 17/19] sched: Inherit task cookie on fork()
Quoted 61 lines · Chris Hyser
Quoted 52 lines · Joel Fernandes
Hi Peter,
Quoted 49 lines · Peter Zijlstra
Note that
sched_core_fork()is called from undertasklist_lock, and not fromsched_fork()earlier. This avoids a few races later.Signed-off-by Peter Zijlstra (Intel) <[email protected]>--- include/linux/sched.h | 2 ++ kernel/fork.c | 3 +++ kernel/sched/core_sched.c | 6 ++++++ 3 files changed, 11 insertions(+)include/linux/sched.h +2
-0@@ -2172,8 +2172,10 @@ const struct cpumask *sched_trace_rd_spa#ifdef CONFIG_SCHED_CORE extern void sched_core_free (struct task_struct *tsk); +extern void sched_core_fork (struct task_struct *p);#else static inline void sched_core_free (struct task_struct *tsk) { } +static inline void sched_core_fork (struct task_struct *p) { }#endif #endif kernel/fork.c +3
-0@@ -2249,6 +2249,8 @@ static __latent_entropy struct task_struklp_copy_process (p); +sched_core_fork (p); +spin_lock (¤t->sighand->siglock);/* @@ -2336,6 +2338,7 @@ static __latent_entropy struct task_strureturn p; bad_fork_cancel_cgroup: +sched_core_free (p);spin_unlock (¤t->sighand->siglock);write_unlock_irq (&tasklist_lock);cgroup_cancel_fork (p, args);kernel/sched/core_sched.c +0
-0@@ -100,6 +100,12 @@ static unsigned long sched_core_clone_coreturn cookie; }Quoted 6 lines
kernel/sched/core_sched.c static unsigned long sched_core_clone_co · line 106
Quoted 4 lines
kernel/sched/core_sched.c +4
-0+void sched_core_fork (struct task_struct *p) +{ +RB_CLEAR_NODE (&p->core_node); + p->core_cookie =sched_core_clone_cookie (current);Does this make sense also for !CLONE_THREAD forks?
Quoted 3 lines
Yes. Given the absence of a cgroup interface, fork inheritance (clone the cookie) is the best way to create shared cookie hierarchies. The security issue you mentioned was handled in my original code by setting a unique cookie on 'exec', but Peter took that out for the reason mentioned above. It was part of the "lets get this in compromise" effort.
Thanks for sharing the history of it. I guess one can argue that this policy is better to be hardcoded in userspace since core-scheduling can be used for non-security usecases as well. Maybe one could simply call the prctl(2) from userspace if they so desire, before calling exec() ?
- Joel
Re: [PATCH 17/19] sched: Inherit task cookie on fork()
Quoted 13 lines · Joel Fernandes
Quoted 10 lines · Chris Hyser
Quoted 6 lines
kernel/sched/core_sched.c static unsigned long sched_core_clone_co · line 106
Quoted 4 lines
kernel/sched/core_sched.c +4
-0+void sched_core_fork (struct task_struct *p) +{ +RB_CLEAR_NODE (&p->core_node); + p->core_cookie =sched_core_clone_cookie (current);Does this make sense also for !CLONE_THREAD forks?
Yes. Given the absence of a cgroup interface, fork inheritance (clone the cookie) is the best way to create shared cookie hierarchies. The security issue you mentioned was handled in my original code by setting a unique cookie on 'exec', but Peter took that out for the reason mentioned above. It was part of the "lets get this in compromise" effort.
Quoted 5 lines
Thanks for sharing the history of it. I guess one can argue that this policy is better to be hardcoded in userspace since core-scheduling can be used for non-security usecases as well. Maybe one could simply call the prctl(2) from userspace if they so desire, before calling
exec()?
I think the defining use case is a container's init. If the cookie is set for it by the container creator and without any other user code knowing about core_sched, every descendant spawned will have the same cookie and be in the same core_sched group much like the cgroup interface had provided. If we create a unique cookie in the kernel either on fork or exec, we are secure, but we will now have 1000's of core sched groups.
CLEAR was also removed (temporarily, I hope) because a core_sched knowledgeable program in the example core_sched container group should not be able to remove itself from _all_ core sched groups. It can modify it's cookie, but that is no different than the normal case.
Both of these beg for a kernel policy, but that discussion was TBD.
-chrish
Peter Zijlstra
maintainerRe: [PATCH 17/19] sched: Inherit task cookie on fork()
Quoted 10 lines · Chris Hyser
Quoted 9 lines · Joel Fernandes
Quoted 7 lines · Chris Hyser
Quoted 6 lines
kernel/sched/core_sched.c static unsigned long sched_core_clone_co · line 106
Quoted 4 lines
kernel/sched/core_sched.c +4
-0+void sched_core_fork (struct task_struct *p) +{ +RB_CLEAR_NODE (&p->core_node); + p->core_cookie =sched_core_clone_cookie (current);Does this make sense also for !CLONE_THREAD forks?
Quoted 6 lines
Quoted 6 lines
Quoted 6 lines
Yes. Given the absence of a cgroup interface, fork inheritance (clone the cookie) is the best way to create shared cookie hierarchies. The security issue you mentioned was handled in my original code by setting a unique cookie on 'exec', but Peter took that out for the reason mentioned above. It was part of the "lets get this in compromise" effort.
Right, not only that, given all this is moot when parent and child have the same PTRACE permissions, since if they do, they can inspect one another's innards anyway, exec()/fork() just fundamentally isn't a magical boundary we should care about.
The only special case there is SUID exec(), because in that case the actual credentials change and the PTRACE permissions do actually change.
I sorta had a patch to do that, but it's yuck because that cred change happens after the point of no return and we need an allocation for the new cookie. Now, we could rely on the fact that a task context allocation (GFP_KERNEL) for something as small as our cookie will never fail and hence we shouldn't be bothered by it, we should do the error path and yuck.
Quoted 13 lines
Quoted 5 lines
Thanks for sharing the history of it. I guess one can argue that this policy is better to be hardcoded in userspace since core-scheduling can be used for non-security usecases as well. Maybe one could simply call the prctl(2) from userspace if they so desire, before calling
exec()?I think the defining use case is a container's init. If the cookie is set for it by the container creator and without any other user code knowing about
core_sched, every descendant spawned will have the same cookie and be in the samecore_schedgroup much like the cgroup interface had provided. If we create a unique cookie in the kernel either on fork or exec, we are secure, but we will now have 1000's of core sched groups.
Quoted 4 lines
CLEAR was also removed (temporarily, I hope) because a
core_schedknowledgeable program in the examplecore_schedcontainer group should not be able to remove itself from _all_ core sched groups. It can modify it's cookie, but that is no different than the normal case.
Note that much of clear is possible by using SHARE_FROM on your parent to reset the cookie.
Quoted 1 line
Both of these beg for a kernel policy, but that discussion was TBD.
Right, I need a Champion that actually cares about cgroups and has use-cases to go argue with TJ on this. I've proposed code that I think has sane semantics, but I'm not in a position to argue for it, given I think a world without cgroups is a better world :-)))
Re: [PATCH 17/19] sched: Inherit task cookie on fork()
Quoted 4 lines · Peter Zijlstra
Right, I need a Champion that actually cares about cgroups and has use-cases to go argue with TJ on this. I've proposed code that I think has sane semantics, but I'm not in a position to argue for it, given I think a world without cgroups is a better world :-)))
Not sure if Tejun has any thoughts on
http://lkml.kernel.org/r/CABk29NtahuW6UERvRdK5v8My_MfPsoESDKXUjGdvaQcHOJEMvg@mail.gmail.com.
We're looking at using the prctl interface with one of our main internal users of core scheduling. As an example, suppose we have a management process that wants to make tasks A and B share a cookie:
- Spawn a new thread m, which then does the following, and exits.
- PR_SCHED_CORE_CREATE for just its own PID
- PR_SCHED_CORE_SHARE_TO A
- PR_SCHED_CORE_SHARE_TO B
That seems to work ok; I'll follow up if there are any pain points that aren't easily addressed with the prctl interface.
Re: [PATCH 17/19] sched: Inherit task cookie on fork()
Quoted 18 lines · Josh Don
Quoted 4 lines · Peter Zijlstra
Right, I need a Champion that actually cares about cgroups and has use-cases to go argue with TJ on this. I've proposed code that I think has sane semantics, but I'm not in a position to argue for it, given I think a world without cgroups is a better world :-)))
Not sure if Tejun has any thoughts on
http://lkml.kernel.org/r/CABk29NtahuW6UERvRdK5v8My_MfPsoESDKXUjGdvaQcHOJEMvg@mail.gmail.com.We're looking at using the prctl interface with one of our main internal users of core scheduling. As an example, suppose we have a management process that wants to make tasks A and B share a cookie:
- Spawn a new thread m, which then does the following, and exits.
-PR_SCHED_CORE_CREATEfor just its own PID
-PR_SCHED_CORE_SHARE_TOA
-PR_SCHED_CORE_SHARE_TOB
Quoted 2 lines
That seems to work ok; I'll follow up if there are any pain points that aren't easily addressed with the prctl interface.
This is exactly what I'm doing to tag all the processes for qemu. I have as shell script that creates a cookie for $BASHPID and then SHARE_TO all the process and it works just great. :)
don
[tip: sched/core] sched: Inherit task cookie on fork()
In reply to Peter ZijlstraThe following commit has been merged into the sched/core branch of tip:
The following commit has been merged into the sched/core branch of tip:
Commit-ID: 85dd3f61203c5cfa72b308ff327b5fbf3fc1ce5e Gitweb: https://git.kernel.org/tip/85dd3f61203c5cfa72b308ff327b5fbf3fc1ce5e Author: Peter Zijlstra <[email protected]> AuthorDate: Mon, 29 Mar 2021 15:18:35 +02:00 Committer: Peter Zijlstra <[email protected]> CommitterDate: Wed, 12 May 2021 11:43:31 +02:00
sched: Inherit task cookie on fork()
Note that sched_core_fork() is called from under tasklist_lock, and not from sched_fork() earlier. This avoids a few races later.
--- include/linux/sched.h | 2 ++ kernel/fork.c | 3 +++ kernel/sched/core_sched.c | 6 ++++++ 3 files changed, 11 insertions(+)
include/linux/sched.h +2 -0
@@ -2181,8 +2181,10 @@ const struct cpumask *sched_trace_rd_span(struct root_domain *rd);#ifdef CONFIG_SCHED_CORE extern void sched_core_free (struct task_struct *tsk); +extern void sched_core_fork (struct task_struct *p);#else static inline void sched_core_free (struct task_struct *tsk) { } +static inline void sched_core_fork (struct task_struct *p) { }#endif #endif
kernel/fork.c +3 -0
@@ -2251,6 +2251,8 @@ static __latent_entropy struct task_struct *copy_process(klp_copy_process (p); +sched_core_fork (p); +spin_lock (¤t->sighand->siglock);/*
@@ -2338,6 +2340,7 @@ static __latent_entropy struct task_struct *copy_process(return p; bad_fork_cancel_cgroup: +sched_core_free (p);spin_unlock (¤t->sighand->siglock);write_unlock_irq (&tasklist_lock);cgroup_cancel_fork (p, args);
kernel/sched/core_sched.c +6 -0
@@ -103,6 +103,12 @@ static unsigned long sched_core_clone_cookie(struct task_struct *p)return cookie; } +void sched_core_fork (struct task_struct *p) +{ +RB_CLEAR_NODE (&p->core_node); + p->core_cookie =sched_core_clone_cookie (current); +} +void sched_core_free (struct task_struct *p) {sched_core_put_cookie (p->core_cookie);
Peter Zijlstra
maintainerPATCH 18/19 sched: prctl() core-scheduling interface
In reply to Peter ZijlstraFrom: Chris Hyser <[email protected]>
This patch provides support for setting and copying core scheduling 'task cookies' between threads (PID), processes (TGID), and process groups (PGID).
The value of core scheduling isn't that tasks don't share a core, 'nosmt' can do that. The value lies in exploiting all the sharing opportunities that exist to recover possible lost performance and that requires a degree of flexibility in the API.
Quoted 1 line
From a security perspective (and there are others), the thread,
process and process group distinction is an existent hierarchal categorization of tasks that reflects many of the security concerns about 'data sharing'. For example, protecting against cache-snooping by a thread that can just read the memory directly isn't all that useful.
With this in mind, subcommands to CREATE/SHARE (TO/FROM) provide a mechanism to create and share cookies. CREATE/SHARE_TO specify a target pid with enum pidtype used to specify the scope of the targeted tasks. For example, PIDTYPE_TGID will share the cookie with the process and all of it's threads as typically desired in a security scenario.
API:
prctl(PR_SCHED_CORE, PR_SCHED_CORE_GET, tgtpid, pidtype, &cookie) prctl(PR_SCHED_CORE, PR_SCHED_CORE_CREATE, tgtpid, pidtype, NULL) prctl(PR_SCHED_CORE, PR_SCHED_CORE_SHARE_TO, tgtpid, pidtype, NULL) prctl(PR_SCHED_CORE, PR_SCHED_CORE_SHARE_FROM, srcpid, pidtype, NULL)
where 'tgtpid/srcpid == 0' implies the current process and pidtype is kernel enum pid_type {PIDTYPE_PID, PIDTYPE_TGID, PIDTYPE_PGID, ...}.
For return values, EINVAL, ENOMEM are what they say. ESRCH means the tgtpid/srcpid was not found. EPERM indicates lack of PTRACE permission access to tgtpid/srcpid. ENODEV indicates your machines lacks SMT.
--- include/linux/sched.h | 2 include/uapi/linux/prctl.h | 8 ++ kernel/sched/core_sched.c | 114 +++++++++++++++++++++++++++++++++++++++ kernel/sys.c | 5 + tools/include/uapi/linux/prctl.h | 8 ++ 5 files changed, 137 insertions(+)
include/linux/sched.h +2 -0
@@ -2173,6 +2173,8 @@ const struct cpumask *sched_trace_rd_spa#ifdef CONFIG_SCHED_CORE extern void sched_core_free (struct task_struct *tsk);extern void sched_core_fork (struct task_struct *p); +extern int sched_core_share_pid (unsigned int cmd,pid_t pid,enum pid_type type, +unsigned long uaddr);#else static inline void sched_core_free (struct task_struct *tsk) { }static inline void sched_core_fork (struct task_struct *p) { }
include/uapi/linux/prctl.h +8 -0
@@ -255,4 +255,12 @@ struct prctl_mm_map {# define SYSCALL_DISPATCH_FILTER_ALLOW 0# define SYSCALL_DISPATCH_FILTER_BLOCK 1 +/* Request the scheduler to share a core */ +#define PR_SCHED_CORE 60 +# define PR_SCHED_CORE_GET 0 +# define PR_SCHED_CORE_CREATE 1/* create unique core_sched cookie */ +# define PR_SCHED_CORE_SHARE_TO 2/* push core_sched cookie to pid */ +# define PR_SCHED_CORE_SHARE_FROM 3/* pull core_sched cookie to pid */ +# define PR_SCHED_CORE_MAX 4 +#endif /* _LINUX_PRCTL_H */
kernel/sched/core_sched.c +114 -0
122 lines not on this page · raw message
kernel/sys.c +5 -0
@@ -2534,6 +2534,11 @@ SYSCALL_DEFINE5(prctl, int, option, unsi error =set_syscall_user_dispatch (arg2, arg3, arg4, (char __user *) arg5);break ; +#ifdef CONFIG_SCHED_CORE +case PR_SCHED_CORE : + error =sched_core_share_pid (arg2, arg3, arg4, arg5); +break ; +#endif default : error = -EINVAL ;break ;
tools/include/uapi/linux/prctl.h +8 -0
@@ -255,4 +255,12 @@ struct prctl_mm_map {# define SYSCALL_DISPATCH_FILTER_ALLOW 0# define SYSCALL_DISPATCH_FILTER_BLOCK 1 +/* Request the scheduler to share a core */ +#define PR_SCHED_CORE 60 +# define PR_SCHED_CORE_GET 0 +# define PR_SCHED_CORE_CREATE 1/* create unique core_sched cookie */ +# define PR_SCHED_CORE_SHARE_TO 2/* push core_sched cookie to pid */ +# define PR_SCHED_CORE_SHARE_FROM 3/* pull core_sched cookie to pid */ +# define PR_SCHED_CORE_MAX 4 +#endif /* _LINUX_PRCTL_H */
[tip: sched/core] sched: prctl() core-scheduling interface
The following commit has been merged into the sched/core branch of tip:
The following commit has been merged into the sched/core branch of tip:
Commit-ID: 7ac592aa35a684ff1858fb9ec282886b9e3575ac Gitweb: https://git.kernel.org/tip/7ac592aa35a684ff1858fb9ec282886b9e3575ac Author: Chris Hyser <[email protected]> AuthorDate: Wed, 24 Mar 2021 17:40:15 -04:00 Committer: Peter Zijlstra <[email protected]> CommitterDate: Wed, 12 May 2021 11:43:31 +02:00
sched: prctl() core-scheduling interface
This patch provides support for setting and copying core scheduling 'task cookies' between threads (PID), processes (TGID), and process groups (PGID).
The value of core scheduling isn't that tasks don't share a core, 'nosmt' can do that. The value lies in exploiting all the sharing opportunities that exist to recover possible lost performance and that requires a degree of flexibility in the API.
Quoted 1 line
From a security perspective (and there are others), the thread,
process and process group distinction is an existent hierarchal categorization of tasks that reflects many of the security concerns about 'data sharing'. For example, protecting against cache-snooping by a thread that can just read the memory directly isn't all that useful.
With this in mind, subcommands to CREATE/SHARE (TO/FROM) provide a mechanism to create and share cookies. CREATE/SHARE_TO specify a target pid with enum pidtype used to specify the scope of the targeted tasks. For example, PIDTYPE_TGID will share the cookie with the process and all of it's threads as typically desired in a security scenario.
API:
prctl(PR_SCHED_CORE, PR_SCHED_CORE_GET, tgtpid, pidtype, &cookie) prctl(PR_SCHED_CORE, PR_SCHED_CORE_CREATE, tgtpid, pidtype, NULL) prctl(PR_SCHED_CORE, PR_SCHED_CORE_SHARE_TO, tgtpid, pidtype, NULL) prctl(PR_SCHED_CORE, PR_SCHED_CORE_SHARE_FROM, srcpid, pidtype, NULL)
where 'tgtpid/srcpid == 0' implies the current process and pidtype is kernel enum pid_type {PIDTYPE_PID, PIDTYPE_TGID, PIDTYPE_PGID, ...}.
For return values, EINVAL, ENOMEM are what they say. ESRCH means the tgtpid/srcpid was not found. EPERM indicates lack of PTRACE permission access to tgtpid/srcpid. ENODEV indicates your machines lacks SMT.
--- include/linux/sched.h | 2 +- include/uapi/linux/prctl.h | 8 ++- kernel/sched/core_sched.c | 114 ++++++++++++++++++++++++++++++- kernel/sys.c | 5 +- tools/include/uapi/linux/prctl.h | 8 ++- 5 files changed, 137 insertions(+)
include/linux/sched.h +2 -0
@@ -2182,6 +2182,8 @@ const struct cpumask *sched_trace_rd_span(struct root_domain *rd);#ifdef CONFIG_SCHED_CORE extern void sched_core_free (struct task_struct *tsk);extern void sched_core_fork (struct task_struct *p); +extern int sched_core_share_pid (unsigned int cmd,pid_t pid,enum pid_type type, +unsigned long uaddr);#else static inline void sched_core_free (struct task_struct *tsk) { }static inline void sched_core_fork (struct task_struct *p) { }
include/uapi/linux/prctl.h +8 -0
@@ -259,4 +259,12 @@ struct prctl_mm_map {#define PR_PAC_SET_ENABLED_KEYS 60#define PR_PAC_GET_ENABLED_KEYS 61 +/* Request the scheduler to share a core */ +#define PR_SCHED_CORE 62 +# define PR_SCHED_CORE_GET 0 +# define PR_SCHED_CORE_CREATE 1/* create unique core_sched cookie */ +# define PR_SCHED_CORE_SHARE_TO 2/* push core_sched cookie to pid */ +# define PR_SCHED_CORE_SHARE_FROM 3/* pull core_sched cookie to pid */ +# define PR_SCHED_CORE_MAX 4 +#endif /* _LINUX_PRCTL_H */
kernel/sched/core_sched.c +114 -0
122 lines not on this page · raw message
kernel/sys.c +5 -0
@@ -2550,6 +2550,11 @@ SYSCALL_DEFINE5(prctl, int, option, unsigned long, arg2, unsigned long, arg3, error =set_syscall_user_dispatch (arg2, arg3, arg4, (char __user *) arg5);break ; +#ifdef CONFIG_SCHED_CORE +case PR_SCHED_CORE : + error =sched_core_share_pid (arg2, arg3, arg4, arg5); +break ; +#endif default : error = -EINVAL ;break ;
tools/include/uapi/linux/prctl.h +8 -0
@@ -259,4 +259,12 @@ struct prctl_mm_map {#define PR_PAC_SET_ENABLED_KEYS 60#define PR_PAC_GET_ENABLED_KEYS 61 +/* Request the scheduler to share a core */ +#define PR_SCHED_CORE 62 +# define PR_SCHED_CORE_GET 0 +# define PR_SCHED_CORE_CREATE 1/* create unique core_sched cookie */ +# define PR_SCHED_CORE_SHARE_TO 2/* push core_sched cookie to pid */ +# define PR_SCHED_CORE_SHARE_FROM 3/* pull core_sched cookie to pid */ +# define PR_SCHED_CORE_MAX 4 +#endif /* _LINUX_PRCTL_H */
Re: [PATCH 18/19] sched: prctl() core-scheduling interface
In reply to Peter ZijlstraQuoted 6 lines · Peter Zijlstra
From: Chris Hyser <[email protected]>
This patch provides support for setting and copying core scheduling 'task cookies' between threads (PID), processes (TGID), and process groups (PGID).
[snip]
Internally, we have lots of trusted processes that don't have a security need for coresched cookies. However, these processes could still decide to create cookies for themselves, which will degrade machine capacity and performance for other jobs on the machine.
Any thoughts on whether it would be desirable to have the ability to restrict use of SCHED_CORE_CREATE? Perhaps a new SCHED_CORE capability would be appropriate?
- Josh
Re: [PATCH 18/19] sched: prctl() core-scheduling interface
Quoted 11 lines · Josh Don
Quoted 6 lines · Peter Zijlstra
From: Chris Hyser <[email protected]>
This patch provides support for setting and copying core scheduling 'task cookies' between threads (PID), processes (TGID), and process groups (PGID).
[snip]
Quoted 8 lines
Internally, we have lots of trusted processes that don't have a security need for coresched cookies. However, these processes could still decide to create cookies for themselves, which will degrade machine capacity and performance for other jobs on the machine.
Any thoughts on whether it would be desirable to have the ability to restrict use of
SCHED_CORE_CREATE? Perhaps a newSCHED_COREcapability would be appropriate?
Hi,
Maybe a capability may not work because then other users who don't care for the issue you mention will be required to manage/assign the capability as well?
How about you use seccomp to filter the prctl based on the PID, and CREATE command?
-Joel
Re: [PATCH 18/19] sched: prctl() core-scheduling interface
In reply to Peter ZijlstraQuoted 9 lines · Peter Zijlstra
API:
prctl(PR_SCHED_CORE, PR_SCHED_CORE_GET, tgtpid, pidtype, &cookie)prctl(PR_SCHED_CORE, PR_SCHED_CORE_CREATE, tgtpid, pidtype, NULL)prctl(PR_SCHED_CORE, PR_SCHED_CORE_SHARE_TO, tgtpid, pidtype, NULL)prctl(PR_SCHED_CORE, PR_SCHED_CORE_SHARE_FROM, srcpid, pidtype, NULL)where 'tgtpid/srcpid == 0' implies the current process and pidtype is kernel enum
pid_type{PIDTYPE_PID,PIDTYPE_TGID,PIDTYPE_PGID, ...}.
It means that enum pid_tipe has to be a part of UAPI now. Would you like to address it, or rather I'd send a patch?
Peter Zijlstra
maintainerRe: [PATCH 18/19] sched: prctl() core-scheduling interface
Quoted 8 lines · Eugene Syromiatnikov
Quoted 7 lines · Peter Zijlstra
API:
prctl(PR_SCHED_CORE, PR_SCHED_CORE_GET, tgtpid, pidtype, &cookie)prctl(PR_SCHED_CORE, PR_SCHED_CORE_CREATE, tgtpid, pidtype, NULL)prctl(PR_SCHED_CORE, PR_SCHED_CORE_SHARE_TO, tgtpid, pidtype, NULL)prctl(PR_SCHED_CORE, PR_SCHED_CORE_SHARE_FROM, srcpid, pidtype, NULL)
Quoted 5 lines
Quoted 2 lines
where 'tgtpid/srcpid == 0' implies the current process and pidtype is kernel enum
pid_type{PIDTYPE_PID,PIDTYPE_TGID,PIDTYPE_PGID, ...}.It means that enum
pid_tipehas to be a part of UAPI now. Would you like to address it, or rather I'd send a patch?
Please send a patch; I'm more sparse than usual atm.
Re: [PATCH 18/19] sched: prctl() core-scheduling interface
In reply to Peter ZijlstraQuoted 5 lines · Peter Zijlstra
From: Chris Hyser <[email protected]>
This patch provides support for setting and copying core scheduling 'task cookies' between threads (PID), processes (TGID), and process groups (PGID).
Hello.
It seems that there is some issue within the scheduler code that can be triggered via this interface:
# gcc -std=gnu99 -Wextra -Werror prctl-sched-core-oops-repro.c -o prctl-sched-core-oops-repro
# ../src/strace -fvq -eprctl,clone,setsid ./prctl-sched-core-oops-repro
clone(child_stack=NULL, flags=CLONE_CHILD_CLEARTID|CLONE_CHILD_SETTID|SIGCHLD, child_tidptr=0x7f271f875890) = 239820
[pid 239820] setsid() = 239820
[pid 239820] +++ exited with 0 +++
--- SIGCHLD {si_signo=SIGCHLD, si_code=CLD_EXITED, si_pid=239820, si_uid=0, si_status=0, si_utime=0, si_stime=0} ---
Iteration 0 status: 0
prctl(PR_SCHED_CORE, PR_SCHED_CORE_CREATE, 239816, 0x2 /* PIDTYPE_PGID */, NULL) = 0
clone(child_stack=NULL, flags=CLONE_CHILD_CLEARTID|CLONE_CHILD_SETTID|SIGCHLD, child_tidptr=0x7f271f875890) = 239821
[pid 239821] setsid() = ?
[pid 239821] +++ killed by SIGKILL +++
--- SIGCHLD {si_signo=SIGCHLD, si_code=CLD_KILLED, si_pid=239821, si_uid=0, si_status=SIGKILL, si_utime=0, si_stime=0} ---
Iteration 1 status: 0x9
prctl(PR_SCHED_CORE, PR_SCHED_CORE_CREATE, 239816, 0x2 /* PIDTYPE_PGID */, NULL) = 0
+++ exited with 0 +++kmsg indicates that a NULL pointer dereference has occurred:
[76195.611570] BUG: kernel NULL pointer dereference, address: 0000000000000000
...
[76195.621771] RIP: 0010:do_raw_spin_trylock+0x5/0x40
...
[76195.640144] Call Trace:
[76195.640706] _raw_spin_lock_nested+0x37/0x80
[76195.641645] ? raw_spin_rq_lock_nested+0x4b/0x80
[76195.642693] raw_spin_rq_lock_nested+0x4b/0x80
[76195.643669] online_fair_sched_group+0x39/0x240
[76195.644663] sched_autogroup_create_attach+0x9d/0x170
[76195.645765] ksys_setsid+0xe6/0x110
[76195.646533] __do_sys_setsid+0xa/0x10
[76195.647358] do_syscall_64+0x3b/0x90
[76195.648219] entry_SYSCALL_64_after_hwframe+0x44/0xaeThe full kmsg excerpt and the reproducer code are attached.
There's also additional "BUG: sleeping function called from invalid context at include/linux/percpu-rwsem.h:49" message is produced (see the full log in the attached file "prctl-sched-core-oops-bug-dmesg.log") when the full test case[1] is run, but I haven't been successful so far in producing a minimal reproduced for it.
[1] https://github.com/strace/strace/commit/a90a5a56d2b76ba3ebd417472a02f40d3d6599d8
Run with ./bootstrap && ./configure CFLAGS='-g -Og' --enable-gcc-Werror && make check TESTS=prctl-sched-core--pidns-translation.gen
#include <stdio.h>
#include <linux/prctl.h>
#include <sys/prctl.h>
#include <sys/wait.h>
#include <unistd.h>
int main()
{
int pid;
int pgid = getpgid(0);
for (size_t i = 0; i < 2; i++) {
if ((pid = fork())) {
int status;
wait(&status);
printf("Iteration %d status: %#x\n", i, status);
} else {
setsid();
return 0;
}
prctl(PR_SCHED_CORE, PR_SCHED_CORE_CREATE, pgid, 0x2 /* PIDTYPE_PGID */, NULL);
}
return 0;
}[76195.611570] BUG: kernel NULL pointer dereference, address: 0000000000000000 [76195.613059] #PF: supervisor read access in kernel mode [76195.614174] #PF: error_code(0x0000) - not-present page [76195.615329] PGD 800000005f27e067 P4D 800000005f27e067 PUD 3f7a3067 PMD 0 [76195.616801] Oops: 0000 [#67] SMP PTI [76195.617586] CPU: 2 PID: 239821 Comm: prctl-sched-cor Tainted: G D W --------- --- 5.14.0-0.rc5.20210813gitf8e6dfc64f61.46.fc36.x86_64 #1 [76195.620374] Hardware name: HP ProLiant BL480c G1, BIOS I14 10/04/2007 [76195.621771] RIP: 0010:do_raw_spin_trylock+0x5/0x40 [76195.622821] Code: c6 a4 12 5f 9f 48 89 ef e8 c8 fe ff ff eb a9 89 c6 48 89 ef e8 0c f5 ff ff 66 90 eb a9 0f 1f 84 00 00 00 00 00 0f 1f 44 00 00 <8b> 07 85 c0 75 28 ba 01 00 00 00 f0 0f b1 17 75 1d 65 8b 05 fb 98 [76195.626797] RSP: 0018:ffffa366014abe58 EFLAGS: 00010086 [76195.627936] RAX: 0000000000000001 RBX: 0000000000000004 RCX: 0000000000000000 [76195.629470] RDX: 0000000000000000 RSI: 0000000000000000 RDI: 0000000000000000 [76195.631048] RBP: 0000000000000000 R08: 0000000000000001 R09: 0000000000000000 [76195.632585] R10: 0000000000000000 R11: ffff98292b21ad48 R12: 0000000000000018 [76195.634078] R13: 0000000000000000 R14: ffff98292b7ef940 R15: ffff982813938e00 [76195.635621] FS: 00007f271f8755c0(0000) GS:ffff98292b200000(0000) knlGS:0000000000000000 [76195.637354] CS: 0010 DS: 0000 ES: 0000 CR0: 0000000080050033 [76195.638606] CR2: 0000000000000000 CR3: 000000000fdd0000 CR4: 00000000000006e0 [76195.640144] Call Trace: [76195.640706] _raw_spin_lock_nested+0x37/0x80 [76195.641645] ? raw_spin_rq_lock_nested+0x4b/0x80 [76195.642693] raw_spin_rq_lock_nested+0x4b/0x80 [76195.643669] online_fair_sched_group+0x39/0x240 [76195.644663] sched_autogroup_create_attach+0x9d/0x170 [76195.645765] ksys_setsid+0xe6/0x110 [76195.646533] __do_sys_setsid+0xa/0x10 [76195.647358] do_syscall_64+0x3b/0x90 [76195.648219] entry_SYSCALL_64_after_hwframe+0x44/0xae [76195.649401] RIP: 0033:0x7f271f7496cb [76195.650241] Code: 73 01 c3 48 8b 0d 5d a7 11 00 f7 d8 64 89 01 48 83 c8 ff c3 66 2e 0f 1f 84 00 00 00 00 00 90 f3 0f 1e fa b8 70 00 00 00 0f 05 <48> 3d 01 f0 ff ff 73 01 c3 48 8b 0d 2d a7 11 00 f7 d8 64 89 01 48 [76195.654522] RSP: 002b:00007ffed2ea9888 EFLAGS: 00000206 ORIG_RAX: 0000000000000070 [76195.656275] RAX: ffffffffffffffda RBX: 0000000000000000 RCX: 00007f271f7496cb [76195.657934] RDX: 0000000000000001 RSI: 0000000000000000 RDI: 0000000000000001 [76195.659566] RBP: 00007ffed2ea98b0 R08: 0000000000000000 R09: 00007f271f8755c0 [76195.661227] R10: 00007f271f67ca60 R11: 0000000000000206 R12: 00007ffed2ea99e8 [76195.662869] R13: 0000000000401176 R14: 00007f271f8adc00 R15: 0000000000403e18 [76195.664510] Modules linked in: rfkill iTCO_wdt intel_pmc_bxt iTCO_vendor_support kvm_intel gpio_ich kvm snd_pcsp snd_pcm snd_timer ipmi_ssif irqbypass snd soundcore lpc_ich acpi_ipmi e1000e tg3 hpilo ipmi_si bnx2 i5000_edac ipmi_devintf i5k_amb ipmi_msghandler fuse zram ip_tables xfs radeon i2c_algo_bit drm_ttm_helper ttm drm_kms_helper cec drm hpsa serio_raw hpwdt scsi_transport_sas [76195.672185] CR2: 0000000000000000 [76195.672973] ---[ end trace ab4b5b9489802ac5 ]--- [76195.674050] RIP: 0010:__lock_acquire+0x5c1/0x1e00 [76195.675146] Code: 00 00 83 f8 2f 0f 87 59 05 00 00 3b 05 cc 46 f6 02 41 bd 01 00 00 00 0f 86 03 01 00 00 89 05 ba 46 f6 02 e9 f8 00 00 00 31 c9 <48> 81 3f 20 8e cc a0 41 0f 45 c8 83 fe 01 0f 87 6f fa ff ff 89 f0 [76195.679386] RSP: 0018:ffffa36602d5fd40 EFLAGS: 00010046 [76195.680606] RAX: 0000000000000046 RBX: 0000000000000001 RCX: 0000000000000000 [76195.682239] RDX: 0000000000000000 RSI: 0000000000000000 RDI: 0000000000000018 [76195.683881] RBP: 0000000000000018 R08: 0000000000000001 R09: 0000000000000001 [76195.685554] R10: 0000000000000001 R11: 0000000000000000 R12: ffff98285fad33c0 [76195.687197] R13: 0000000000000000 R14: 0000000000000000 R15: 0000000000000000 [76195.688840] FS: 00007f271f8755c0(0000) GS:ffff98292b200000(0000) knlGS:0000000000000000 [76195.690690] CS: 0010 DS: 0000 ES: 0000 CR0: 0000000080050033 [76195.692034] CR2: 0000000000000000 CR3: 000000000fdd0000 CR4: 00000000000006e0 [76195.693677] note: prctl-sched-cor[239821] exited with preempt_count 2 [31148.057893] BUG: kernel NULL pointer dereference, address: 0000000000000000 [31148.059390] #PF: supervisor read access in kernel mode [31148.060530] #PF: error_code(0x0000) - not-present page [31148.061601] PGD 800000005f693067 P4D 800000005f693067 PUD 114fe067 PMD 0 [31148.063082] Oops: 0000 [#21] SMP PTI [31148.063880] CPU: 0 PID: 214019 Comm: prctl-sched-cor Tainted: G D W --------- --- 5.14.0-0.rc5.20210813gitf8e6dfc64f61.46.fc36.x86_64 #1 [31148.067019] Hardware name: HP ProLiant BL480c G1, BIOS I14 10/04/2007 [31148.068536] RIP: 0010:do_raw_spin_trylock+0x5/0x40 [31148.069629] Code: c6 a4 12 5f 9f 48 89 ef e8 c8 fe ff ff eb a9 89 c6 48 89 ef e8 0c f5 ff ff 66 90 eb a9 0f 1f 84 00 00 00 00 00 0f 1f 44 00 00 <8b> 07 85 c0 75 28 ba 01 00 00 00 f0 0f b1 17 75 1d 65 8b 05 fb 98 [31148.073601] RSP: 0018:ffffa366030ffe58 EFLAGS: 00010086 [31148.074740] RAX: 0000000000000001 RBX: 0000000000000004 RCX: 0000000000000000 [31148.076235] RDX: 0000000000000000 RSI: 0000000000000000 RDI: 0000000000000000 [31148.077770] RBP: 0000000000000000 R08: 0000000000000001 R09: 0000000000000000 [31148.079264] R10: 0000000000000000 R11: 000000000121ee36 R12: 0000000000000018 [31148.080752] R13: 0000000000000000 R14: ffff98292b7ef940 R15: ffff98280d75da00 [31148.082295] FS: 00007f1aefadc600(0000) GS:ffff98292ae00000(0000) knlGS:0000000000000000 [31148.084029] CS: 0010 DS: 0000 ES: 0000 CR0: 0000000080050033 [31148.085282] CR2: 0000000000000000 CR3: 00000000013be000 CR4: 00000000000006f0 [31148.086770] Call Trace: [31148.087286] _raw_spin_lock_nested+0x37/0x80 [31148.088212] ? raw_spin_rq_lock_nested+0x4b/0x80 [31148.089180] raw_spin_rq_lock_nested+0x4b/0x80 [31148.090189] online_fair_sched_group+0x39/0x240 [31148.091185] sched_autogroup_create_attach+0x9d/0x170 [31148.092310] ksys_setsid+0xe6/0x110 [31148.093073] __do_sys_setsid+0xa/0x10 [31148.093881] do_syscall_64+0x3b/0x90 [31148.094678] entry_SYSCALL_64_after_hwframe+0x44/0xae [31148.095783] RIP: 0033:0x7f1aef9b06cb [31148.096579] Code: 73 01 c3 48 8b 0d 5d a7 11 00 f7 d8 64 89 01 48 83 c8 ff c3 66 2e 0f 1f 84 00 00 00 00 00 90 f3 0f 1e fa b8 70 00 00 00 0f 05 <48> 3d 01 f0 ff ff 73 01 c3 48 8b 0d 2d a7 11 00 f7 d8 64 89 01 48 [31148.100758] RSP: 002b:00007fff4b7eef78 EFLAGS: 00000206 ORIG_RAX: 0000000000000070 [31148.102535] RAX: ffffffffffffffda RBX: 0000000000000000 RCX: 00007f1aef9b06cb [31148.104188] RDX: 0000000000000010 RSI: 0000000000406230 RDI: 0000000000000003 [31148.105859] RBP: 00000000ffffffff R08: 0000000000000000 R09: 00007f1aefadc600 [31148.107523] R10: 00007f1aef8e3a60 R11: 0000000000000206 R12: 0000000000000001 [31148.109155] R13: 0000000000401326 R14: 00007f1aefb14c00 R15: 0000000000405e18 [31148.110819] Modules linked in: rfkill iTCO_wdt intel_pmc_bxt iTCO_vendor_support kvm_intel gpio_ich kvm snd_pcsp snd_pcm snd_timer ipmi_ssif irqbypass snd soundcore lpc_ich acpi_ipmi e1000e tg3 hpilo ipmi_si bnx2 i5000_edac ipmi_devintf i5k_amb ipmi_msghandler fuse zram ip_tables xfs radeon i2c_algo_bit drm_ttm_helper ttm drm_kms_helper cec drm hpsa serio_raw hpwdt scsi_transport_sas [31148.118454] CR2: 0000000000000000 [31148.119246] ---[ end trace ab4b5b9489802a97 ]--- [31148.120379] RIP: 0010:__lock_acquire+0x5c1/0x1e00 [31148.121475] Code: 00 00 83 f8 2f 0f 87 59 05 00 00 3b 05 cc 46 f6 02 41 bd 01 00 00 00 0f 86 03 01 00 00 89 05 ba 46 f6 02 e9 f8 00 00 00 31 c9 <48> 81 3f 20 8e cc a0 41 0f 45 c8 83 fe 01 0f 87 6f fa ff ff 89 f0 [31148.125694] RSP: 0018:ffffa36602d5fd40 EFLAGS: 00010046 [31148.126907] RAX: 0000000000000046 RBX: 0000000000000001 RCX: 0000000000000000 [31148.128546] RDX: 0000000000000000 RSI: 0000000000000000 RDI: 0000000000000018 [31148.130195] RBP: 0000000000000018 R08: 0000000000000001 R09: 0000000000000001 [31148.131853] R10: 0000000000000001 R11: 0000000000000000 R12: ffff98285fad33c0 [31148.133493] R13: 0000000000000000 R14: 0000000000000000 R15: 0000000000000000 [31148.135168] FS: 00007f1aefadc600(0000) GS:ffff98292ae00000(0000) knlGS:0000000000000000 [31148.137018] CS: 0010 DS: 0000 ES: 0000 CR0: 0000000080050033 [31148.138352] CR2: 0000000000000000 CR3: 00000000013be000 CR4: 00000000000006f0 [31148.139984] note: prctl-sched-cor[214019] exited with preempt_count 2 [31148.141470] BUG: sleeping function called from invalid context at include/linux/percpu-rwsem.h:49 [31148.143494] in_atomic(): 0, irqs_disabled(): 1, non_block: 0, pid: 214019, name: prctl-sched-cor [31148.145497] INFO: lockdep is turned off. [31148.146422] irq event stamp: 0 [31148.147195] hardirqs last enabled at (0): [<0000000000000000>] 0x0 [31148.148640] hardirqs last disabled at (0): [<ffffffff9e0e4e9f>] copy_process+0x7cf/0x1fb0 [31148.150511] softirqs last enabled at (0): [<ffffffff9e0e4e9f>] copy_process+0x7cf/0x1fb0 [31148.152384] softirqs last disabled at (0): [<0000000000000000>] 0x0 [31148.153825] CPU: 0 PID: 214019 Comm: prctl-sched-cor Tainted: G D W --------- --- 5.14.0-0.rc5.20210813gitf8e6dfc64f61.46.fc36.x86_64 #1 [31148.156852] Hardware name: HP ProLiant BL480c G1, BIOS I14 10/04/2007 [31148.158338] Call Trace: [31148.158945] dump_stack_lvl+0x57/0x72 [31148.159859] ___might_sleep.cold+0xb6/0xc6 [31148.160822] exit_signals+0x1c/0x2d0 [31148.161668] do_exit+0xbf/0xc10 [31148.162419] ? ksys_setsid+0xe6/0x110 [31148.163228] rewind_stack_do_exit+0x17/0x20 [31148.164203] RIP: 0033:0x7f1aef9b06cb [31148.165043] Code: 73 01 c3 48 8b 0d 5d a7 11 00 f7 d8 64 89 01 48 83 c8 ff c3 66 2e 0f 1f 84 00 00 00 00 00 90 f3 0f 1e fa b8 70 00 00 00 0f 05 <48> 3d 01 f0 ff ff 73 01 c3 48 8b 0d 2d a7 11 00 f7 d8 64 89 01 48 [31148.169275] RSP: 002b:00007fff4b7eef78 EFLAGS: 00000206 ORIG_RAX: 0000000000000070 [31148.171019] RAX: ffffffffffffffda RBX: 0000000000000000 RCX: 00007f1aef9b06cb [31148.172700] RDX: 0000000000000010 RSI: 0000000000406230 RDI: 0000000000000003 [31148.174343] RBP: 00000000ffffffff R08: 0000000000000000 R09: 00007f1aefadc600 [31148.175994] R10: 00007f1aef8e3a60 R11: 0000000000000206 R12: 0000000000000001 [31148.177638] R13: 0000000000401326 R14: 00007f1aefb14c00 R15: 0000000000405e18
Peter Zijlstra
maintainerRe: [PATCH 18/19] sched: prctl() core-scheduling interface
Quoted 15 lines · Eugene Syromiatnikov
[76195.611570] BUG: kernel NULL pointer dereference, address: 0000000000000000 [76195.613059] #PF: supervisor read access in kernel mode [76195.614174] #PF: error_code(0x0000) - not-present page [76195.615329] PGD 800000005f27e067 P4D 800000005f27e067 PUD 3f7a3067 PMD 0 [76195.616801] Oops: 0000 [#67] SMP PTI [76195.617586] CPU: 2 PID: 239821 Comm: prctl-sched-cor Tainted: G D W --------- --- 5.14.0-0.rc5.20210813gitf8e6dfc64f61.46.fc36.x86_64 #1 [76195.620374] Hardware name: HP ProLiant BL480c G1, BIOS I14 10/04/2007 [76195.621771] RIP: 0010:do_raw_spin_trylock+0x5/0x40 [76195.622821] Code: c6 a4 12 5f 9f 48 89 ef e8 c8 fe ff ff eb a9 89 c6 48 89 ef e8 0c f5 ff ff 66 90 eb a9 0f 1f 84 00 00 00 00 00 0f 1f 44 00 00 <8b> 07 85 c0 75 28 ba 01 00 00 00 f0 0f b1 17 75 1d 65 8b 05 fb 98 [76195.626797] RSP: 0018:ffffa366014abe58 EFLAGS: 00010086 [76195.627936] RAX: 0000000000000001 RBX: 0000000000000004 RCX: 0000000000000000 [76195.629470] RDX: 0000000000000000 RSI: 0000000000000000 RDI: 0000000000000000 [76195.631048] RBP: 0000000000000000 R08: 0000000000000001 R09: 0000000000000000 [76195.632585] R10: 0000000000000000 R11: ffff98292b21ad48 R12: 0000000000000018 [76195.634078] R13: 0000000000000000 R14: ffff98292b7ef940 R15: ffff982813938e00
Quoted 8 lines
[76195.635621] FS: 00007f271f8755c0(0000) GS:ffff98292b200000(0000) knlGS:0000000000000000 [76195.637354] CS: 0010 DS: 0000 ES: 0000 CR0: 0000000080050033 [76195.638606] CR2: 0000000000000000 CR3: 000000000fdd0000 CR4: 00000000000006e0 [76195.640144] Call Trace: [76195.640706] _raw_spin_lock_nested+0x37/0x80 [76195.641645] ? raw_spin_rq_lock_nested+0x4b/0x80 [76195.642693] raw_spin_rq_lock_nested+0x4b/0x80 [76195.643669] online_fair_sched_group+0x39/0x240
Urgh... lemme guess, your HP BIOS is funny and reports more possible CPUs than you actually have resulting in cpu_possible_mask != cpu_online_mask. Alternatively, you booted with nr_cpus= or something daft like that.
That code does for_each_possible_cpus(i) { rq_lock_irq(cpu_rq(i)); }, which, because of core-sched, needs rq->core set-up, but because these CPUs have never been online, that's not done and *BOOM*.
Or something like that.. I'll try and have a look tomorrow, I'm in dire need of sleep.
Re: [PATCH 18/19] sched: prctl() core-scheduling interface
Quoted 4 lines · Peter Zijlstra
Urgh... lemme guess, your HP BIOS is funny and reports more possible CPUs than you actually have resulting in
cpu_possible_mask!=cpu_online_mask. Alternatively, you booted with nr_cpus= or something daft like that.
Yep, it seems to be the case:
# cat /sys/devices/system/cpu/possible
0-7
# cat /sys/devices/system/cpu/online
0-3Quoted 8 lines
That code does
for_each_possible_cpus(i){ rq_lock_irq(cpu_rq(i)); }, which, because of core-sched, needsrq->coreset-up, but because these CPUs have never been online, that's not done and *BOOM*.Or something like that.. I'll try and have a look tomorrow, I'm in dire need of sleep.
Peter Zijlstra
maintainerPATCH 1/1 sched: Fix Core-wide rq->lock for uninitialized CPUs
Quoted 6 lines · Eugene Syromiatnikov
Quoted 4 lines · Peter Zijlstra
Urgh... lemme guess, your HP BIOS is funny and reports more possible CPUs than you actually have resulting in
cpu_possible_mask!=cpu_online_mask. Alternatively, you booted with nr_cpus= or something daft like that.
Quoted 7 lines
Yep, it seems to be the case:
# cat /sys/devices/system/cpu/possible 0-7 # cat /sys/devices/system/cpu/online 0-3
I think the below should work... can you please verify?
---
Subject: sched: Fix Core-wide rq->lock for uninitialized CPUs
Eugene tripped over the case where rq_lock(), as called in a for_each_possible_cpu() loop came apart because rq->core hadn't been setup yet.
This is a somewhat unusual, but valid case.
Rework things such that rq->core is initialized to point at itself. IOW initialize each CPU as a single threaded Core. CPU online will then join the new CPU (thread) to an existing Core where needed.
For completeness sake, have CPU offline fully undo the state so as to not presume the topology will match the next time it comes online.
--- kernel/sched/core.c | 143 +++++++++++++++++++++++++++++++++++++++++---------- kernel/sched/sched.h | 2 +- 2 files changed, 118 insertions(+), 27 deletions(-)
kernel/sched/core.c +117 -26
@@ -237,9 +237,30 @@ static DEFINE_MUTEX(sched_core_mutex);static atomic_t sched_core_count;static struct cpumask sched_core_mask; +static void sched_core_lock (int cpu,unsigned long *flags) +{ +const struct cpumask *smt_mask =cpu_smt_mask (cpu); +int t, i =0 ; + +local_irq_save (*flags); +for_each_cpu (t, smt_mask) +raw_spin_lock_nested (&cpu_rq (t)->__lock, i++); +} + +static void sched_core_unlock (int cpu,unsigned long *flags) +{ +const struct cpumask *smt_mask =cpu_smt_mask (cpu); +int t; + +for_each_cpu (t, smt_mask) +raw_spin_unlock (&cpu_rq (t)->__lock); +local_irq_restore (*flags); +} +static void __sched_core_flip (bool enabled) { -int cpu, t, i; +unsigned long flags; +int cpu, t;cpus_read_lock ();
@@ -250,19 +271,12 @@ static void __sched_core_flip(bool enabled)for_each_cpu (cpu, &sched_core_mask) {const struct cpumask *smt_mask =cpu_smt_mask (cpu); - i =0 ; -local_irq_disable (); -for_each_cpu (t, smt_mask) { -/* supports up to SMT8 */ -raw_spin_lock_nested (&cpu_rq (t)->__lock, i++); - } +sched_core_lock (cpu, &flags);for_each_cpu (t, smt_mask)cpu_rq (t)->core_enabled = enabled; -for_each_cpu (t, smt_mask) -raw_spin_unlock (&cpu_rq (t)->__lock); -local_irq_enable (); +sched_core_unlock (cpu, &flags);cpumask_andnot (&sched_core_mask, &sched_core_mask, smt_mask); }
@@ -5979,35 +5993,109 @@ void queue_core_balance(struct rq *rq)queue_balance_callback (rq, &per_cpu (core_balance_head, rq->cpu), sched_core_balance); } -static inline void sched_core_cpu_starting (unsigned int cpu) +static void sched_core_cpu_starting (unsigned int cpu) {const struct cpumask *smt_mask =cpu_smt_mask (cpu); -struct rq *rq, *core_rq =NULL ; -int i; +struct rq *rq =cpu_rq (cpu), *core_rq =NULL ; +unsigned long flags; +int t; - core_rq =cpu_rq (cpu)->core; +sched_core_lock (cpu, &flags); -if (!core_rq) { -for_each_cpu (i, smt_mask) { - rq =cpu_rq (i); -if (rq->core && rq->core == rq) - core_rq = rq; +WARN_ON_ONCE (rq->core != rq); + +/* if we're the first, we'll be our own leader */ +if (cpumask_weight (smt_mask) ==1 ) + goto unlock; + +/* find the leader */ +for_each_cpu (t, smt_mask) { +if (t == cpu) +continue ; + rq =cpu_rq (t); +if (rq->core == rq) { + core_rq = rq; +break ; } + } -if (!core_rq) - core_rq =cpu_rq (cpu); +if (WARN_ON_ONCE (!core_rq))/* whoopsie */ + goto unlock; -for_each_cpu (i, smt_mask) { - rq =cpu_rq (i); +/* install and validate core_rq */ +for_each_cpu (t, smt_mask) { + rq =cpu_rq (t); -WARN_ON_ONCE (rq->core && rq->core != core_rq); +if (t == cpu) rq->core = core_rq; - } + +WARN_ON_ONCE (rq->core != core_rq); } + +unlock: +sched_core_unlock (cpu, &flags); } + +static void sched_core_cpu_deactivate (unsigned int cpu) +{ +const struct cpumask *smt_mask =cpu_smt_mask (cpu); +struct rq *rq =cpu_rq (cpu), *core_rq =NULL ; +unsigned long flags; +int t; + +sched_core_lock (cpu, &flags); + +/* if we're the last man standing, nothing to do */ +if (cpumask_weight (smt_mask) ==1 ) { +WARN_ON_ONCE (rq->core != rq); + goto unlock; + } + +/* if we're not the leader, nothing to do */ +if (rq->core != rq) + goto unlock; + +/* find a new leader */ +for_each_cpu (t, smt_mask) { +if (t == cpu) +continue ; + core_rq =cpu_rq (t); +break ; + } + +if (WARN_ON_ONCE (!core_rq))/* impossible */ + goto unlock; + +/* copy the shared state to the new leader */ + core_rq->core_task_seq = rq->core_task_seq; + core_rq->core_pick_seq = rq->core_pick_seq; + core_rq->core_cookie = rq->core_cookie; + core_rq->core_forceidle = rq->core_forceidle; + core_rq->core_forceidle_seq = rq->core_forceidle_seq; + +/* install new leader */ +for_each_cpu (t, smt_mask) { + rq =cpu_rq (t); + rq->core = core_rq; + } + +unlock: +sched_core_unlock (cpu, &flags); +} + +static inline void sched_core_cpu_dying (unsigned int cpu) +{ +struct rq *rq =cpu_rq (cpu); + +if (rq->core != rq) + rq->core = rq; +} +#else /* !CONFIG_SCHED_CORE */ static inline void sched_core_cpu_starting (unsigned int cpu) {} +static inline void sched_core_cpu_deactivate (unsigned int cpu) {} +static inline void sched_core_cpu_dying (unsigned int cpu) {}static struct task_struct *pick_next_task (struct rq *rq,struct task_struct *prev,struct rq_flags *rf)
@@ -9009,6 +9097,8 @@ int sched_cpu_deactivate(unsigned int cpu)*/ if (cpumask_weight (cpu_smt_mask (cpu)) ==2 )static_branch_dec_cpuslocked (&sched_smt_present); + +sched_core_cpu_deactivate (cpu);#endif if (!sched_smp_initialized)
@@ -9113,6 +9203,7 @@ int sched_cpu_dying(unsigned int cpu)calc_load_migrate (rq);update_max_interval ();hrtick_clear (rq); +sched_core_cpu_dying (cpu); return 0; }#endif
@@ -9324,7 +9415,7 @@ void __init sched_init(void)atomic_set (&rq->nr_iowait,0 );#ifdef CONFIG_SCHED_CORE - rq->core =NULL ; + rq->core = rq; rq->core_pick =NULL ; rq->core_enabled =0 ; rq->core_tree =RB_ROOT ;
kernel/sched/sched.h +1 -1
@@ -1103,7 +1103,7 @@ struct rq {unsigned int core_sched_seq;struct rb_root core_tree; -/* shared state */ +/* shared state -- careful with sched_core_cpu_deactivate() */ unsigned int core_task_seq;unsigned int core_pick_seq;unsigned long core_cookie;
Re: [PATCH] sched: Fix Core-wide rq->lock for uninitialized CPUs
Quoted 146 lines · Peter Zijlstra
Quoted 6 lines · Eugene Syromiatnikov
Quoted 4 lines · Peter Zijlstra
Urgh... lemme guess, your HP BIOS is funny and reports more possible CPUs than you actually have resulting in
cpu_possible_mask!=cpu_online_mask. Alternatively, you booted with nr_cpus= or something daft like that.Quoted 7 lines
Yep, it seems to be the case:
# cat /sys/devices/system/cpu/possible 0-7 # cat /sys/devices/system/cpu/online 0-3I think the below should work... can you please verify?
---
Subject: sched: Fix Core-widerq->lockfor uninitialized CPUsEugene tripped over the case where
rq_lock(), as called in afor_each_possible_cpu()loop came apart becauserq->corehadn't been setup yet.This is a somewhat unusual, but valid case.
Rework things such that
rq->coreis initialized to point at itself. IOW initialize each CPU as a single threaded Core. CPU online will then join the new CPU (thread) to an existing Core where needed.For completeness sake, have CPU offline fully undo the state so as to not presume the topology will match the next time it comes online.
Fixes 9edeaea1bc45 ("sched: Core-wide rq->lock")Reported-by Eugene Syromiatnikov <[email protected]>Signed-off-by Peter Zijlstra (Intel) <[email protected]>--- kernel/sched/core.c | 143 +++++++++++++++++++++++++++++++++++++++++---------- kernel/sched/sched.h | 2 +- 2 files changed, 118 insertions(+), 27 deletions(-)kernel/sched/core.c +47
-23@@ -237,9 +237,30 @@ static DEFINE_MUTEX(sched_core_mutex);static atomic_t sched_core_count;static struct cpumask sched_core_mask; +static void sched_core_lock (int cpu,unsigned long *flags) +{ +const struct cpumask *smt_mask =cpu_smt_mask (cpu); +int t, i =0 ; + +local_irq_save (*flags); +for_each_cpu (t, smt_mask) +raw_spin_lock_nested (&cpu_rq (t)->__lock, i++); +} + +static void sched_core_unlock (int cpu,unsigned long *flags) +{ +const struct cpumask *smt_mask =cpu_smt_mask (cpu); +int t; + +for_each_cpu (t, smt_mask) +raw_spin_unlock (&cpu_rq (t)->__lock); +local_irq_restore (*flags); +} +static void __sched_core_flip (bool enabled) { -int cpu, t, i; +unsigned long flags; +int cpu, t;cpus_read_lock ();@@ -250,19 +271,12 @@ static void __sched_core_flip(bool enabled)for_each_cpu (cpu, &sched_core_mask) {const struct cpumask *smt_mask =cpu_smt_mask (cpu); - i =0 ; -local_irq_disable (); -for_each_cpu (t, smt_mask) { -/* supports up to SMT8 */ -raw_spin_lock_nested (&cpu_rq (t)->__lock, i++); - } +sched_core_lock (cpu, &flags);for_each_cpu (t, smt_mask)cpu_rq (t)->core_enabled = enabled; -for_each_cpu (t, smt_mask) -raw_spin_unlock (&cpu_rq (t)->__lock); -local_irq_enable (); +sched_core_unlock (cpu, &flags);cpumask_andnot (&sched_core_mask, &sched_core_mask, smt_mask); }@@ -5979,35 +5993,109 @@ void queue_core_balance(struct rq *rq)queue_balance_callback (rq, &per_cpu (core_balance_head, rq->cpu), sched_core_balance); } -static inline void sched_core_cpu_starting (unsigned int cpu) +static void sched_core_cpu_starting (unsigned int cpu) {const struct cpumask *smt_mask =cpu_smt_mask (cpu); -struct rq *rq, *core_rq =NULL ; -int i; +struct rq *rq =cpu_rq (cpu), *core_rq =NULL ; +unsigned long flags; +int t; - core_rq =cpu_rq (cpu)->core; +sched_core_lock (cpu, &flags); -if (!core_rq) { -for_each_cpu (i, smt_mask) { - rq =cpu_rq (i); -if (rq->core && rq->core == rq) - core_rq = rq; +WARN_ON_ONCE (rq->core != rq); + +/* if we're the first, we'll be our own leader */ +if (cpumask_weight (smt_mask) ==1 ) + goto unlock; + +/* find the leader */ +for_each_cpu (t, smt_mask) { +if (t == cpu) +continue ; + rq =cpu_rq (t); +if (rq->core == rq) { + core_rq = rq; +break ; } + } -if (!core_rq) - core_rq =cpu_rq (cpu); +if (WARN_ON_ONCE (!core_rq))/* whoopsie */ + goto unlock; -for_each_cpu (i, smt_mask) { - rq =cpu_rq (i); +/* install and validate core_rq */
kernel/sched/core.c void queue_core_balance(struct rq *rq) · line 6033
Quoted 10 lines
kernel/sched/core.c +5
-2+for_each_cpu (t, smt_mask) { + rq =cpu_rq (t); -WARN_ON_ONCE (rq->core && rq->core != core_rq); +if (t == cpu) rq->core = core_rq; - } + +WARN_ON_ONCE (rq->core != core_rq); }
Yoh, Peter!
Read not more than ten times but stay here long enough until I realize something that I am not sure. You are not only just fix the problem, you change the rq's core selection method(manner.). Original source now set the core to one rq, here you distribute the core_rq to not just only one rq.
For example(SMT3): rq1 --> rq0, rq0 --> rq2, rq2 --> rq2(/* whoopsie */).
And the core can change when deactivate.
Not sure I'm getting all the things right, but this is more performance efficient because the lock contention is avoided when pick task from my imagination.
Again, I'm not know much about everything, just put some words.
Quoted 99 lines
kernel/sched/core.c +65
-1+ +unlock: +sched_core_unlock (cpu, &flags); } + +static void sched_core_cpu_deactivate (unsigned int cpu) +{ +const struct cpumask *smt_mask =cpu_smt_mask (cpu); +struct rq *rq =cpu_rq (cpu), *core_rq =NULL ; +unsigned long flags; +int t; + +sched_core_lock (cpu, &flags); + +/* if we're the last man standing, nothing to do */ +if (cpumask_weight (smt_mask) ==1 ) { +WARN_ON_ONCE (rq->core != rq); + goto unlock; + } + +/* if we're not the leader, nothing to do */ +if (rq->core != rq) + goto unlock; + +/* find a new leader */ +for_each_cpu (t, smt_mask) { +if (t == cpu) +continue ; + core_rq =cpu_rq (t); +break ; + } + +if (WARN_ON_ONCE (!core_rq))/* impossible */ + goto unlock; + +/* copy the shared state to the new leader */ + core_rq->core_task_seq = rq->core_task_seq; + core_rq->core_pick_seq = rq->core_pick_seq; + core_rq->core_cookie = rq->core_cookie; + core_rq->core_forceidle = rq->core_forceidle; + core_rq->core_forceidle_seq = rq->core_forceidle_seq; + +/* install new leader */ +for_each_cpu (t, smt_mask) { + rq =cpu_rq (t); + rq->core = core_rq; + } + +unlock: +sched_core_unlock (cpu, &flags); +} + +static inline void sched_core_cpu_dying (unsigned int cpu) +{ +struct rq *rq =cpu_rq (cpu); + +if (rq->core != rq) + rq->core = rq; +} +#else /* !CONFIG_SCHED_CORE */ static inline void sched_core_cpu_starting (unsigned int cpu) {} +static inline void sched_core_cpu_deactivate (unsigned int cpu) {} +static inline void sched_core_cpu_dying (unsigned int cpu) {}static struct task_struct *pick_next_task (struct rq *rq,struct task_struct *prev,struct rq_flags *rf)@@ -9009,6 +9097,8 @@ int sched_cpu_deactivate(unsigned int cpu)*/ if (cpumask_weight (cpu_smt_mask (cpu)) ==2 )static_branch_dec_cpuslocked (&sched_smt_present); + +sched_core_cpu_deactivate (cpu);#endif if (!sched_smp_initialized)@@ -9113,6 +9203,7 @@ int sched_cpu_dying(unsigned int cpu)calc_load_migrate (rq);update_max_interval ();hrtick_clear (rq); +sched_core_cpu_dying (cpu); return 0; }#endif @@ -9324,7 +9415,7 @@ void __init sched_init(void)atomic_set (&rq->nr_iowait,0 );#ifdef CONFIG_SCHED_CORE - rq->core =NULL ; + rq->core = rq; rq->core_pick =NULL ; rq->core_enabled =0 ; rq->core_tree =RB_ROOT ;kernel/sched/sched.h +0
-0
kernel/sched/sched.h struct rq { · line 1109
Quoted 8 lines
kernel/sched/sched.h +1
-1@@ -1103,7 +1103,7 @@ struct rq {unsigned int core_sched_seq;struct rb_root core_tree; -/* shared state */ +/* shared state -- careful with sched_core_cpu_deactivate() */ unsigned int core_task_seq;unsigned int core_pick_seq;unsigned long core_cookie;
Thanks,
Tao
Re: [PATCH] sched: Fix Core-wide rq->lock for uninitialized CPUs
In reply to Peter ZijlstraQuoted 9 lines · Peter Zijlstra
Quoted 8 lines · Eugene Syromiatnikov
Quoted 4 lines · Peter Zijlstra
Urgh... lemme guess, your HP BIOS is funny and reports more possible CPUs than you actually have resulting in
cpu_possible_mask!=cpu_online_mask. Alternatively, you booted with nr_cpus= or something daft like that.Yep, it seems to be the case:
Quoted 7 lines
Quoted 5 lines
# cat /sys/devices/system/cpu/possible 0-7 # cat /sys/devices/system/cpu/online 0-3I think the below should work... can you please verify?
Yes, it no longer oops'es now, thank you!
# cat /sys/devices/system/cpu/possible
0-7
# cat /sys/devices/system/cpu/online
0-3
# ./prctl-sched-core-oops-repro
Iteration 0 status: 0
Iteration 1 status: 0
# ../src/strace -fvq -eprctl,clone,setsid -esignal=none ./prctl-sched-core-oops-repro
clone(child_stack=NULL, flags=CLONE_CHILD_CLEARTID|CLONE_CHILD_SETTID|SIGCHLD, child_tidptr=0x7f510b1c7890) = 108328
[pid 108328] setsid() = 108328
[pid 108328] +++ exited with 0 +++
Iteration 0 status: 0
prctl(PR_SCHED_CORE, PR_SCHED_CORE_CREATE, 108324, 0x2 /* PIDTYPE_PGID */, NULL) = 0
clone(child_stack=NULL, flags=CLONE_CHILD_CLEARTID|CLONE_CHILD_SETTID|SIGCHLD, child_tidptr=0x7f510b1c7890) = 108329
[pid 108329] setsid() = 108329
[pid 108329] +++ exited with 0 +++
Iteration 1 status: 0
prctl(PR_SCHED_CORE, PR_SCHED_CORE_CREATE, 108324, 0x2 /* PIDTYPE_PGID */, NULL) = 0
+++ exited with 0 +++Quoted 13 lines
---
Subject: sched: Fix Core-widerq->lockfor uninitialized CPUsEugene tripped over the case where
rq_lock(), as called in afor_each_possible_cpu()loop came apart becauserq->corehadn't been setup yet.This is a somewhat unusual, but valid case.
Rework things such that
rq->coreis initialized to point at itself. IOW initialize each CPU as a single threaded Core. CPU online will then join the new CPU (thread) to an existing Core where needed.
Quoted 6 lines
For completeness sake, have CPU offline fully undo the state so as to not presume the topology will match the next time it comes online.
Fixes 9edeaea1bc45 ("sched: Core-wide rq->lock")Reported-by Eugene Syromiatnikov <[email protected]>Signed-off-by Peter Zijlstra (Intel) <[email protected]>
Re: [PATCH] sched: Fix Core-wide rq->lock for uninitialized CPUs
In reply to Peter ZijlstraHi Peter,
kernel/sched/sched.h struct rq { · line 1106
Quoted 3 lines · Peter Zijlstra
kernel/sched/sched.h +1
-1-/* shared state */ +/* shared state -- careful with sched_core_cpu_deactivate() */
Could also throw these fields into a wrapping struct. Either way seems fine.
Peter Zijlstra
maintainerRe: [PATCH] sched: Fix Core-wide rq->lock for uninitialized CPUs
Quoted 10 lines · Josh Don
Hi Peter,
kernel/sched/sched.h struct rq { · line 1106
Quoted 3 lines · Peter Zijlstra
kernel/sched/sched.h +1
-1-/* shared state */ +/* shared state -- careful with sched_core_cpu_deactivate() */ Could also throw these fields into a wrapping struct. Either way seems fine.
Reviewed-by Josh Don <[email protected]>
Yes, I considered that, but didn't want the churn at this time.
[tip: sched/urgent] sched: Fix Core-wide rq->lock for uninitialized CPUs
In reply to Peter ZijlstraThe following commit has been merged into the sched/urgent branch of tip:
The following commit has been merged into the sched/urgent branch of tip:
Commit-ID: 3c474b3239f12fe0b00d7e82481f36a1f31e79ab Gitweb: https://git.kernel.org/tip/3c474b3239f12fe0b00d7e82481f36a1f31e79ab Author: Peter Zijlstra <[email protected]> AuthorDate: Thu, 19 Aug 2021 13:09:17 +02:00 Committer: Peter Zijlstra <[email protected]> CommitterDate: Fri, 20 Aug 2021 12:32:53 +02:00
sched: Fix Core-wide rq->lock for uninitialized CPUs
Eugene tripped over the case where rq_lock(), as called in a for_each_possible_cpu() loop came apart because rq->core hadn't been setup yet.
This is a somewhat unusual, but valid case.
Rework things such that rq->core is initialized to point at itself. IOW initialize each CPU as a single threaded Core. CPU online will then join the new CPU (thread) to an existing Core where needed.
For completeness sake, have CPU offline fully undo the state so as to not presume the topology will match the next time it comes online.
--- kernel/sched/core.c | 143 ++++++++++++++++++++++++++++++++++-------- kernel/sched/sched.h | 2 +- 2 files changed, 118 insertions(+), 27 deletions(-)
kernel/sched/core.c +117 -26
@@ -237,9 +237,30 @@ static DEFINE_MUTEX(sched_core_mutex);static atomic_t sched_core_count;static struct cpumask sched_core_mask; +static void sched_core_lock (int cpu,unsigned long *flags) +{ +const struct cpumask *smt_mask =cpu_smt_mask (cpu); +int t, i =0 ; + +local_irq_save (*flags); +for_each_cpu (t, smt_mask) +raw_spin_lock_nested (&cpu_rq (t)->__lock, i++); +} + +static void sched_core_unlock (int cpu,unsigned long *flags) +{ +const struct cpumask *smt_mask =cpu_smt_mask (cpu); +int t; + +for_each_cpu (t, smt_mask) +raw_spin_unlock (&cpu_rq (t)->__lock); +local_irq_restore (*flags); +} +static void __sched_core_flip (bool enabled) { -int cpu, t, i; +unsigned long flags; +int cpu, t;cpus_read_lock ();
@@ -250,19 +271,12 @@ static void __sched_core_flip(bool enabled)for_each_cpu (cpu, &sched_core_mask) {const struct cpumask *smt_mask =cpu_smt_mask (cpu); - i =0 ; -local_irq_disable (); -for_each_cpu (t, smt_mask) { -/* supports up to SMT8 */ -raw_spin_lock_nested (&cpu_rq (t)->__lock, i++); - } +sched_core_lock (cpu, &flags);for_each_cpu (t, smt_mask)cpu_rq (t)->core_enabled = enabled; -for_each_cpu (t, smt_mask) -raw_spin_unlock (&cpu_rq (t)->__lock); -local_irq_enable (); +sched_core_unlock (cpu, &flags);cpumask_andnot (&sched_core_mask, &sched_core_mask, smt_mask); }
@@ -5736,35 +5750,109 @@ void queue_core_balance(struct rq *rq)queue_balance_callback (rq, &per_cpu (core_balance_head, rq->cpu), sched_core_balance); } -static inline void sched_core_cpu_starting (unsigned int cpu) +static void sched_core_cpu_starting (unsigned int cpu) {const struct cpumask *smt_mask =cpu_smt_mask (cpu); -struct rq *rq, *core_rq =NULL ; -int i; +struct rq *rq =cpu_rq (cpu), *core_rq =NULL ; +unsigned long flags; +int t; - core_rq =cpu_rq (cpu)->core; +sched_core_lock (cpu, &flags); -if (!core_rq) { -for_each_cpu (i, smt_mask) { - rq =cpu_rq (i); -if (rq->core && rq->core == rq) - core_rq = rq; +WARN_ON_ONCE (rq->core != rq); + +/* if we're the first, we'll be our own leader */ +if (cpumask_weight (smt_mask) ==1 ) + goto unlock; + +/* find the leader */ +for_each_cpu (t, smt_mask) { +if (t == cpu) +continue ; + rq =cpu_rq (t); +if (rq->core == rq) { + core_rq = rq; +break ; } + } -if (!core_rq) - core_rq =cpu_rq (cpu); +if (WARN_ON_ONCE (!core_rq))/* whoopsie */ + goto unlock; -for_each_cpu (i, smt_mask) { - rq =cpu_rq (i); +/* install and validate core_rq */ +for_each_cpu (t, smt_mask) { + rq =cpu_rq (t); -WARN_ON_ONCE (rq->core && rq->core != core_rq); +if (t == cpu) rq->core = core_rq; - } + +WARN_ON_ONCE (rq->core != core_rq); } + +unlock: +sched_core_unlock (cpu, &flags); } + +static void sched_core_cpu_deactivate (unsigned int cpu) +{ +const struct cpumask *smt_mask =cpu_smt_mask (cpu); +struct rq *rq =cpu_rq (cpu), *core_rq =NULL ; +unsigned long flags; +int t; + +sched_core_lock (cpu, &flags); + +/* if we're the last man standing, nothing to do */ +if (cpumask_weight (smt_mask) ==1 ) { +WARN_ON_ONCE (rq->core != rq); + goto unlock; + } + +/* if we're not the leader, nothing to do */ +if (rq->core != rq) + goto unlock; + +/* find a new leader */ +for_each_cpu (t, smt_mask) { +if (t == cpu) +continue ; + core_rq =cpu_rq (t); +break ; + } + +if (WARN_ON_ONCE (!core_rq))/* impossible */ + goto unlock; + +/* copy the shared state to the new leader */ + core_rq->core_task_seq = rq->core_task_seq; + core_rq->core_pick_seq = rq->core_pick_seq; + core_rq->core_cookie = rq->core_cookie; + core_rq->core_forceidle = rq->core_forceidle; + core_rq->core_forceidle_seq = rq->core_forceidle_seq; + +/* install new leader */ +for_each_cpu (t, smt_mask) { + rq =cpu_rq (t); + rq->core = core_rq; + } + +unlock: +sched_core_unlock (cpu, &flags); +} + +static inline void sched_core_cpu_dying (unsigned int cpu) +{ +struct rq *rq =cpu_rq (cpu); + +if (rq->core != rq) + rq->core = rq; +} +#else /* !CONFIG_SCHED_CORE */ static inline void sched_core_cpu_starting (unsigned int cpu) {} +static inline void sched_core_cpu_deactivate (unsigned int cpu) {} +static inline void sched_core_cpu_dying (unsigned int cpu) {}static struct task_struct *pick_next_task (struct rq *rq,struct task_struct *prev,struct rq_flags *rf)
@@ -8707,6 +8795,8 @@ int sched_cpu_deactivate(unsigned int cpu)*/ if (cpumask_weight (cpu_smt_mask (cpu)) ==2 )static_branch_dec_cpuslocked (&sched_smt_present); + +sched_core_cpu_deactivate (cpu);#endif if (!sched_smp_initialized)
@@ -8811,6 +8901,7 @@ int sched_cpu_dying(unsigned int cpu)calc_load_migrate (rq);update_max_interval ();hrtick_clear (rq); +sched_core_cpu_dying (cpu); return 0; }#endif
@@ -9022,7 +9113,7 @@ void __init sched_init(void)atomic_set (&rq->nr_iowait,0 );#ifdef CONFIG_SCHED_CORE - rq->core =NULL ; + rq->core = rq; rq->core_pick =NULL ; rq->core_enabled =0 ; rq->core_tree =RB_ROOT ;
kernel/sched/sched.h +1 -1
@@ -1093,7 +1093,7 @@ struct rq {unsigned int core_sched_seq;struct rb_root core_tree; -/* shared state */ +/* shared state -- careful with sched_core_cpu_deactivate() */ unsigned int core_task_seq;unsigned int core_pick_seq;unsigned long core_cookie;
Peter Zijlstra
maintainerPATCH 19/19 kselftest: Add test for core sched prctl interface
In reply to Peter ZijlstraFrom: Chris Hyser <[email protected]>
Provides a selftest and examples of using the interface.
--- tools/testing/selftests/sched/.gitignore | 1 tools/testing/selftests/sched/Makefile | 14 + tools/testing/selftests/sched/config | 1 tools/testing/selftests/sched/cs_prctl_test.c | 338 ++++++++++++++++++++++++++ 4 files changed, 354 insertions(+) create mode 100644 tools/testing/selftests/sched/.gitignore create mode 100644 tools/testing/selftests/sched/Makefile create mode 100644 tools/testing/selftests/sched/config create mode 100644 tools/testing/selftests/sched/cs_prctl_test.c
tools/testing/selftests/sched/.gitignore new file +1 -0
@@ -0,0 +1 @@ +cs_prctl_test
tools/testing/selftests/sched/Makefile new file +14 -0
@@ -0,0 +1,14 @@ +# SPDX-License-Identifier: GPL-2.0+ + +ifneq ($(shell $(CC) --version 2>&1 | head -n 1 | grep clang),) +CLANG_FLAGS += -no-integrated-as +endif + +CFLAGS += -O2 -Wall -g -I./ -I../../../../usr/include/ -Wl,-rpath=./ \ + $(CLANG_FLAGS) +LDLIBS += -lpthread + +TEST_GEN_FILES := cs_prctl_test +TEST_PROGS := cs_prctl_test + +include ../lib.mk
tools/testing/selftests/sched/config new file +1 -0
@@ -0,0 +1 @@ +CONFIG_SCHED_DEBUG=y
tools/testing/selftests/sched/cs_prctl_test.c new file +338 -0
338 lines not on this page · raw message
[tip: sched/core] kselftest: Add test for core sched prctl interface
The following commit has been merged into the sched/core branch of tip:
The following commit has been merged into the sched/core branch of tip:
Commit-ID: 9f26990074931bbf797373e53104216059b300b1 Gitweb: https://git.kernel.org/tip/9f26990074931bbf797373e53104216059b300b1 Author: Chris Hyser <[email protected]> AuthorDate: Wed, 24 Mar 2021 17:40:16 -04:00 Committer: Peter Zijlstra <[email protected]> CommitterDate: Wed, 12 May 2021 11:43:32 +02:00
kselftest: Add test for core sched prctl interface
Provides a selftest and examples of using the interface.
--- tools/testing/selftests/sched/.gitignore | 1 +- tools/testing/selftests/sched/Makefile | 14 +- tools/testing/selftests/sched/config | 1 +- tools/testing/selftests/sched/cs_prctl_test.c | 338 +++++++++++++++++- 4 files changed, 354 insertions(+) create mode 100644 tools/testing/selftests/sched/.gitignore create mode 100644 tools/testing/selftests/sched/Makefile create mode 100644 tools/testing/selftests/sched/config create mode 100644 tools/testing/selftests/sched/cs_prctl_test.c
tools/testing/selftests/sched/.gitignore new file +1 -0
@@ -0,0 +1 @@ +cs_prctl_test
tools/testing/selftests/sched/Makefile new file +14 -0
@@ -0,0 +1,14 @@ +# SPDX-License-Identifier: GPL-2.0+ + +ifneq ($(shell $(CC) --version 2>&1 | head -n 1 | grep clang),) +CLANG_FLAGS += -no-integrated-as +endif + +CFLAGS += -O2 -Wall -g -I./ -I../../../../usr/include/ -Wl,-rpath=./ \ + $(CLANG_FLAGS) +LDLIBS += -lpthread + +TEST_GEN_FILES := cs_prctl_test +TEST_PROGS := cs_prctl_test + +include ../lib.mk
tools/testing/selftests/sched/config new file +1 -0
@@ -0,0 +1 @@ +CONFIG_SCHED_DEBUG=y
tools/testing/selftests/sched/cs_prctl_test.c new file +338 -0
338 lines not on this page · raw message
Quoted 11 lines · Peter Zijlstra
Hai,
This is an agressive fold of all the core-scheduling work so far. I've stripped a whole bunch of tags along the way (hopefully not too many, please yell if you feel I made a mistake), including tested-by. Please retest.
Changes since the last partial post is dropping all the cgroup stuff and
PR_SCHED_CORE_CLEARas well as thatexec()behaviour in order to later resolve the cgroup issue.
Hi Peter,
Is there a reason that PR_SCHED_CORE_CLEAR got removed? It's handy to be able to clear
cookies.
Thanks,
Don
Quoted 33 lines
Since we're really rather late for the coming merge window, my plan was to merge the lot right after the merge window.
Again, please test.
These patches should shortly be available in my queue.git.
--- b/kernel/sched/core_sched.c | 229 ++++++ b/tools/testing/selftests/sched/.gitignore | 1 b/tools/testing/selftests/sched/Makefile | 14 b/tools/testing/selftests/sched/config | 1 b/tools/testing/selftests/sched/cs_prctl_test.c | 338 +++++++++ include/linux/sched.h | 19 include/uapi/linux/prctl.h | 8 kernel/Kconfig.preempt | 6 kernel/fork.c | 4 kernel/sched/Makefile | 1 kernel/sched/core.c | 858 ++++++++++++++++++++++-- kernel/sched/cpuacct.c | 12 kernel/sched/deadline.c | 38 - kernel/sched/debug.c | 4 kernel/sched/fair.c | 276 +++++-- kernel/sched/idle.c | 13 kernel/sched/pelt.h | 2 kernel/sched/rt.c | 31 kernel/sched/sched.h | 393 ++++++++-- kernel/sched/stop_task.c | 14 kernel/sched/topology.c | 4 kernel/sys.c | 5 tools/include/uapi/linux/prctl.h | 8 23 files changed, 2057 insertions(+), 222 deletions(-)
Peter Zijlstra
maintainerQuoted 8 lines · Don Hiatt
Quoted 7 lines · Peter Zijlstra
Hai,
This is an agressive fold of all the core-scheduling work so far. I've stripped a whole bunch of tags along the way (hopefully not too many, please yell if you feel I made a mistake), including tested-by. Please retest.
Quoted 8 lines
Quoted 4 lines
Changes since the last partial post is dropping all the cgroup stuff and
PR_SCHED_CORE_CLEARas well as thatexec()behaviour in order to later resolve the cgroup issue.Hi Peter,
Is there a reason that
PR_SCHED_CORE_CLEARgot removed? It's handy to be able to clear cookies.
I agree, but if we're forced to do cgroup with cookie inherit, when CLEAR allows a trivial escape.
Until that whole cgroup thing is sorted, it is easiest to not have CLEAR, lest we have to change/augment CLEAR semantics in unfortunate ways, which is always harder once an API is out there.
Also, see the entire previous discussion here:
Quoted 34 lines · Peter Zijlstra
Hai,
This is an agressive fold of all the core-scheduling work so far. I've stripped a whole bunch of tags along the way (hopefully not too many, please yell if you feel I made a mistake), including tested-by. Please retest.
Changes since the last partial post is dropping all the cgroup stuff and
PR_SCHED_CORE_CLEARas well as thatexec()behaviour in order to later resolve the cgroup issue.Since we're really rather late for the coming merge window, my plan was to merge the lot right after the merge window.
Again, please test.
These patches should shortly be available in my queue.git.
--- b/kernel/sched/core_sched.c | 229 ++++++ b/tools/testing/selftests/sched/.gitignore | 1 b/tools/testing/selftests/sched/Makefile | 14 b/tools/testing/selftests/sched/config | 1 b/tools/testing/selftests/sched/cs_prctl_test.c | 338 +++++++++ include/linux/sched.h | 19 include/uapi/linux/prctl.h | 8 kernel/Kconfig.preempt | 6 kernel/fork.c | 4 kernel/sched/Makefile | 1 kernel/sched/core.c | 858 ++++++++++++++++++++++-- kernel/sched/cpuacct.c | 12 kernel/sched/deadline.c | 38 - kernel/sched/debug.c | 4 kernel/sched/fair.c | 276 +++++-- kernel/sched/idle.c | 13
Quoted 9 lines
kernel/sched/pelt.h | 2 kernel/sched/rt.c | 31 kernel/sched/sched.h | 393 ++++++++-- kernel/sched/stop_task.c | 14 kernel/sched/topology.c | 4 kernel/sys.c | 5 tools/include/uapi/linux/prctl.h | 8 23 files changed, 2057 insertions(+), 222 deletions(-)
Adding sysbench/uperf/wis performance results for reference:
- kernel under test:
-- above patchset of core-scheduling + local fix for softlockup issue: https://lore.kernel.org/lkml/[email protected]/ -- coresched_v10 kernel source: https://github.com/digitalocean/linux-coresched/commits/coresched/v10-v5.10.y
- workloads:
-- A. sysbench cpu (192 threads) + sysbench cpu (192 threads) -- B. sysbench cpu (192 threads) + sysbench mysql (192 threads) -- C. uperf netperf.xml (192 threads over TCP or UDP protocol separately) -- D. will-it-scale context_switch via pipe (192 threads)
- test machine setup:
CPU(s): 192
On-line CPU(s) list: 0-191
Thread(s) per core: 2
Core(s) per socket: 48
Socket(s): 2
NUMA node(s): 4
- performance change key info:
--workload B: coresched (cs_on), sysbench mysql performance drop around 20% vs coresched_v10
--workload C, coresched (cs_on), uperf performance increased almost double vs coresched_v10
--workload C, default (cs_off), uperf performance drop over 20% vs coresched_v10, same issue seen on v5.12-rc8 base (w/o coresched patchset) --workload D, coresched (cs_on), wis performance increased almost double vs coresched_v10
- performance info of workloads, normalized based on coresched_v10 results --workload A:
Note:
* no performance change compared to coresched_v10
+---------------------------------------+------+----------------------------------------------+------------------------------------------------+-------+-------------------------------+---------------------------------+ | | ** | coresched_peterz_aubrey_fix_base_v5.12-rc8 | coresched_peterz_aubrey_fix_base_v5.12-rc8 | *** | coresched_v10_base_v5.10.11 | coresched_v10_base_v5.10.11 | +=======================================+======+==============================================+================================================+=======+===============================+=================================+ | workload | ** | sysbench cpu * 192 | sysbench cpu * 192 | *** | sysbench cpu * 192 | sysbench cpu * 192 | +---------------------------------------+------+----------------------------------------------+------------------------------------------------+-------+-------------------------------+---------------------------------+ | prctl/cgroup | ** | prctl on workload cpu_0 | prctl on workload cpu_1 | *** | cg_sysbench_cpu_0 | cg_sysbench_cpu_1 | +---------------------------------------+------+----------------------------------------------+------------------------------------------------+-------+-------------------------------+---------------------------------+ | record_item | ** | Tput_avg (events/s) | Tput_avg (events/s) | *** | Tput_avg (events/s) | Tput_avg (events/s) | +---------------------------------------+------+----------------------------------------------+------------------------------------------------+-------+-------------------------------+---------------------------------+ | coresched normalized vs coresched_v10 | ** | 0.99 | 1.01 | *** | 1 | 1 | +---------------------------------------+------+----------------------------------------------+------------------------------------------------+-------+-------------------------------+---------------------------------+ | default normalized vs coresched_v10 | ** | 1.03 | 0.98 | *** | 1 | 1 | +---------------------------------------+------+----------------------------------------------+------------------------------------------------+-------+-------------------------------+---------------------------------+ | smtoff normalized vs coresched_v10 | ** | 1.01 | 0.99 | *** | 1 | 1 | +---------------------------------------+------+----------------------------------------------+------------------------------------------------+-------+-------------------------------+---------------------------------+
--workload B:
Note:
* coresched (cs_on), sysbench mysql performance drop around 20% vs coresched_v10
+---------------------------------------+------+----------------------------------------------+------------------------------------------------+-------+-------------------------------+---------------------------------+ | | ** | coresched_peterz_aubrey_fix_base_v5.12-rc8 | coresched_peterz_aubrey_fix_base_v5.12-rc8 | *** | coresched_v10_base_v5.10.11 | coresched_v10_base_v5.10.11 | +=======================================+======+==============================================+================================================+=======+===============================+=================================+ | workload | ** | sysbench cpu * 192 | sysbench mysql * 192 | *** | sysbench cpu * 192 | sysbench mysql * 192 | +---------------------------------------+------+----------------------------------------------+------------------------------------------------+-------+-------------------------------+---------------------------------+ | prctl/cgroup | ** | prctl on workload cpu_0 | prctl on workload mysql_0 | *** | cg_sysbench_cpu_0 | cg_sysbench_mysql_0 | +---------------------------------------+------+----------------------------------------------+------------------------------------------------+-------+-------------------------------+---------------------------------+ | record_item | ** | Tput_avg (events/s) | Tput_avg (events/s) | *** | Tput_avg (events/s) | Tput_avg (events/s) | +---------------------------------------+------+----------------------------------------------+------------------------------------------------+-------+-------------------------------+---------------------------------+ | coresched normalized vs coresched_v10 | ** | 1.03 | 0.77 | *** | 1 | 1 | +---------------------------------------+------+----------------------------------------------+------------------------------------------------+-------+-------------------------------+---------------------------------+ | default normalized vs coresched_v10 | ** | 1.02 | 0.9 | *** | 1 | 1 | +---------------------------------------+------+----------------------------------------------+------------------------------------------------+-------+-------------------------------+---------------------------------+ | smtoff normalized vs coresched_v10 | ** | 0.94 | 1.14 | *** | 1 | 1 | +---------------------------------------+------+----------------------------------------------+------------------------------------------------+-------+-------------------------------+---------------------------------+
--workload C:
Note:
* coresched (cs_on), uperf performance increased almost double vs coresched_v10
* default (cs_off), uperf performance drop over 20% vs coresched_v10, same issue seen on v5.12-rc8 base (w/o coresched patchset)
+---------------------------------------+------+----------------------------------------------+------------------------------------------------+-------+-------------------------------+---------------------------------+ | | ** | coresched_peterz_aubrey_fix_base_v5.12-rc8 | coresched_peterz_aubrey_fix_base_v5.12-rc8 | *** | coresched_v10_base_v5.10.11 | coresched_v10_base_v5.10.11 | +=======================================+======+==============================================+================================================+=======+===============================+=================================+ | workload | ** | uperf netperf TCP * 192 | uperf netperf UDP * 192 | *** | uperf netperf TCP * 192 | uperf netperf UDP * 192 | +---------------------------------------+------+----------------------------------------------+------------------------------------------------+-------+-------------------------------+---------------------------------+ | prctl/cgroup | ** | prctl on workload uperf | prctl on workload uperf | *** | cg_uperf | cg_uperf | +---------------------------------------+------+----------------------------------------------+------------------------------------------------+-------+-------------------------------+---------------------------------+ | record_item | ** | Tput_avg (Gb/s) | Tput_avg (Gb/s) | *** | Tput_avg (Gb/s) | Tput_avg (Gb/s) | +---------------------------------------+------+----------------------------------------------+------------------------------------------------+-------+-------------------------------+---------------------------------+ | coresched normalized vs coresched_v10 | ** | 1.87 | 1.99 | *** | 1 | 1 | +---------------------------------------+------+----------------------------------------------+------------------------------------------------+-------+-------------------------------+---------------------------------+ | default normalized vs coresched_v10 | ** | 0.78 | 0.74 | *** | 1 | 1 | +---------------------------------------+------+----------------------------------------------+------------------------------------------------+-------+-------------------------------+---------------------------------+ | smtoff normalized vs coresched_v10 | ** | 0.87 | 0.95 | *** | 1 | 1 | +---------------------------------------+------+----------------------------------------------+------------------------------------------------+-------+-------------------------------+---------------------------------+
--workload D:
Note:
* coresched (cs_on), wis performance increased almost double vs coresched_v10
+---------------------------------------+------+----------------------------------------------+-------+-------------------------------+ | | ** | coresched_peterz_aubrey_fix_base_v5.12-rc8 | *** | coresched_v10_base_v5.10.11 | +=======================================+======+==============================================+=======+===============================+ | workload | ** | will-it-scale * 192 | *** | will-it-scale * 192 | | | | (pipe based context_switch) | | (pipe based context_switch) | +---------------------------------------+------+----------------------------------------------+-------+-------------------------------+ | prctl/cgroup | ** | prctl on workload wis | *** | cg_wis | +---------------------------------------+------+----------------------------------------------+-------+-------------------------------+ | record_item | ** | threads_avg | *** | threads_avg | +---------------------------------------+------+----------------------------------------------+-------+-------------------------------+ | coresched normalized vs coresched_v10 | ** | 1.98 | *** | 1 | +---------------------------------------+------+----------------------------------------------+-------+-------------------------------+ | default normalized vs coresched_v10 | ** | 1.13 | *** | 1 | +---------------------------------------+------+----------------------------------------------+-------+-------------------------------+ | smtoff normalized vs coresched_v10 | ** | 1.32 | *** | 1 | +---------------------------------------+------+----------------------------------------------+-------+-------------------------------+
-- notes on record_item:
* coresched normalized vs coresched_v10: smton, cs enabled, test result normalized by result of coresched_v10 under same config
* default normalized vs coresched_v10: smton, cs disabled, test result normalized by result of coresched_v10 under same config
* smtoff normalized vs coresched_v10: smtoff, test result normalized by result of coresched_v10 under same config
Hongyu
Quoted 5 lines · Hongyu
Adding sysbench/uperf/wis performance results for reference:
- kernel under test:
-- above patchset of core-scheduling + local fix for softlockup issue: https://lore.kernel.org/lkml/[email protected]/ -- coresched_v10 kernel source: https://github.com/digitalocean/linux-coresched/commits/coresched/v10-v5.10.y
Shall I summarize all this as:
?
Quoted 7 lines · Peter Zijlstra
Quoted 5 lines · Hongyu
Adding sysbench/uperf/wis performance results for reference:
- kernel under test:
-- above patchset of core-scheduling + local fix for softlockup issue: https://lore.kernel.org/lkml/[email protected]/ -- coresched_v10 kernel source: https://github.com/digitalocean/linux-coresched/commits/coresched/v10-v5.10.y
Quoted 6 lines
Shall I summarize all this as:
Tested-by Hongyu Ning <[email protected]>?
Yes, that would be great.
Regards,
Hongyu
Quoted 7 lines · Peter Zijlstra
Hai,
This is an agressive fold of all the core-scheduling work so far. I've stripped a whole bunch of tags along the way (hopefully not too many, please yell if you feel I made a mistake), including tested-by. Please retest.
Quoted 8 lines
Changes since the last partial post is dropping all the cgroup stuff and
PR_SCHED_CORE_CLEARas well as thatexec()behaviour in order to later resolve the cgroup issue.Since we're really rather late for the coming merge window, my plan was to merge the lot right after the merge window.
Again, please test.
Thanks Peter, I uploaded this set to our ChromeOS 5.10 kernel tree with minor changes to make it apply, to:
https://chromium-review.googlesource.com/q/topic:cs510
Let me know if anything looks weird, but it does build. I will be testing it further in the coming days.
Of course product kernels slightly lag upstream but such is life <:-)
- Joel
Quoted 29 lines
These patches should shortly be available in my queue.git.
--- b/kernel/sched/core_sched.c | 229 ++++++ b/tools/testing/selftests/sched/.gitignore | 1 b/tools/testing/selftests/sched/Makefile | 14 b/tools/testing/selftests/sched/config | 1 b/tools/testing/selftests/sched/cs_prctl_test.c | 338 +++++++++ include/linux/sched.h | 19 include/uapi/linux/prctl.h | 8 kernel/Kconfig.preempt | 6 kernel/fork.c | 4 kernel/sched/Makefile | 1 kernel/sched/core.c | 858 ++++++++++++++++++++++-- kernel/sched/cpuacct.c | 12 kernel/sched/deadline.c | 38 - kernel/sched/debug.c | 4 kernel/sched/fair.c | 276 +++++-- kernel/sched/idle.c | 13 kernel/sched/pelt.h | 2 kernel/sched/rt.c | 31 kernel/sched/sched.h | 393 ++++++++-- kernel/sched/stop_task.c | 14 kernel/sched/topology.c | 4 kernel/sys.c | 5 tools/include/uapi/linux/prctl.h | 8 23 files changed, 2057 insertions(+), 222 deletions(-)
Vincent Guittot
maintainerHi Peter,
Quoted 7 lines · Peter Zijlstra
Hai,
This is an agressive fold of all the core-scheduling work so far. I've stripped a whole bunch of tags along the way (hopefully not too many, please yell if you feel I made a mistake), including tested-by. Please retest.
Quoted 8 lines
Changes since the last partial post is dropping all the cgroup stuff and
PR_SCHED_CORE_CLEARas well as thatexec()behaviour in order to later resolve the cgroup issue.Since we're really rather late for the coming merge window, my plan was to merge the lot right after the merge window.
Again, please test.
I have run various tests with your sched/core-sched branch on my arm64
machine: 2 numa * 28 cores * SMT4
List of tests:
perf sched pipe
hackbench process and threads with 1,4,16,32,64,128,256 groups tbench with 1,4,16,32,64,128,256 groups
Unixbench shell and exec
reaim
core-sched was compiled but not used: CONFIG_SCHED_CORE=y
I haven't seen any regressions or problems; it's a mix of +/- 1-2%. I have only seen one significant improvement for hackbench -g 1 (+4.92%) that I don't really explain. Will dig a bit one this later
So Tested-by: Vincent Guittot <[email protected]>
Quoted 29 lines
These patches should shortly be available in my queue.git.
--- b/kernel/sched/core_sched.c | 229 ++++++ b/tools/testing/selftests/sched/.gitignore | 1 b/tools/testing/selftests/sched/Makefile | 14 b/tools/testing/selftests/sched/config | 1 b/tools/testing/selftests/sched/cs_prctl_test.c | 338 +++++++++ include/linux/sched.h | 19 include/uapi/linux/prctl.h | 8 kernel/Kconfig.preempt | 6 kernel/fork.c | 4 kernel/sched/Makefile | 1 kernel/sched/core.c | 858 ++++++++++++++++++++++-- kernel/sched/cpuacct.c | 12 kernel/sched/deadline.c | 38 - kernel/sched/debug.c | 4 kernel/sched/fair.c | 276 +++++-- kernel/sched/idle.c | 13 kernel/sched/pelt.h | 2 kernel/sched/rt.c | 31 kernel/sched/sched.h | 393 ++++++++-- kernel/sched/stop_task.c | 14 kernel/sched/topology.c | 4 kernel/sys.c | 5 tools/include/uapi/linux/prctl.h | 8 23 files changed, 2057 insertions(+), 222 deletions(-)
Quoted 30 lines · Vincent Guittot
Hi Peter,
Quoted 7 lines · Peter Zijlstra
Hai,
This is an agressive fold of all the core-scheduling work so far. I've stripped a whole bunch of tags along the way (hopefully not too many, please yell if you feel I made a mistake), including tested-by. Please retest.
Quoted 8 lines
Changes since the last partial post is dropping all the cgroup stuff and
PR_SCHED_CORE_CLEARas well as thatexec()behaviour in order to later resolve the cgroup issue.Since we're really rather late for the coming merge window, my plan was to merge the lot right after the merge window.
Again, please test.
I have run various tests with your sched/core-sched branch on my arm64
machine: 2 numa * 28 cores * SMT4List of tests:
perf sched pipe
hackbench process and threads with 1,4,16,32,64,128,256 groups tbench with 1,4,16,32,64,128,256 groups
Unixbench shell and exec
reaim
Quoted 7 lines
core-sched was compiled but not used:
CONFIG_SCHED_CORE=yI haven't seen any regressions or problems; it's a mix of +/- 1-2%. I have only seen one significant improvement for hackbench -g 1 (+4.92%) that I don't really explain. Will dig a bit one this later
So Tested-by: Vincent Guittot <[email protected]>
With the correct email address
Quoted 32 lines
Quoted 32 lines
These patches should shortly be available in my queue.git.
--- b/kernel/sched/core_sched.c | 229 ++++++ b/tools/testing/selftests/sched/.gitignore | 1 b/tools/testing/selftests/sched/Makefile | 14 b/tools/testing/selftests/sched/config | 1 b/tools/testing/selftests/sched/cs_prctl_test.c | 338 +++++++++ include/linux/sched.h | 19 include/uapi/linux/prctl.h | 8 kernel/Kconfig.preempt | 6 kernel/fork.c | 4 kernel/sched/Makefile | 1 kernel/sched/core.c | 858 ++++++++++++++++++++++-- kernel/sched/cpuacct.c | 12 kernel/sched/deadline.c | 38 - kernel/sched/debug.c | 4 kernel/sched/fair.c | 276 +++++-- kernel/sched/idle.c | 13 kernel/sched/pelt.h | 2 kernel/sched/rt.c | 31 kernel/sched/sched.h | 393 ++++++++-- kernel/sched/stop_task.c | 14 kernel/sched/topology.c | 4 kernel/sys.c | 5 tools/include/uapi/linux/prctl.h | 8 23 files changed, 2057 insertions(+), 222 deletions(-)