scheduler changes for v4.1
Linus,
Please pull the latest sched-core-for-linus git tree from:
git://git.kernel.org/pub/scm/linux/kernel/git/tip/tip.git sched-core-for-linus
# HEAD: 62a935b256f68a71697716595347209fb5275426 sched/core: Drop debugging leftover trace_printk call
Major changes:
- Reworked CPU capacity code, for better SMP load balancing on systems with assymetric CPUs. (Vincent Guittot, Morten Rasmussen)
- Reworked RT task SMP balancing to be push based instead of pull based, to reduce latencies on large CPU count systems. (Steven Rostedt)
- SCHED_DEADLINE support updates and fixes. (Juri Lelli)
- SCHED_DEADLINE task migration support during CPU hotplug. (Wanpeng Li)
- x86 mwait-idle optimizations and fixes. (Mike Galbraith, Len Brown)
- sched/numa improvements. (Rik van Riel)
- various cleanups
Thanks,
Ingo
------------------>
Abel Vesa (1):
sched/core: Remove unused argument from init_[rt|dl]_rq()
Borislav Petkov (1):
sched/core: Drop debugging leftover trace_printk call
Jan Beulich (1):
sched/numa: Avoid some pointless iterations
Juri Lelli (2):
sched/deadline: Always enqueue on previous rq when dl_task_timer() fires sched/core: Check for available DL bandwidth in cpuset_cpu_inactive()
Len Brown (1):
sched/idle/x86: Restore mwait_idle() to fix boot hangs, to improve power savings and to improve performance
Mike Galbraith (1):
sched/idle/x86: Optimize unnecessary mwait_idle() resched IPIs
Morten Rasmussen (2):
sched: Track group sched_entity usage contributions
sched: Make sched entity usage tracking scale-invariant
Peter Zijlstra (1):
sched: Optimize freq invariant accounting
Preeti U Murthy (1):
sched: Improve load balancing in the presence of idle CPUs
Rik van Riel (2):
sched/numa: Do not move past the balance point if unbalanced sched/rt/nohz: Stop scheduler tick if running realtime task
Steven Rostedt (2):
irq_work: Fix build failure when CONFIG_IRQ_WORK is not defined sched/rt: Use IPI to trigger RT task push migration instead of pulling
Vincent Guittot (9):
sched: Add sched_avg::utilization_avg_contrib
sched: Remove frequency scaling from cpu_capacity
sched: Make scale_rt invariant with frequency
sched: Add struct rq::cpu_capacity_orig
sched: Calculate CPU's usage statistic and put it into struct sg_lb_stats::group_usage
sched: Replace capacity_factor by usage
sched: Remove unused struct sched_group_capacity::capacity_orig
sched: Add SD_PREFER_SIBLING for SMT level
sched: Move CFS tasks to CPUs with higher capacity
Wanpeng Li (4):
sched/deadline: Add rq->clock update skip for dl task yield sched/deadline: Avoid a superfluous check
sched/deadline: Fix rt runtime corruption when dl fails its global constraints sched/deadline: Support DL task migration during CPU hotplug
arch/x86/include/asm/mwait.h | 8 + arch/x86/kernel/process.c | 51 ++++++ include/linux/irq_work.h | 3 +- include/linux/sched.h | 21 ++- kernel/sched/core.c | 96 +++++----- kernel/sched/deadline.c | 77 ++++++-- kernel/sched/debug.c | 12 +- kernel/sched/fair.c | 425 ++++++++++++++++++++++++++----------------- kernel/sched/features.h | 13 ++ kernel/sched/rt.c | 181 +++++++++++++++++- kernel/sched/sched.h | 38 +++- 11 files changed, 690 insertions(+), 235 deletions(-)
arch/x86/include/asm/mwait.h +8 -0
@@ -30,6 +30,14 @@ static inline void __mwait(unsigned long eax, unsigned long ecx) :: "a" (eax), "c" (ecx)); } +static inline void __sti_mwait (unsigned long eax,unsigned long ecx) +{ +trace_hardirqs_on (); +/* "mwait %eax, %ecx;" */ + asmvolatile ("sti; .byte 0x0f, 0x01, 0xc9;" + ::"a" (eax),"c" (ecx)); +} +/* * This uses new MONITOR/MWAIT instructions on P4 processors with PNI, * which can obviate IPI to trigger checking of need_resched.
arch/x86/kernel/process.c +51 -0
69 lines not on this page · raw message
include/linux/irq_work.h +2 -1
@@ -38,16 +38,17 @@ bool irq_work_queue(struct irq_work *work);bool irq_work_queue_on (struct irq_work *work,int cpu);#endif -void irq_work_run (void );void irq_work_tick (void );void irq_work_sync (struct irq_work *work);#ifdef CONFIG_IRQ_WORK #include <asm/irq_work.h> +void irq_work_run (void );bool irq_work_needs_cpu (void );#else static inline bool irq_work_needs_cpu (void ) {return false; } +static inline void irq_work_run (void ) { }#endif #endif /* _LINUX_IRQ_WORK_H */
include/linux/sched.h +17 -4
@@ -1115,15 +1115,28 @@ struct load_weight { };struct sched_avg { +u64 last_runnable_update; +s64 decay_count; +/* +* utilization_avg_contrib describes the amount of time that a +* sched_entity is running on a CPU. It is based on running_avg_sum +* and is scaled in the range [0..SCHED_LOAD_SCALE]. +* load_avg_contrib described the amount of time that a sched_entity +* is runnable on a rq. It is based on both runnable_avg_sum and the +* weight of the task. +*/ +unsigned long load_avg_contrib, utilization_avg_contrib;/* * These sums represent an infinite geometric series and so are bound * above by 1024/(1-y). Thus we only need a u32 to store them for all * choices of y < 1-2^(-32)*1024. +* running_avg_sum reflects the time that the sched_entity is +* effectively running on the CPU. +* runnable_avg_sum represents the amount of time a sched_entity is on +* a runqueue which includes the running time that is monitored by +* running_avg_sum. */ -u32 runnable_avg_sum, runnable_avg_period; -u64 last_runnable_update; -s64 decay_count; -unsigned long load_avg_contrib; +u32 runnable_avg_sum, avg_period, running_avg_sum; };#ifdef CONFIG_SCHEDSTATS
kernel/sched/core.c +50 -46
170 lines not on this page · raw message
kernel/sched/deadline.c +66 -11
119 lines not on this page · raw message
kernel/sched/debug.c +9 -3
@@ -71,7 +71,7 @@ static void print_cfs_group_stats(struct seq_file *m, int cpu, struct task_groupif (!se) {struct sched_avg *avg = &cpu_rq (cpu)->avg;P (avg->runnable_avg_sum); -P (avg->runnable_avg_period); +P (avg->avg_period);return ; }
@@ -94,8 +94,10 @@ static void print_cfs_group_stats(struct seq_file *m, int cpu, struct task_groupP (se->load.weight);#ifdef CONFIG_SMP P (se->avg.runnable_avg_sum); -P (se->avg.runnable_avg_period); +P (se->avg.running_avg_sum); +P (se->avg.avg_period);P (se->avg.load_avg_contrib); +P (se->avg.utilization_avg_contrib);P (se->avg.decay_count);#endif #undef PN
@@ -214,6 +216,8 @@ void print_cfs_rq(struct seq_file *m, int cpu, struct cfs_rq *cfs_rq) cfs_rq->runnable_load_avg);SEQ_printf (m," .%-30s: %ld\n" ,"blocked_load_avg" , cfs_rq->blocked_load_avg); +SEQ_printf (m," .%-30s: %ld\n" ,"utilization_load_avg" , + cfs_rq->utilization_load_avg);#ifdef CONFIG_FAIR_GROUP_SCHED SEQ_printf (m," .%-30s: %ld\n" ,"tg_load_contrib" , cfs_rq->tg_load_contrib);
@@ -636,8 +640,10 @@ void proc_sched_show_task(struct task_struct *p, struct seq_file *m)P (se.load.weight);#ifdef CONFIG_SMP P (se.avg.runnable_avg_sum); -P (se.avg.runnable_avg_period); +P (se.avg.running_avg_sum); +P (se.avg.avg_period);P (se.avg.load_avg_contrib); +P (se.avg.utilization_avg_contrib);P (se.avg.decay_count);#endif P (policy);
kernel/sched/fair.c +262 -163
818 lines not on this page · raw message
kernel/sched/features.h +13 -0
@@ -56,6 +56,19 @@ SCHED_FEAT(NONTASK_CAPACITY, true)*/ SCHED_FEAT(TTWU_QUEUE, true) +#ifdef HAVE_RT_PUSH_IPI +/* +* In order to avoid a thundering herd attack of CPUs that are +* lowering their priorities at the same time, and there being +* a single CPU that has an RT task that can migrate and is waiting +* to run, where the other CPUs will try to take that CPUs +* rq lock and possibly create a large contention, sending an +* IPI to that CPU and let that CPU push the RT task to where +* it should go may be a better scenario. +*/ +SCHED_FEAT(RT_PUSH_IPI, true) +#endif + SCHED_FEAT(FORCE_SD_OVERLAP, false) SCHED_FEAT(RT_RUNTIME_SHARE, true)SCHED_FEAT (LB_MIN , false)
kernel/sched/rt.c +179 -2
218 lines not on this page · raw message
kernel/sched/sched.h +33 -5
@@ -6,6 +6,7 @@#include <linux/mutex.h> #include <linux/spinlock.h> #include <linux/stop_machine.h> +#include <linux/irq_work.h> #include <linux/tick.h> #include <linux/slab.h>
@@ -362,8 +363,14 @@ struct cfs_rq {* Under CFS, load is tracked on a per-entity basis and aggregated up. * This allows for the description of both thread and group usage (in * the FAIR_GROUP_SCHED case). +* runnable_load_avg is the sum of the load_avg_contrib of the +* sched_entities on the rq. +* blocked_load_avg is similar to runnable_load_avg except that its +* the blocked sched_entities on the rq. +* utilization_load_avg is the sum of the average running time of the +* sched_entities on the rq. */ -unsigned long runnable_load_avg, blocked_load_avg; +unsigned long runnable_load_avg, blocked_load_avg, utilization_load_avg;atomic64_t decay_counter;u64 last_decay;atomic_long_t removed_load;
@@ -418,6 +425,11 @@ static inline int rt_bandwidth_enabled(void)return sysctl_sched_rt_runtime >=0 ; } +/* RT IPI pull logic requires IRQ_WORK */ +#ifdef CONFIG_IRQ_WORK +# define HAVE_RT_PUSH_IPI +#endif +/* Real-Time classes' related field in a runqueue: */ struct rt_rq {struct rt_prio_array active;
@@ -435,7 +447,13 @@ struct rt_rq {unsigned long rt_nr_total;int overloaded;struct plist_head pushable_tasks; +#ifdef HAVE_RT_PUSH_IPI +int push_flags; +int push_cpu; +struct irq_work push_work; +raw_spinlock_t push_lock;#endif +#endif /* CONFIG_SMP */ int rt_queued;int rt_throttled;
@@ -597,6 +615,7 @@ struct rq {struct sched_domain *sd;unsigned long cpu_capacity; +unsigned long cpu_capacity_orig;unsigned char idle_balance;/* For active balancing */
@@ -807,7 +826,7 @@ struct sched_group_capacity {* CPU capacity of this group, SCHED_LOAD_SCALE being max capacity * for a single CPU. */ -unsigned int capacity, capacity_orig; +unsigned int capacity;unsigned long next_update;int imbalance;/* XXX unrelated to capacity but shared group state */ /*
@@ -1368,9 +1387,18 @@ static inline int hrtick_enabled(struct rq *rq)#ifdef CONFIG_SMP extern void sched_avg_update (struct rq *rq); + +#ifndef arch_scale_freq_capacity +static __always_inline +unsigned long arch_scale_freq_capacity (struct sched_domain *sd,int cpu) +{ +return SCHED_CAPACITY_SCALE ; +} +#endif +static inline void sched_rt_avg_update (struct rq *rq,u64 rt_delta) { - rq->rt_avg += rt_delta; + rq->rt_avg += rt_delta *arch_scale_freq_capacity (NULL ,cpu_of (rq));sched_avg_update (rq); }#else
@@ -1643,8 +1671,8 @@ extern void print_rt_stats(struct seq_file *m, int cpu);extern void print_dl_stats (struct seq_file *m,int cpu);extern void init_cfs_rq (struct cfs_rq *cfs_rq); -extern void init_rt_rq (struct rt_rq *rt_rq,struct rq *rq); -extern void init_dl_rq (struct dl_rq *dl_rq,struct rq *rq); +extern void init_rt_rq (struct rt_rq *rt_rq); +extern void init_dl_rq (struct dl_rq *dl_rq);extern void cfs_bandwidth_usage_inc (void );extern void cfs_bandwidth_usage_dec (void );