scheduler updates for v2.6.32, part 2
Linus,
Please pull the latest sched-core-for-linus git tree from:
git://git.kernel.org/pub/scm/linux/kernel/git/tip/linux-2.6-tip.git sched-core-for-linus
This is the second (and final) part of scheduler updates. Note, if you merge this there will be an (easy) conflict in arch/x86/include/asm/processor.h.
These changes finish the load balancer rewrite/fixing (fresh code but we didnt want to wait with this for v2.6.33 and it fixes performance regressions as well), plus does re-tuning based on various new testcases brought up in recent lkml scheduler threads.
As a summary of expected impact, in our tests this has improved:
- make -j<nr_cpus> kernel build results
- sysbench (balancing regression, sleeper preemption regression)
- scalability of heavy forkers (the x264 testcase)
- interactivity (buddy latencies, gentler sleeper-fairness)
Right now we dont know about any flip side of these changes yet, and we tried to diligently measure all sides of the coin (and eliminate negative effects). (famous last words)
Thanks,
Ingo
------------------>
Ingo Molnar (2):
sched: Implement a gentler fair-sleepers feature
sched: Fix TASK_WAKING & loadaverage breakage
Mike Galbraith (2):
sched: Complete buddy switches
sched: Improve latencies and throughput
Peter Zijlstra (33):
sched: Fix double_rq_lock() compile warning
sched: Split WAKEUP_OVERLAP
sched: Add come comments to the sched features
sched: Move code around
sched: Move sched_balance_self() into sched_fair.c
sched: Hook sched_balance_self() into sched_class::select_task_rq()
sched: Add TASK_WAKING
sched: Merge select_task_rq_fair() and sched_balance_self()
sched: Weaken SD_POWERSAVINGS_BALANCE
sched: for_each_domain() vs RCU
sched: Fix task affinity for select_task_rq_fair
sched: Tweak wake_idx
sched: Fix some domain tunings
sched: Reduce forkexec_idx
sched: Provide arch_scale_freq_power
x86: Move APERF/MPERF into a X86_FEATURE
x86: Add generic aperf/mperf code
x86: sched: Provide arch implementations using aperf/mperf
sched: Feature to disable APERF/MPERF cpu_power
sched: Rename select_task_rq() argument
sched: Rename sync arguments
sched: Add WF_FORK
sched: Fix sync wakeups again
sched: Add a few SYNC hint knobs to play with
sched: Add SD_PREFER_LOCAL
sched: x86: Name old_perf in a unique way
sched: Optimize cgroup vs wakeup a bit
sched: Clean up the load_idx selection in select_task_rq_fair
sched: Rename flags to wake_flags
sched: Disable wakeup balancing
sched: Add new wakeup preemption mode: WAKEUP_RUNNING
sched: Stop buddies from hogging the system
sched: Fix SD_POWERSAVING_BALANCE|SD_PREFER_LOCAL vs SD_WAKE_AFFINE
arch/ia64/include/asm/topology.h | 17 +- arch/mips/include/asm/mach-ip27/topology.h | 1 - arch/powerpc/include/asm/topology.h | 9 +- arch/sh/include/asm/topology.h | 10 +- arch/sparc/include/asm/topology_64.h | 7 +- arch/x86/include/asm/cpufeature.h | 1 + arch/x86/include/asm/processor.h | 30 ++ arch/x86/include/asm/topology.h | 14 +- arch/x86/kernel/cpu/Makefile | 2 +- arch/x86/kernel/cpu/cpufreq/acpi-cpufreq.c | 88 +----- arch/x86/kernel/cpu/intel.c | 6 + arch/x86/kernel/cpu/sched.c | 55 ++++ include/linux/sched.h | 23 ++- include/linux/topology.h | 32 +-- include/linux/wait.h | 4 +- kernel/sched.c | 444 +++++++++------------------- kernel/sched_debug.c | 1 + kernel/sched_fair.c | 414 ++++++++++++++++---------- kernel/sched_features.h | 122 +++++++- kernel/sched_idletask.c | 4 +- kernel/sched_rt.c | 7 +- 21 files changed, 688 insertions(+), 603 deletions(-) create mode 100644 arch/x86/kernel/cpu/sched.c
arch/ia64/include/asm/topology.h +9 -8
@@ -61,12 +61,13 @@ void build_cpu_to_node_map(void); .cache_nice_tries = 2, \ .busy_idx = 2, \ .idle_idx = 1, \ - .newidle_idx = 2, \ - .wake_idx = 1, \ - .forkexec_idx = 1, \ + .newidle_idx = 0, \ + .wake_idx = 0, \ + .forkexec_idx = 0, \ .flags =SD_LOAD_BALANCE \ |SD_BALANCE_NEWIDLE \ |SD_BALANCE_EXEC \ + |SD_BALANCE_FORK \ | SD_WAKE_AFFINE, \ .last_balance = jiffies, \ .balance_interval = 1, \
@@ -85,14 +86,14 @@ void build_cpu_to_node_map(void); .cache_nice_tries = 2, \ .busy_idx = 3, \ .idle_idx = 2, \ - .newidle_idx = 2, \ - .wake_idx = 1, \ - .forkexec_idx = 1, \ + .newidle_idx = 0, \ + .wake_idx = 0, \ + .forkexec_idx = 0, \ .flags =SD_LOAD_BALANCE \ + |SD_BALANCE_NEWIDLE \ |SD_BALANCE_EXEC \ |SD_BALANCE_FORK \ - |SD_SERIALIZE \ - | SD_WAKE_BALANCE, \ + | SD_SERIALIZE, \ .last_balance = jiffies, \ .balance_interval = 64, \ .nr_balance_failed = 0, \
arch/mips/include/asm/mach-ip27/topology.h +0 -1
@@ -48,7 +48,6 @@ extern unsigned char __node_distances[MAX_COMPACT_NODES][MAX_COMPACT_NODES]; .cache_nice_tries = 1, \ .flags =SD_LOAD_BALANCE \ |SD_BALANCE_EXEC \ - | SD_WAKE_BALANCE, \ .last_balance = jiffies, \ .balance_interval = 1, \ .nr_balance_failed = 0, \
arch/powerpc/include/asm/topology.h +4 -5
@@ -57,14 +57,13 @@ static inline int pcibus_to_node(struct pci_bus *bus) .cache_nice_tries = 1, \ .busy_idx = 3, \ .idle_idx = 1, \ - .newidle_idx = 2, \ - .wake_idx = 1, \ + .newidle_idx = 0, \ + .wake_idx = 0, \ .flags =SD_LOAD_BALANCE \ |SD_BALANCE_EXEC \ + |SD_BALANCE_FORK \ |SD_BALANCE_NEWIDLE \ - |SD_WAKE_IDLE \ - |SD_SERIALIZE \ - | SD_WAKE_BALANCE, \ + | SD_SERIALIZE, \ .last_balance = jiffies, \ .balance_interval = 1, \ .nr_balance_failed = 0, \
arch/sh/include/asm/topology.h +5 -5
@@ -15,14 +15,14 @@ .cache_nice_tries = 2, \ .busy_idx = 3, \ .idle_idx = 2, \ - .newidle_idx = 2, \ - .wake_idx = 1, \ - .forkexec_idx = 1, \ + .newidle_idx = 0, \ + .wake_idx = 0, \ + .forkexec_idx = 0, \ .flags =SD_LOAD_BALANCE \ |SD_BALANCE_FORK \ |SD_BALANCE_EXEC \ - |SD_SERIALIZE \ - | SD_WAKE_BALANCE, \ + |SD_BALANCE_NEWIDLE \ + | SD_SERIALIZE, \ .last_balance = jiffies, \ .balance_interval = 1, \ .nr_balance_failed = 0, \
arch/sparc/include/asm/topology_64.h +3 -4
@@ -52,13 +52,12 @@ static inline int pcibus_to_node(struct pci_bus *pbus) .busy_idx = 3, \ .idle_idx = 2, \ .newidle_idx = 0, \ - .wake_idx = 1, \ - .forkexec_idx = 1, \ + .wake_idx = 0, \ + .forkexec_idx = 0, \ .flags =SD_LOAD_BALANCE \ |SD_BALANCE_FORK \ |SD_BALANCE_EXEC \ - |SD_SERIALIZE \ - | SD_WAKE_BALANCE, \ + | SD_SERIALIZE, \ .last_balance = jiffies, \ .balance_interval = 1, \ }
arch/x86/include/asm/cpufeature.h +1 -0
@@ -96,6 +96,7 @@#define X86_FEATURE_CLFLUSH_MONITOR (3*32+25)/* "" clflush reqd with monitor */ #define X86_FEATURE_EXTD_APICID (3*32+26)/* has extended APICID (8 bits) */ #define X86_FEATURE_AMD_DCM (3*32+27)/* multi-node processor */ +#define X86_FEATURE_APERFMPERF (3*32+28)/* APERFMPERF */ /* Intel-defined CPU features, CPUID level 0x00000001 (ecx), word 4 */ #define X86_FEATURE_XMM3 (4*32+ 0)/* "pni" SSE-3 */
arch/x86/include/asm/processor.h +30 -0
@@ -27,6 +27,7 @@ struct mm_struct;#include <linux/cpumask.h> #include <linux/cache.h> #include <linux/threads.h> +#include <linux/math64.h> #include <linux/init.h> /*
@@ -1020,4 +1021,33 @@ extern void start_thread(struct pt_regs *regs, unsigned long new_ip,extern int get_tsc_mode (unsigned long adr);extern int set_tsc_mode (unsigned int val); +struct aperfmperf { +u64 aperf, mperf; +}; + +static inline void get_aperfmperf (struct aperfmperf *am) +{ +WARN_ON_ONCE (!boot_cpu_has (X86_FEATURE_APERFMPERF )); + +rdmsrl (MSR_IA32_APERF , am->aperf); +rdmsrl (MSR_IA32_MPERF , am->mperf); +} + +#define APERFMPERF_SHIFT 10 + +static inline +unsigned long calc_aperfmperf_ratio (struct aperfmperf *old, +struct aperfmperf *new) +{ +u64 aperf = new->aperf - old->aperf; +u64 mperf = new->mperf - old->mperf; +unsigned long ratio = aperf; + + mperf >>=APERFMPERF_SHIFT ; +if (mperf) + ratio =div64_u64 (aperf, mperf); + +return ratio; +} +#endif /* _ASM_X86_PROCESSOR_H */
arch/x86/include/asm/topology.h +4 -10
@@ -116,15 +116,11 @@ extern unsigned long node_remap_size[];# define SD_CACHE_NICE_TRIES 1# define SD_IDLE_IDX 1 -# define SD_NEWIDLE_IDX 2 -# define SD_FORKEXEC_IDX 0#else # define SD_CACHE_NICE_TRIES 2# define SD_IDLE_IDX 2 -# define SD_NEWIDLE_IDX 2 -# define SD_FORKEXEC_IDX 1#endif
@@ -137,22 +133,20 @@ extern unsigned long node_remap_size[]; .cache_nice_tries = SD_CACHE_NICE_TRIES, \ .busy_idx = 3, \ .idle_idx = SD_IDLE_IDX, \ - .newidle_idx = SD_NEWIDLE_IDX, \ - .wake_idx = 1, \ - .forkexec_idx = SD_FORKEXEC_IDX, \ + .newidle_idx = 0, \ + .wake_idx = 0, \ + .forkexec_idx = 0, \ \ .flags =1 *SD_LOAD_BALANCE \ |1 *SD_BALANCE_NEWIDLE \ |1 *SD_BALANCE_EXEC \ |1 *SD_BALANCE_FORK \ - |0 *SD_WAKE_IDLE \ + |0 *SD_BALANCE_WAKE \ |1 *SD_WAKE_AFFINE \ - |1 *SD_WAKE_BALANCE \ |0 *SD_SHARE_CPUPOWER \ |0 *SD_POWERSAVINGS_BALANCE \ |0 *SD_SHARE_PKG_RESOURCES \ |1 *SD_SERIALIZE \ - |1 *SD_WAKE_IDLE_FAR \ |0 *SD_PREFER_SIBLING \ , \ .last_balance = jiffies, \
arch/x86/kernel/cpu/Makefile +1 -1
@@ -13,7 +13,7 @@ CFLAGS_common.o := $(nostackp) obj-y := intel_cacheinfo.o addon_cpuid_features.o obj-y += proc.o capflags.o powerflags.o common.o -obj-y += vmware.o hypervisor.o +obj-y += vmware.o hypervisor.o sched.o obj-$(CONFIG_X86_32) += bugs.o cmpxchg.o obj-$(CONFIG_X86_64) += bugs_64.o
arch/x86/kernel/cpu/cpufreq/acpi-cpufreq.c +11 -77
127 lines not on this page · raw message
arch/x86/kernel/cpu/intel.c +6 -0
@@ -350,6 +350,12 @@ static void __cpuinit init_intel(struct cpuinfo_x86 *c)set_cpu_cap (c,X86_FEATURE_ARCH_PERFMON ); } +if (c->cpuid_level >6 ) { +unsigned ecx =cpuid_ecx (6 ); +if (ecx &0x01 ) +set_cpu_cap (c,X86_FEATURE_APERFMPERF ); + } +if (cpu_has_xmm2)set_cpu_cap (c,X86_FEATURE_LFENCE_RDTSC );if (cpu_has_ds) {
arch/x86/kernel/cpu/sched.c new file +55 -0
55 lines not on this page · raw message
include/linux/sched.h +18 -5
@@ -190,6 +190,7 @@ extern unsigned long long time_sync_thresh;/* in tsk->state again */ #define TASK_DEAD 64#define TASK_WAKEKILL 128 +#define TASK_WAKING 256/* Convenience macros for the sake of set_task_state */ #define TASK_KILLABLE (TASK_WAKEKILL | TASK_UNINTERRUPTIBLE)
@@ -802,14 +803,14 @@ enum cpu_idle_type {#define SD_BALANCE_NEWIDLE 0x0002/* Balance when about to become idle */ #define SD_BALANCE_EXEC 0x0004/* Balance on exec */ #define SD_BALANCE_FORK 0x0008/* Balance on fork, clone */ -#define SD_WAKE_IDLE 0x0010/* Wake to idle CPU on task wakeup */ +#define SD_BALANCE_WAKE 0x0010/* Balance on wakeup */ #define SD_WAKE_AFFINE 0x0020/* Wake task to waking CPU */ -#define SD_WAKE_BALANCE 0x0040/* Perform balancing at task wakeup */ +#define SD_PREFER_LOCAL 0x0040/* Prefer to keep tasks local to this domain */ #define SD_SHARE_CPUPOWER 0x0080/* Domain members share cpu power */ #define SD_POWERSAVINGS_BALANCE 0x0100/* Balance for power savings */ #define SD_SHARE_PKG_RESOURCES 0x0200/* Domain members share cpu pkg resources */ #define SD_SERIALIZE 0x0400/* Only a single load balancing instance */ -#define SD_WAKE_IDLE_FAR 0x0800/* Gain latency sacrificing cache hit */ +#define SD_PREFER_SIBLING 0x1000/* Prefer to place tasks in a sibling domain */ enum powersavings_balance_level {
@@ -991,6 +992,9 @@ static inline int test_sd_parent(struct sched_domain *sd, int flag) return 0; } +unsigned long default_scale_freq_power (struct sched_domain *sd,int cpu); +unsigned long default_scale_smt_power (struct sched_domain *sd,int cpu); +#else /* CONFIG_SMP */ struct sched_domain_attr ;
@@ -1002,6 +1006,7 @@ partition_sched_domains(int ndoms_new, struct cpumask *doms_new, }#endif /* !CONFIG_SMP */ +struct io_context ;/* See blkdev.h */
@@ -1019,6 +1024,12 @@ struct uts_namespace;struct rq ;struct sched_domain ; +/* +* wake flags +*/ +#define WF_SYNC 0x01/* waker goes to sleep after wakup */ +#define WF_FORK 0x02/* child wakeup after fork */ +struct sched_class {const struct sched_class *next;
@@ -1026,13 +1037,13 @@ struct sched_class {void (*dequeue_task) (struct rq *rq,struct task_struct *p,int sleep);void (*yield_task) (struct rq *rq); -void (*check_preempt_curr) (struct rq *rq,struct task_struct *p,int sync); +void (*check_preempt_curr) (struct rq *rq,struct task_struct *p,int flags);struct task_struct * (*pick_next_task) (struct rq *rq);void (*put_prev_task) (struct rq *rq,struct task_struct *p);#ifdef CONFIG_SMP -int (*select_task_rq)(struct task_struct *p,int sync); +int (*select_task_rq)(struct task_struct *p,int sd_flag,int flags); unsigned long (*load_balance) (struct rq *this_rq, int this_cpu, struct rq *busiest, unsigned long max_load_move,
@@ -1102,6 +1113,8 @@ struct sched_entity {u64 start_runtime;u64 avg_wakeup; +u64 avg_running; +#ifdef CONFIG_SCHEDSTATS u64 wait_start;u64 wait_max;
include/linux/topology.h +13 -19
@@ -95,14 +95,12 @@ int arch_update_cpu_topology(void); | 1*SD_BALANCE_NEWIDLE \ |1 *SD_BALANCE_EXEC \ |1 *SD_BALANCE_FORK \ - |0 *SD_WAKE_IDLE \ + |0 *SD_BALANCE_WAKE \ |1 *SD_WAKE_AFFINE \ - |1 *SD_WAKE_BALANCE \ |1 *SD_SHARE_CPUPOWER \ |0 *SD_POWERSAVINGS_BALANCE \ |0 *SD_SHARE_PKG_RESOURCES \ |0 *SD_SERIALIZE \ - |0 *SD_WAKE_IDLE_FAR \ |0 *SD_PREFER_SIBLING \ , \ .last_balance = jiffies, \
@@ -122,20 +120,19 @@ int arch_update_cpu_topology(void); .imbalance_pct = 125, \ .cache_nice_tries = 1, \ .busy_idx = 2, \ - .wake_idx = 1, \ - .forkexec_idx = 1, \ + .wake_idx = 0, \ + .forkexec_idx = 0, \ \ .flags =1 *SD_LOAD_BALANCE \ |1 *SD_BALANCE_NEWIDLE \ |1 *SD_BALANCE_EXEC \ |1 *SD_BALANCE_FORK \ - |1 *SD_WAKE_IDLE \ + |0 *SD_BALANCE_WAKE \ |1 *SD_WAKE_AFFINE \ - |1 *SD_WAKE_BALANCE \ + |1 *SD_PREFER_LOCAL \ |0 *SD_SHARE_CPUPOWER \ |1 *SD_SHARE_PKG_RESOURCES \ |0 *SD_SERIALIZE \ - |0 *SD_WAKE_IDLE_FAR \ |sd_balance_for_mc_power () \ |sd_power_saving_flags () \ , \
@@ -155,21 +152,20 @@ int arch_update_cpu_topology(void); .cache_nice_tries = 1, \ .busy_idx = 2, \ .idle_idx = 1, \ - .newidle_idx = 2, \ - .wake_idx = 1, \ - .forkexec_idx = 1, \ + .newidle_idx = 0, \ + .wake_idx = 0, \ + .forkexec_idx = 0, \ \ .flags =1 *SD_LOAD_BALANCE \ |1 *SD_BALANCE_NEWIDLE \ |1 *SD_BALANCE_EXEC \ |1 *SD_BALANCE_FORK \ - |1 *SD_WAKE_IDLE \ - |0 *SD_WAKE_AFFINE \ - |1 *SD_WAKE_BALANCE \ + |0 *SD_BALANCE_WAKE \ + |1 *SD_WAKE_AFFINE \ + |1 *SD_PREFER_LOCAL \ |0 *SD_SHARE_CPUPOWER \ |0 *SD_SHARE_PKG_RESOURCES \ |0 *SD_SERIALIZE \ - |0 *SD_WAKE_IDLE_FAR \ |sd_balance_for_package_power () \ |sd_power_saving_flags () \ , \
@@ -191,14 +187,12 @@ int arch_update_cpu_topology(void); | 1*SD_BALANCE_NEWIDLE \ |0 *SD_BALANCE_EXEC \ |0 *SD_BALANCE_FORK \ - |0 *SD_WAKE_IDLE \ - |1 *SD_WAKE_AFFINE \ - |0 *SD_WAKE_BALANCE \ + |0 *SD_BALANCE_WAKE \ + |0 *SD_WAKE_AFFINE \ |0 *SD_SHARE_CPUPOWER \ |0 *SD_POWERSAVINGS_BALANCE \ |0 *SD_SHARE_PKG_RESOURCES \ |1 *SD_SERIALIZE \ - |1 *SD_WAKE_IDLE_FAR \ |0 *SD_PREFER_SIBLING \ , \ .last_balance = jiffies, \
include/linux/wait.h +2 -2
@@ -26,8 +26,8 @@#include <asm/current.h> typedef struct __wait_queue wait_queue_t ; -typedef int (*wait_queue_func_t )(wait_queue_t *wait,unsigned mode,int sync,void *key); -int default_wake_function (wait_queue_t *wait,unsigned mode,int sync,void *key); +typedef int (*wait_queue_func_t )(wait_queue_t *wait,unsigned mode,int flags,void *key); +int default_wake_function (wait_queue_t *wait,unsigned mode,int flags,void *key);struct __wait_queue {unsigned int flags;
kernel/sched.c +143 -301
672 lines not on this page · raw message
kernel/sched_debug.c +1 -0
@@ -395,6 +395,7 @@ void proc_sched_show_task(struct task_struct *p, struct seq_file *m)PN (se.sum_exec_runtime);PN (se.avg_overlap);PN (se.avg_wakeup); +PN (se.avg_running); nr_switches = p->nvcsw + p->nivcsw;
kernel/sched_fair.c +261 -153
537 lines not on this page · raw message
kernel/sched_features.h +114 -8
131 lines not on this page · raw message
kernel/sched_idletask.c +2 -2
@@ -6,7 +6,7 @@*/ #ifdef CONFIG_SMP -static int select_task_rq_idle (struct task_struct *p,int sync) +static int select_task_rq_idle (struct task_struct *p,int sd_flag,int flags) {return task_cpu (p);/* IDLE tasks as never migrated */ }
@@ -14,7 +14,7 @@ static int select_task_rq_idle(struct task_struct *p, int sync)/* * Idle tasks are unconditionally rescheduled: */ -static void check_preempt_curr_idle (struct rq *rq,struct task_struct *p,int sync) +static void check_preempt_curr_idle (struct rq *rq,struct task_struct *p,int flags) {resched_task (rq->idle); }
kernel/sched_rt.c +5 -2
@@ -938,10 +938,13 @@ static void yield_task_rt(struct rq *rq)#ifdef CONFIG_SMP static int find_lowest_rq (struct task_struct *task); -static int select_task_rq_rt (struct task_struct *p,int sync) +static int select_task_rq_rt (struct task_struct *p,int sd_flag,int flags) {struct rq *rq =task_rq (p); +if (sd_flag !=SD_BALANCE_WAKE ) +return smp_processor_id (); +/* * If the current task is an RT task, then * try to see if we can wake this RT task up on another
@@ -999,7 +1002,7 @@ static void check_preempt_equal_prio(struct rq *rq, struct task_struct *p)/* * Preempt the current task with a newly woken task if needed: */ -static void check_preempt_curr_rt (struct rq *rq,struct task_struct *p,int sync) +static void check_preempt_curr_rt (struct rq *rq,struct task_struct *p,int flags) {if (p->prio < rq->curr->prio) {resched_task (rq->curr);