scheduler fixes
Linus,
Please pull the latest sched-urgent-for-linus git tree from:
git://git.kernel.org/pub/scm/linux/kernel/git/tip/tip.git sched-urgent-for-linus
# HEAD: f94c8d116997597fc00f0812b0ab9256e7b0c58f sched/clock, x86/tsc: Rework the x86 'unstable' sched_clock() interface
A fix for KVM's scheduler clock which (erroneously) was always marked unstable, a fix for RT/DL load balancing, plus latency fixes.
Thanks,
Ingo
------------------>
Peter Zijlstra (3):
sched/fair: Make select_idle_cpu() more aggressive sched/core: Fix pick_next_task() for RT,DL
sched/clock, x86/tsc: Rework the x86 'unstable' sched_clock() interface
arch/x86/kernel/cpu/amd.c | 4 ---- arch/x86/kernel/cpu/centaur.c | 2 -- arch/x86/kernel/cpu/common.c | 3 --- arch/x86/kernel/cpu/cyrix.c | 1 - arch/x86/kernel/cpu/intel.c | 4 ---- arch/x86/kernel/cpu/transmeta.c | 2 -- arch/x86/kernel/tsc.c | 35 +++++++++++++++++++++++------------ kernel/sched/core.c | 11 ++++++++--- kernel/sched/fair.c | 2 +- kernel/sched/features.h | 5 +++++ 10 files changed, 37 insertions(+), 32 deletions(-)
arch/x86/kernel/cpu/amd.c +0 -4
@@ -555,10 +555,6 @@ static void early_init_amd(struct cpuinfo_x86 *c)if (c->x86_power & (1 <<8 )) {set_cpu_cap (c,X86_FEATURE_CONSTANT_TSC );set_cpu_cap (c,X86_FEATURE_NONSTOP_TSC ); -if (check_tsc_unstable ()) -clear_sched_clock_stable (); - }else { -clear_sched_clock_stable (); }/* Bit 12 of 8000_0007 edx is accumulated power mechanism. */
arch/x86/kernel/cpu/centaur.c +0 -2
@@ -104,8 +104,6 @@ static void early_init_centaur(struct cpuinfo_x86 *c)#ifdef CONFIG_X86_64 set_cpu_cap (c,X86_FEATURE_SYSENTER32 );#endif - -clear_sched_clock_stable (); }static void init_centaur (struct cpuinfo_x86 *c)
arch/x86/kernel/cpu/common.c +0 -3
@@ -86,7 +86,6 @@ static void default_init(struct cpuinfo_x86 *c)strcpy (c->x86_model_id,"386" ); }#endif -clear_sched_clock_stable (); } static const struct cpu_dev default_cpu = {
@@ -1075,8 +1074,6 @@ static void identify_cpu(struct cpuinfo_x86 *c)*/ if (this_cpu->c_init) this_cpu->c_init (c); -else -clear_sched_clock_stable ();/* Disable the PN if appropriate */ squash_the_stupid_serial_number (c);
arch/x86/kernel/cpu/cyrix.c +0 -1
@@ -184,7 +184,6 @@ static void early_init_cyrix(struct cpuinfo_x86 *c)set_cpu_cap (c,X86_FEATURE_CYRIX_ARR );break ; } -clear_sched_clock_stable (); }static void init_cyrix (struct cpuinfo_x86 *c)
arch/x86/kernel/cpu/intel.c +0 -4
@@ -161,10 +161,6 @@ static void early_init_intel(struct cpuinfo_x86 *c)if (c->x86_power & (1 <<8 )) {set_cpu_cap (c,X86_FEATURE_CONSTANT_TSC );set_cpu_cap (c,X86_FEATURE_NONSTOP_TSC ); -if (check_tsc_unstable ()) -clear_sched_clock_stable (); - }else { -clear_sched_clock_stable (); }/* Penwell and Cloverview have the TSC which doesn't sleep on S3 */
arch/x86/kernel/cpu/transmeta.c +0 -2
@@ -15,8 +15,6 @@ static void early_init_transmeta(struct cpuinfo_x86 *c)if (xlvl >=0x80860001 ) c->x86_capability[CPUID_8086_0001_EDX ] =cpuid_edx (0x80860001 ); } - -clear_sched_clock_stable (); }static void init_transmeta (struct cpuinfo_x86 *c)
arch/x86/kernel/tsc.c +23 -12
@@ -326,9 +326,16 @@ unsigned long long sched_clock(void) {return paravirt_sched_clock (); } + +static inline bool using_native_sched_clock (void ) +{ +return pv_time_ops.sched_clock == native_sched_clock; +}#else unsigned long long sched_clock (void ) __attribute__((alias ("native_sched_clock" ))); + +static inline bool using_native_sched_clock (void ) {return true; }#endif int check_tsc_unstable (void )
@@ -1111,8 +1118,10 @@ static void tsc_cs_mark_unstable(struct clocksource *cs) {if (tsc_unstable)return ; + tsc_unstable =1 ; -clear_sched_clock_stable (); +if (using_native_sched_clock ()) +clear_sched_clock_stable ();disable_sched_clock_irqtime ();pr_info ("Marking TSC unstable due to clocksource watchdog\n" ); }
@@ -1134,18 +1143,20 @@ static struct clocksource clocksource_tsc = {void mark_tsc_unstable (char *reason) { -if (!tsc_unstable) { - tsc_unstable =1 ; +if (tsc_unstable) +return ; + + tsc_unstable =1 ; +if (using_native_sched_clock ())clear_sched_clock_stable (); -disable_sched_clock_irqtime (); -pr_info ("Marking TSC unstable due to %s\n" , reason); -/* Change only the rating, when not registered */ -if (clocksource_tsc.mult) -clocksource_mark_unstable (&clocksource_tsc); -else { - clocksource_tsc.flags |=CLOCK_SOURCE_UNSTABLE ; - clocksource_tsc.rating =0 ; - } +disable_sched_clock_irqtime (); +pr_info ("Marking TSC unstable due to %s\n" , reason); +/* Change only the rating, when not registered */ +if (clocksource_tsc.mult) { +clocksource_mark_unstable (&clocksource_tsc); + }else { + clocksource_tsc.flags |=CLOCK_SOURCE_UNSTABLE ; + clocksource_tsc.rating =0 ; } }
kernel/sched/core.c +8 -3
@@ -3273,10 +3273,15 @@ pick_next_task(struct rq *rq, struct task_struct *prev, struct rq_flags *rf)struct task_struct *p;/* -* Optimization: we know that if all tasks are in -* the fair class we can call that function directly: +* Optimization: we know that if all tasks are in the fair class we can +* call that function directly, but only if the @prev task wasn't of a +* higher scheduling class, because otherwise those loose the +* opportunity to pull in more work from other CPUs. */ -if (likely (rq->nr_running == rq->cfs.h_nr_running)) { +if (likely ((prev->sched_class == &idle_sched_class || + prev->sched_class == &fair_sched_class) && + rq->nr_running == rq->cfs.h_nr_running)) { + p = fair_sched_class.pick_next_task (rq, prev, rf);if (unlikely (p ==RETRY_TASK )) goto again;
kernel/sched/fair.c +1 -1
@@ -5797,7 +5797,7 @@ static int select_idle_cpu(struct task_struct *p, struct sched_domain *sd, int t* Due to large variance we need a large fuzz factor; hackbench in * particularly is sensitive here. */ -if ((avg_idle /512 ) < avg_cost) +if (sched_feat (SIS_AVG_CPU ) && (avg_idle /512 ) < avg_cost)return -1 ; time =local_clock ();
kernel/sched/features.h +5 -0
@@ -51,6 +51,11 @@ SCHED_FEAT(NONTASK_CAPACITY, true)*/ SCHED_FEAT (TTWU_QUEUE , true) +/* +* When doing wakeups, attempt to limit superfluous scans of the LLC domain. + */ +SCHED_FEAT(SIS_AVG_CPU, false) +#ifdef HAVE_RT_PUSH_IPI /* * In order to avoid a thundering herd attack of CPUs that are