Scheduler updates for v2.6.36
Subsystems (11)
Linus,
Please pull the latest sched-core-for-linus git tree from:
git://git.kernel.org/pub/scm/linux/kernel/git/tip/linux-2.6-tip.git sched-core-for-linus
The most user-visible bits could be the load-balancer enhancements - hopefully for the better ;-)
Thanks,
Ingo
------------------>
Josh Hunt (1):
sched: Use correct macro to display sched_child_runs_first in /proc/sched_debug
Michael Neuling (5):
sched: Add asymmetric group packing option for sibling domain
powerpc: Enable asymmetric SMT scheduling on POWER7
sched: Fix fix_small_capacity
sched: Fix comments to make them DocBook happy
sched: Fix spelling of sibling
Oleg Nesterov (7):
sched: Simplify the reacquire_kernel_lock() logic
sched: __sched_setscheduler: Read the RLIMIT_RTPRIO value lockless
sched: task_tick_rt: Remove the obsolete ->signal != NULL check
sched: Remove the obsolete exit_state/signal hacks
sched: thread_group_cputime: Simplify, document the "alive" check
sched: run_posix_cpu_timers: Don't check ->exit_state, use lock_task_sighand()
sched: Fix the racy usage of thread_group_cputimer() in fastpath_timer_check()
Pekka Enberg (1):
sched: No need for bootmem special cases
Peter Zijlstra (4):
sched_clock: Add local_clock() API and improve documentation
powerpc: Exclude arch_sd_sibiling_asym_packing() on UP
sched: Reduce update_group_power() calls
sched: Revert nohz_ratelimit() for now
Srivatsa Vaddagiri (1):
sched: Fix capacity calculations for SMT4
Suresh Siddha (1):
sched: Update rq->clock for nohz balanced cpus
Tejun Heo (5):
sched: define and use CPU_PRI_* enums for cpu notifier priorities
sched: adjust when cpu_active and cpuset configurations are updated during cpu on/offlining
sched: refactor try_to_wake_up()
sched: add hooks for workqueue
sched, cpuset: Drop __cpuexit from cpu hotplug callbacks
Venkatesh Pallipadi (2):
sched: Avoid side-effect of tickless idle on update_cpu_load
sched: Change nohz idle load balancing logic to push model
arch/parisc/kernel/ftrace.c | 4 +- arch/powerpc/include/asm/cputable.h | 3 +- arch/powerpc/kernel/process.c | 11 + include/linux/cpu.h | 25 ++ include/linux/cpuset.h | 6 + include/linux/perf_event.h | 2 +- include/linux/sched.h | 59 ++-- include/linux/topology.h | 1 + kernel/cpu.c | 6 - kernel/cpuset.c | 21 +-- kernel/fork.c | 2 +- kernel/hrtimer.c | 8 +- kernel/lockdep.c | 2 +- kernel/perf_event.c | 2 +- kernel/posix-cpu-timers.c | 36 +-- kernel/rcutorture.c | 3 +- kernel/sched.c | 391 +++++++++++++++++++------- kernel/sched_clock.c | 95 ++++++- kernel/sched_cpupri.c | 8 +- kernel/sched_cpupri.h | 2 +- kernel/sched_debug.c | 2 +- kernel/sched_fair.c | 532 ++++++++++++++++++++++++----------- kernel/sched_rt.c | 3 - kernel/sched_stats.h | 27 +-- kernel/time/tick-sched.c | 10 +- kernel/timer.c | 8 +- kernel/trace/trace_clock.c | 2 +- kernel/workqueue_sched.h | 16 + 28 files changed, 877 insertions(+), 410 deletions(-) create mode 100644 kernel/workqueue_sched.h
arch/parisc/kernel/ftrace.c +2 -2
@@ -82,7 +82,7 @@ unsigned long ftrace_return_to_handler(unsigned long retval0,unsigned long ret;pop_return_trace (&trace, &ret); - trace.rettime =cpu_clock (raw_smp_processor_id ()); + trace.rettime =local_clock ();ftrace_graph_return (&trace);if (unlikely (!ret)) {
@@ -126,7 +126,7 @@ void prepare_ftrace_return(unsigned long *parent, unsigned long self_addr)return ; } - calltime =cpu_clock (raw_smp_processor_id ()); + calltime =local_clock ();if (push_return_trace (old, calltime, self_addr, &trace.depth) == -EBUSY ) {
arch/powerpc/include/asm/cputable.h +2 -1
@@ -197,6 +197,7 @@ extern const char *powerpc_base_platform;#define CPU_FTR_SAO LONG_ASM_CONST(0x0020000000000000)#define CPU_FTR_CP_USE_DCBTZ LONG_ASM_CONST(0x0040000000000000)#define CPU_FTR_UNALIGNED_LD_STD LONG_ASM_CONST(0x0080000000000000) +#define CPU_FTR_ASYM_SMT LONG_ASM_CONST(0x0100000000000000)#ifndef __ASSEMBLY__
@@ -412,7 +413,7 @@ extern const char *powerpc_base_platform; CPU_FTR_MMCRA | CPU_FTR_SMT | \ CPU_FTR_COHERENT_ICACHE | CPU_FTR_LOCKLESS_TLBIE | \ CPU_FTR_PURR | CPU_FTR_SPURR | CPU_FTR_REAL_LE | \ - CPU_FTR_DSCR | CPU_FTR_SAO) + CPU_FTR_DSCR | CPU_FTR_SAO | CPU_FTR_ASYM_SMT)#define CPU_FTRS_CELL (CPU_FTR_USE_TB | CPU_FTR_LWSYNC | \ CPU_FTR_PPCAS_ARCH_V2 | CPU_FTR_CTRL | \ CPU_FTR_ALTIVEC_COMP | CPU_FTR_MMCRA | CPU_FTR_SMT | \
arch/powerpc/kernel/process.c +11 -0
@@ -1263,3 +1263,14 @@ unsigned long randomize_et_dyn(unsigned long base)return ret; } + +#ifdef CONFIG_SMP +int arch_sd_sibling_asym_packing (void ) +{ +if (cpu_has_feature (CPU_FTR_ASYM_SMT )) { +printk_once (KERN_INFO "Enabling Asymmetric SMT scheduling\n" ); +return SD_ASYM_PACKING ; + } +return 0 ; +} +#endif
include/linux/cpu.h +25 -0
@@ -48,6 +48,31 @@ extern ssize_t arch_cpu_release(const char *, size_t);#endif struct notifier_block ; +/* +* CPU notifier priorities. +*/ +enum { +/* +* SCHED_ACTIVE marks a cpu which is coming up active during +* CPU_ONLINE and CPU_DOWN_FAILED and must be the first +* notifier. CPUSET_ACTIVE adjusts cpuset according to +* cpu_active mask right after SCHED_ACTIVE. During +* CPU_DOWN_PREPARE, SCHED_INACTIVE and CPUSET_INACTIVE are +* ordered in the similar way. +* +* This ordering guarantees consistent cpu_active mask and +* migration behavior to all cpu notifiers. +*/ +CPU_PRI_SCHED_ACTIVE =INT_MAX , +CPU_PRI_CPUSET_ACTIVE =INT_MAX -1 , +CPU_PRI_SCHED_INACTIVE =INT_MIN +1 , +CPU_PRI_CPUSET_INACTIVE =INT_MIN , + +/* migration should happen before other stuff but after perf */ +CPU_PRI_PERF =20 , +CPU_PRI_MIGRATION =10 , +}; +#ifdef CONFIG_SMP /* Need to know about CPUs going up/down? */ #if defined(CONFIG_HOTPLUG_CPU ) || !defined(MODULE )
include/linux/cpuset.h +6 -0
@@ -20,6 +20,7 @@ extern int number_of_cpusets; /* How many cpusets are defined in system? */extern int cpuset_init (void );extern void cpuset_init_smp (void ); +extern void cpuset_update_active_cpus (void );extern void cpuset_cpus_allowed (struct task_struct *p,struct cpumask *mask);extern int cpuset_cpus_allowed_fallback (struct task_struct *p);extern nodemask_t cpuset_mems_allowed (struct task_struct *p);
@@ -132,6 +133,11 @@ static inline void set_mems_allowed(nodemask_t nodemask)static inline int cpuset_init (void ) {return 0 ; }static inline void cpuset_init_smp (void ) {} +static inline void cpuset_update_active_cpus (void ) +{ +partition_sched_domains (1 ,NULL ,NULL ); +} +static inline void cpuset_cpus_allowed (struct task_struct *p,struct cpumask *mask) {
include/linux/perf_event.h +1 -1
@@ -1068,7 +1068,7 @@ static inline void perf_event_disable(struct perf_event *event) { }#define perf_cpu_notifier (fn) \ do { \ static struct notifier_block fn##_nb __cpuinitdata = \ - { .notifier_call = fn, .priority = 20 }; \ + { .notifier_call = fn, .priority = CPU_PRI_PERF }; \ fn(&fn##_nb, (unsigned long)CPU_UP_PREPARE, \ (void *)(unsigned long)smp_processor_id()); \ fn(&fn##_nb, (unsigned long)CPU_STARTING, \
include/linux/sched.h +29 -30
111 lines not on this page · raw message
include/linux/topology.h +1 -0
@@ -103,6 +103,7 @@ int arch_update_cpu_topology(void); | 1*SD_SHARE_PKG_RESOURCES \ |0 *SD_SERIALIZE \ |0 *SD_PREFER_SIBLING \ + |arch_sd_sibling_asym_packing () \ , \ .last_balance = jiffies, \ .balance_interval = 1, \
kernel/cpu.c +0 -6
@@ -235,11 +235,8 @@ static int __ref _cpu_down(unsigned int cpu, int tasks_frozen)return -EINVAL ;cpu_hotplug_begin (); -set_cpu_active (cpu, false); err =__cpu_notify (CPU_DOWN_PREPARE | mod, hcpu,-1 , &nr_calls);if (err) { -set_cpu_active (cpu, true); - nr_calls--;__cpu_notify (CPU_DOWN_FAILED | mod, hcpu, nr_calls,NULL ); printk("%s: attempt to take down CPU %u failed\n",
@@ -249,7 +246,6 @@ static int __ref _cpu_down(unsigned int cpu, int tasks_frozen) err =__stop_machine (take_cpu_down, &tcd_param,cpumask_of (cpu));if (err) { -set_cpu_active (cpu, true);/* CPU didn't die: tell everyone. Can't complain. */ cpu_notify_nofail (CPU_DOWN_FAILED | mod, hcpu);
@@ -321,8 +317,6 @@ static int __cpuinit _cpu_up(unsigned int cpu, int tasks_frozen) goto out_notify;BUG_ON (!cpu_online (cpu)); -set_cpu_active (cpu, true); -/* Now call notifier in preparation. */ cpu_notify (CPU_ONLINE | mod, hcpu);
kernel/cpuset.c +2 -19
@@ -2113,31 +2113,17 @@ static void scan_for_empty_cpusets(struct cpuset *root)* but making no active use of cpusets. * * This routine ensures that top_cpuset.cpus_allowed tracks -* cpu_online_map on each CPU hotplug (cpuhp) event. +* cpu_active_mask on each CPU hotplug (cpuhp) event. * * Called within get_online_cpus(). Needs to call cgroup_lock() * before calling generate_sched_domains(). */ -static int cpuset_track_online_cpus (struct notifier_block *unused_nb, -unsigned long phase,void *unused_cpu) +void cpuset_update_active_cpus (void ) {struct sched_domain_attr *attr;cpumask_var_t *doms;int ndoms; -switch (phase) { -case CPU_ONLINE : -case CPU_ONLINE_FROZEN : -case CPU_DOWN_PREPARE : -case CPU_DOWN_PREPARE_FROZEN : -case CPU_DOWN_FAILED : -case CPU_DOWN_FAILED_FROZEN : -break ; - -default : -return NOTIFY_DONE ; - } -cgroup_lock ();mutex_lock (&callback_mutex);cpumask_copy (top_cpuset.cpus_allowed, cpu_active_mask);
@@ -2148,8 +2134,6 @@ static int cpuset_track_online_cpus(struct notifier_block *unused_nb,/* Have scheduler rebuild the domains */ partition_sched_domains (ndoms, doms, attr); - -return NOTIFY_OK ; }#ifdef CONFIG_MEMORY_HOTPLUG
@@ -2203,7 +2187,6 @@ void __init cpuset_init_smp(void)cpumask_copy (top_cpuset.cpus_allowed, cpu_active_mask); top_cpuset.mems_allowed = node_states[N_HIGH_MEMORY ]; -hotcpu_notifier (cpuset_track_online_cpus,0 );hotplug_memory_notifier (cpuset_track_online_nodes,10 ); cpuset_wq =create_singlethread_workqueue ("cpuset" );
kernel/fork.c +1 -1
@@ -907,7 +907,7 @@ static void copy_flags(unsigned long clone_flags, struct task_struct *p) {unsigned long new_flags = p->flags; - new_flags &= ~PF_SUPERPRIV ; + new_flags &= ~(PF_SUPERPRIV |PF_WQ_WORKER ); new_flags |=PF_FORKNOEXEC ; new_flags |=PF_STARTING ; p->flags = new_flags;
kernel/hrtimer.c +2 -6
@@ -144,12 +144,8 @@ struct hrtimer_clock_base *lock_hrtimer_base(const struct hrtimer *timer,static int hrtimer_get_target (int this_cpu,int pinned) {#ifdef CONFIG_NO_HZ -if (!pinned &&get_sysctl_timer_migration () &&idle_cpu (this_cpu)) { -int preferred_cpu =get_nohz_load_balancer (); - -if (preferred_cpu >=0 ) -return preferred_cpu; - } +if (!pinned &&get_sysctl_timer_migration () &&idle_cpu (this_cpu)) +return get_nohz_timer_target ();#endif return this_cpu; }
kernel/lockdep.c +1 -1
@@ -146,7 +146,7 @@ static DEFINE_PER_CPU(struct lock_class_stats[MAX_LOCKDEP_KEYS],static inline u64 lockstat_clock (void ) { -return cpu_clock (smp_processor_id ()); +return local_clock (); }static int lock_point (unsigned long points[],unsigned long ip)
kernel/perf_event.c +1 -1
@@ -214,7 +214,7 @@ static void perf_unpin_context(struct perf_event_context *ctx)static inline u64 perf_clock (void ) { -return cpu_clock (raw_smp_processor_id ()); +return local_clock (); }/*
kernel/posix-cpu-timers.c +15 -21
@@ -232,31 +232,24 @@ static int cpu_clock_sample(const clockid_t which_clock, struct task_struct *p,void thread_group_cputime (struct task_struct *tsk,struct task_cputime *times) { -struct sighand_struct *sighand; -struct signal_struct *sig; +struct signal_struct *sig = tsk->signal;struct task_struct *t; - *times =INIT_CPUTIME ; + times->utime = sig->utime; + times->stime = sig->stime; + times->sum_exec_runtime = sig->sum_sched_runtime;rcu_read_lock (); - sighand =rcu_dereference (tsk->sighand); -if (!sighand) +/* make sure we can trust tsk->thread_group list */ +if (!likely (pid_alive (tsk))) goto out; - sig = tsk->signal; - t = tsk; do { times->utime = cputime_add(times->utime, t->utime); times->stime = cputime_add(times->stime, t->stime); times->sum_exec_runtime += t->se.sum_exec_runtime; - - t =next_thread (t); - }while (t != tsk); - - times->utime =cputime_add (times->utime, sig->utime); - times->stime =cputime_add (times->stime, sig->stime); - times->sum_exec_runtime += sig->sum_sched_runtime; + } while_each_thread(tsk, t); out: rcu_read_unlock(); }
@@ -1279,10 +1272,6 @@ static inline int fastpath_timer_check(struct task_struct *tsk) {struct signal_struct *sig; -/* tsk == current, ensure it is safe to use ->signal/sighand */ -if (unlikely (tsk->exit_state)) -return 0 ; -if (!task_cputime_zero (&tsk->cputime_expires)) {struct task_cputime task_sample = { .utime = tsk->utime,
@@ -1298,7 +1287,10 @@ static inline int fastpath_timer_check(struct task_struct *tsk)if (sig->cputimer.running) {struct task_cputime group_sample; -thread_group_cputimer (tsk, &group_sample); +spin_lock (&sig->cputimer.lock); + group_sample = sig->cputimer.cputime; +spin_unlock (&sig->cputimer.lock); +if (task_cputime_expired (&group_sample, &sig->cputime_expires))return 1 ; }
@@ -1315,6 +1307,7 @@ void run_posix_cpu_timers(struct task_struct *tsk) {LIST_HEAD (firing);struct k_itimer *timer, *next; +unsigned long flags;BUG_ON (!irqs_disabled ());
@@ -1325,7 +1318,8 @@ void run_posix_cpu_timers(struct task_struct *tsk)if (!fastpath_timer_check (tsk))return ; -spin_lock (&tsk->sighand->siglock); +if (!lock_task_sighand (tsk, &flags)) +return ;/* * Here we take off tsk->signal->cpu_timers[N] and * tsk->cpu_timers[N] all the timers that are firing, and
@@ -1347,7 +1341,7 @@ void run_posix_cpu_timers(struct task_struct *tsk)* that gets the timer lock before we do will give it up and * spin until we've taken care of that timer below. */ -spin_unlock (&tsk->sighand->siglock); +unlock_task_sighand (tsk, &flags);/* * Now that all the timers on our list have the firing flag,
kernel/rcutorture.c +1 -2
@@ -239,8 +239,7 @@ static unsigned long rcu_random(struct rcu_random_state *rrsp) {if (--rrsp->rrs_count <0 ) { - rrsp->rrs_state += - (unsigned long )cpu_clock (raw_smp_processor_id ()); + rrsp->rrs_state += (unsigned long )local_clock (); rrsp->rrs_count =RCU_RANDOM_REFRESH ; } rrsp->rrs_state = rrsp->rrs_state *RCU_RANDOM_MULT +RCU_RANDOM_ADD ;
kernel/sched.c +293 -98
609 lines not on this page · raw message
kernel/sched_clock.c +85 -10
137 lines not on this page · raw message
kernel/sched_cpupri.c +2 -6
@@ -166,14 +166,10 @@ void cpupri_set(struct cpupri *cp, int cpu, int newpri)* * Returns: -ENOMEM if memory fails. */ -int cpupri_init (struct cpupri *cp,bool bootmem) +int cpupri_init (struct cpupri *cp) { -gfp_t gfp =GFP_KERNEL ;int i; -if (bootmem) - gfp =GFP_NOWAIT ; -memset (cp,0 ,sizeof (*cp));for (i =0 ; i <CPUPRI_NR_PRIORITIES ; i++) {
@@ -181,7 +177,7 @@ int cpupri_init(struct cpupri *cp, bool bootmem)raw_spin_lock_init (&vec->lock); vec->count =0 ; -if (!zalloc_cpumask_var (&vec->mask, gfp)) +if (!zalloc_cpumask_var (&vec->mask,GFP_KERNEL )) goto cleanup; }
kernel/sched_cpupri.h +1 -1
@@ -27,7 +27,7 @@ struct cpupri {int cpupri_find (struct cpupri *cp,struct task_struct *p,struct cpumask *lowest_mask);void cpupri_set (struct cpupri *cp,int cpu,int pri); -int cpupri_init (struct cpupri *cp,bool bootmem); +int cpupri_init (struct cpupri *cp);void cpupri_cleanup (struct cpupri *cp);#else #define cpupri_set (cp, cpu, pri) do { } while (0)
kernel/sched_debug.c +1 -1
@@ -332,7 +332,7 @@ static int sched_debug_show(struct seq_file *m, void *v)PN (sysctl_sched_latency);PN (sysctl_sched_min_granularity);PN (sysctl_sched_wakeup_granularity); -PN (sysctl_sched_child_runs_first); +P (sysctl_sched_child_runs_first);P (sysctl_sched_features);#undef PN#undef P
kernel/sched_fair.c +371 -161
764 lines not on this page · raw message
kernel/sched_rt.c +0 -3
@@ -1663,9 +1663,6 @@ static void watchdog(struct rq *rq, struct task_struct *p) {unsigned long soft, hard; -if (!p->signal) -return ; -/* max may change after cur was read, this will be fixed next tick */ soft =task_rlimit (p,RLIMIT_RTTIME ); hard =task_rlimit_max (p,RLIMIT_RTTIME );
kernel/sched_stats.h +3 -24
@@ -295,13 +295,7 @@ sched_info_switch(struct task_struct *prev, struct task_struct *next)static inline void account_group_user_time (struct task_struct *tsk,cputime_t cputime) { -struct thread_group_cputimer *cputimer; - -/* tsk == current, ensure it is safe to use ->signal */ -if (unlikely (tsk->exit_state)) -return ; - - cputimer = &tsk->signal->cputimer; +struct thread_group_cputimer *cputimer = &tsk->signal->cputimer;if (!cputimer->running)return ;
@@ -325,13 +319,7 @@ static inline void account_group_user_time(struct task_struct *tsk,static inline void account_group_system_time (struct task_struct *tsk,cputime_t cputime) { -struct thread_group_cputimer *cputimer; - -/* tsk == current, ensure it is safe to use ->signal */ -if (unlikely (tsk->exit_state)) -return ; - - cputimer = &tsk->signal->cputimer; +struct thread_group_cputimer *cputimer = &tsk->signal->cputimer;if (!cputimer->running)return ;
@@ -355,16 +343,7 @@ static inline void account_group_system_time(struct task_struct *tsk,static inline void account_group_exec_runtime (struct task_struct *tsk,unsigned long long ns) { -struct thread_group_cputimer *cputimer; -struct signal_struct *sig; - - sig = tsk->signal; -/* see __exit_signal()->task_rq_unlock_wait() */ -barrier (); -if (unlikely (!sig)) -return ; - - cputimer = &sig->cputimer; +struct thread_group_cputimer *cputimer = &tsk->signal->cputimer;if (!cputimer->running)return ;
kernel/time/tick-sched.c +2 -8
@@ -325,7 +325,7 @@ void tick_nohz_stop_sched_tick(int inidle) }while (read_seqretry (&xtime_lock, seq));if (rcu_needs_cpu (cpu) ||printk_needs_cpu (cpu) || -arch_needs_cpu (cpu) ||nohz_ratelimit (cpu)) { +arch_needs_cpu (cpu)) { next_jiffies = last_jiffies +1 ; delta_jiffies =1 ; }else {
@@ -405,13 +405,7 @@ void tick_nohz_stop_sched_tick(int inidle)* the scheduler tick in nohz_restart_sched_tick. */ if (!ts->tick_stopped) { -if (select_nohz_load_balancer (1 )) { -/* -* sched tick not stopped! -*/ -cpumask_clear_cpu (cpu, nohz_cpu_mask); - goto out; - } +select_nohz_load_balancer (1 ); ts->idle_tick =hrtimer_get_expires (&ts->sched_timer); ts->tick_stopped =1 ;
kernel/timer.c +2 -6
@@ -679,12 +679,8 @@ __mod_timer(struct timer_list *timer, unsigned long expires, cpu =smp_processor_id ();#if defined(CONFIG_NO_HZ ) && defined(CONFIG_SMP ) -if (!pinned &&get_sysctl_timer_migration () &&idle_cpu (cpu)) { -int preferred_cpu =get_nohz_load_balancer (); - -if (preferred_cpu >=0 ) - cpu = preferred_cpu; - } +if (!pinned &&get_sysctl_timer_migration () &&idle_cpu (cpu)) + cpu =get_nohz_timer_target ();#endif new_base =per_cpu (tvec_bases, cpu);
kernel/trace/trace_clock.c +1 -1
@@ -56,7 +56,7 @@ u64 notrace trace_clock_local(void)*/ u64 notrace trace_clock(void) { -return cpu_clock (raw_smp_processor_id ()); +return local_clock (); }
kernel/workqueue_sched.h new file +16 -0
@@ -0,0 +1,16 @@ +/* +* kernel/workqueue_sched.h +* +* Scheduler hooks for concurrency managed workqueue. Only to be +* included from sched.c and workqueue.c. +*/ +static inline void wq_worker_waking_up (struct task_struct *task, +unsigned int cpu) +{ +} + +static inline struct task_struct *wq_worker_sleeping (struct task_struct *task, +unsigned int cpu) +{ +return NULL ; +}