scheduler fixes
Linus, please pull the latest scheduler git tree from:
git://git.kernel.org/pub/scm/linux/kernel/git/mingo/linux-2.6-sched.git
There are two cross-subsystem groups of fixes: three commits that resolve a KVM build fix on !SMP - acked by Avi to go in via the scheduler git tree because it changes a central include file. The other one is a powerpc CPU time accounting regression fix from Paul Mackerras.
The remaining 14 commits: one crash fix (only triggerable via the new control-groups filesystem), a delay-accounting regression fix, two performance regression fixes, a latency fix, two small scheduling-behavior regression fixes and seven cleanups.
Build and boot tested on 64-bit and 32-bit x86.
Ingo
------------------>
Adrian Bunk (1):
sched: proper prototype for kernel/sched.c:migration_init()
Balbir Singh (1):
sched: fix delay accounting regression
Eric Dumazet (1):
sched: cleanup, use NSEC_PER_MSEC and NSEC_PER_SEC
Ingo Molnar (9):
sched: reintroduce SMP tunings again
x86: make ipi_handler() always defined
x86: make nmi_cpu_busy() always defined
KVM: fix !SMP build error
sched: turn off PREEMPT_RESTRICT
sched: remove PREEMPT_RESTRICT
sched: wakeup preemption fix
sched: clean up the wakeup preempt check
sched: clean up the wakeup preempt check, #2
Paul Mackerras (1):
sched: restore deterministic CPU accounting on powerpc
Peter Zijlstra (4):
sched: fix vslice
sched: documentation: place_entity() comments
sched: reintroduce the sched_min_granularity tunable
sched: avoid large irq-latencies in smp-balancing
Srivatsa Vaddagiri (1):
sched: fix copy_namespace() <-> sched_fork() dependency in do_fork
arch/powerpc/kernel/process.c | 2 arch/powerpc/kernel/time.c | 25 ---------- arch/s390/kernel/time.c | 4 - arch/s390/kernel/vtime.c | 8 --- arch/x86/kernel/cpu/mtrr/main.c | 6 -- arch/x86/kernel/nmi_32.c | 4 - include/linux/sched.h | 17 ++++++- include/linux/smp.h | 7 ++ init/main.c | 4 - kernel/fork.c | 6 +- kernel/sched.c | 59 ++++++++++++++++++------ kernel/sched_debug.c | 2 kernel/sched_fair.c | 96 ++++++++++++++++++++++++---------------- kernel/sched_stats.h | 11 ++-- kernel/sysctl.c | 23 +++++++-- kernel/timer.c | 21 +++++--- 16 files changed, 172 insertions(+), 123 deletions(-)
arch/powerpc/kernel/process.c +1 -1
@@ -350,7 +350,7 @@ struct task_struct *__switch_to(struct tlocal_irq_save (flags);account_system_vtime (current); -account_process_vtime (current); +account_process_tick (current,0 );calculate_steal_time (); last =_switch (old_thread, new_thread);
arch/powerpc/kernel/time.c +1 -24
@@ -259,7 +259,7 @@ void account_system_vtime(struct task_st* user and system time records. * Must be called with interrupts disabled. */ -void account_process_vtime (struct task_struct *tsk) +void account_process_tick (struct task_struct *tsk,int user_tick) {cputime_t utime, utimescaled;
@@ -274,18 +274,6 @@ void account_process_vtime(struct task_saccount_user_time_scaled (tsk, utimescaled); } -static void account_process_time (struct pt_regs *regs) -{ -int cpu =smp_processor_id (); - -account_process_vtime (current); -run_local_timers (); -if (rcu_pending (cpu)) -rcu_check_callbacks (cpu,user_mode (regs)); -scheduler_tick (); -run_posix_cpu_timers (current); -} -/* * Stuff for accounting stolen time. */
@@ -375,7 +363,6 @@ static void snapshot_purr(void)#else /* ! CONFIG_VIRT_CPU_ACCOUNTING */ #define calc_cputime_factors () -#define account_process_time (regs) update_process_times(user_mode(regs))#define calculate_steal_time () do { } while (0)#endif
@@ -599,16 +586,6 @@ void timer_interrupt(struct pt_regs * reget_lppaca ()->int_dword.fields.decr_int =0 ;#endif -/* -* We cannot disable the decrementer, so in the period -* between this cpu's being marked offline in cpu_online_map -* and calling stop-self, it is taking timer interrupts. -* Avoid calling into the scheduler rebalancing code if this -* is the case. -*/ -if (!cpu_is_offline (cpu)) -account_process_time (regs); -if (evt->event_handler) evt->event_handler (evt);else
arch/s390/kernel/time.c +0 -4
@@ -145,12 +145,8 @@ void account_ticks(u64 time)do_timer (ticks);#endif -#ifdef CONFIG_VIRT_CPU_ACCOUNTING -account_tick_vtime (current); -#else while (ticks--)update_process_times (user_mode (get_irq_regs ())); -#endif s390_do_profile (); }
arch/s390/kernel/vtime.c +1 -7
@@ -32,7 +32,7 @@ static DEFINE_PER_CPU(struct vtimer_queu* Update process times based on virtual cpu times stored by entry.S * to the lowcore fields user_timer, system_timer & steal_clock. */ -void account_tick_vtime (struct task_struct *tsk) +void account_process_tick (struct task_struct *tsk,int user_tick) {cputime_t cputime;__u64 timer, clock;
@@ -64,12 +64,6 @@ void account_tick_vtime(struct task_stru S390_lowcore.steal_clock -= cputime <<12 ;account_steal_time (tsk, cputime); } - -run_local_timers (); -if (rcu_pending (smp_processor_id ())) -rcu_check_callbacks (smp_processor_id (), rcu_user_flag); -scheduler_tick (); -run_posix_cpu_timers (tsk); }/*
arch/x86/kernel/cpu/mtrr/main.c +2 -4
@@ -139,13 +139,12 @@ struct set_mtrr_data {mtrr_type smp_type; }; -#ifdef CONFIG_SMP -static void ipi_handler (void *info)/* [SUMMARY] Synchronisation handler. Executed by "other" CPUs. [RETURNS] Nothing. */ { +#ifdef CONFIG_SMP struct set_mtrr_data *data = info;unsigned long flags;
@@ -168,9 +167,8 @@ static void ipi_handler(void *info)atomic_dec (&data->count);local_irq_restore (flags); -} -#endif +}static inline int types_compatible (mtrr_type type1,mtrr_type type2) { return type1 == MTRR_TYPE_UNCACHABLE ||
arch/x86/kernel/nmi_32.c +2 -2
@@ -51,13 +51,13 @@ static int unknown_nmi_panic_callback(ststatic int endflag__initdata =0 ; -#ifdef CONFIG_SMP /* The performance counters used by NMI_LOCAL_APIC don't trigger when * the CPU is idle. To make sure the NMI watchdog really ticks on all * CPUs during the test make them busy. */ static __init void nmi_cpu_busy (void *data) { +#ifdef CONFIG_SMP local_irq_enable_in_hardirq ();/* Intentionally don't use cpu_relax here. This is to make sure that the performance counter really ticks,
@@ -67,8 +67,8 @@ static __init void nmi_cpu_busy(void *dacare if they get somewhat less cycles. */ while (endflag ==0 )mb (); -}#endif +}static int __init check_nmi_watchdog (void ) {
include/linux/sched.h +15 -2
@@ -254,6 +254,7 @@ long io_schedule_timeout(long timeout);extern void cpu_init (void );extern void trap_init (void ); +extern void account_process_tick (struct task_struct *task,int user);extern void update_process_times (int user);extern void scheduler_tick (void );
@@ -862,7 +863,6 @@ struct sched_entity {struct load_weight load;/* for load-balancing */ struct rb_node run_node;unsigned int on_rq; -int peer_preempt;u64 exec_start;u64 sum_exec_runtime;
@@ -1460,12 +1460,17 @@ extern void sched_idle_next(void);#ifdef CONFIG_SCHED_DEBUG extern unsigned int sysctl_sched_latency; -extern unsigned int sysctl_sched_nr_latency; +extern unsigned int sysctl_sched_min_granularity;extern unsigned int sysctl_sched_wakeup_granularity;extern unsigned int sysctl_sched_batch_wakeup_granularity;extern unsigned int sysctl_sched_child_runs_first;extern unsigned int sysctl_sched_features;extern unsigned int sysctl_sched_migration_cost; +extern unsigned int sysctl_sched_nr_migrate; + +int sched_nr_latency_handler (struct ctl_table *table,int write, +struct file *file,void __user *buffer,size_t *length, +loff_t *ppos);#endif extern unsigned int sysctl_sched_compat_yield;
@@ -1983,6 +1988,14 @@ static inline void inc_syscw(struct task }#endif +#ifdef CONFIG_SMP +void migration_init (void ); +#else +static inline void migration_init (void ) +{ +} +#endif +#endif /* __KERNEL__ */ #endif
include/linux/smp.h +5 -2
@@ -84,11 +84,12 @@ void smp_prepare_boot_cpu(void);* These macros fold the SMP functionality into a single CPU system */ #define raw_smp_processor_id () 0 -static inline int up_smp_call_function (void ) +static inline int up_smp_call_function (void (*func)(void *),void *info) {return 0 ; } -#define smp_call_function (func,info,retry,wait) (up_smp_call_function()) +#define smp_call_function (func, info, retry, wait) \ + (up_smp_call_function(func, info))#define on_each_cpu (func,info,retry,wait) \ ({ \ local_irq_disable(); \
@@ -107,6 +108,8 @@ static inline void smp_send_reschedule(ilocal_irq_enable (); \0 ; \ }) +#define smp_call_function_mask (mask, func, info, wait) \ + (up_smp_call_function(func, info))#endif /* !SMP */
init/main.c +1 -3
@@ -56,6 +56,7 @@#include <linux/pid_namespace.h> #include <linux/device.h> #include <linux/kthread.h> +#include <linux/sched.h> #include <asm/io.h> #include <asm/bugs.h>
@@ -747,11 +748,8 @@ __setup("nosoftlockup", nosoftlockup_setstatic void __init do_pre_smp_initcalls (void ) {extern int spawn_ksoftirqd (void ); -#ifdef CONFIG_SMP -extern int migration_init (void );migration_init (); -#endif spawn_ksoftirqd ();if (!nosoftlockup)spawn_softlockup_task ();
kernel/fork.c +3 -3
@@ -1123,6 +1123,9 @@ static struct task_struct *copy_process( p->blocked_on =NULL ;/* not blocked yet */ #endif +/* Perform scheduler related setup. Assign this task to a CPU. */ +sched_fork (p, clone_flags); +if ((retval =security_task_alloc (p))) goto bad_fork_cleanup_policy;if ((retval =audit_alloc (p)))
@@ -1212,9 +1215,6 @@ static struct task_struct *copy_process(INIT_LIST_HEAD (&p->ptrace_children);INIT_LIST_HEAD (&p->ptrace_list); -/* Perform scheduler related setup. Assign this task to a CPU. */ -sched_fork (p, clone_flags); -/* Now that the task is set up, run cgroup callbacks if * necessary. We need to run them before the task is visible * on the tasklist. */
kernel/sched.c +44 -15
141 lines not on this page · raw message
kernel/sched_debug.c +1 -1
@@ -211,7 +211,7 @@ static int sched_debug_show(struct seq_f#define PN (x) \ SEQ_printf(m, " .%-40s: %Ld.%06ld\n", #x, SPLIT_NS(x))PN (sysctl_sched_latency); -PN (sysctl_sched_nr_latency); +PN (sysctl_sched_min_granularity);PN (sysctl_sched_wakeup_granularity);PN (sysctl_sched_batch_wakeup_granularity);PN (sysctl_sched_child_runs_first);
kernel/sched_fair.c +59 -37
209 lines not on this page · raw message
kernel/sched_stats.h +6 -5
@@ -127,7 +127,7 @@ rq_sched_info_depart(struct rq *rq, unsi# define schedstat_set (var, val) do { } while (0)#endif -#ifdef CONFIG_SCHEDSTATS +#if defined(CONFIG_SCHEDSTATS ) || defined(CONFIG_TASK_DELAY_ACCT )/* * Called when a process is dequeued from the active array and given * the cpu. We should note that with the exception of interactive
@@ -155,7 +155,7 @@ static inline void sched_info_dequeued(s*/ static void sched_info_arrive (struct task_struct *t) { -unsigned long long now =sched_clock (), delta =0 ; +unsigned long long now =task_rq (t)->clock, delta =0 ;if (t->sched_info.last_queued) delta = now - t->sched_info.last_queued;
@@ -186,7 +186,7 @@ static inline void sched_info_queued(str {if (unlikely (sched_info_on ()))if (!t->sched_info.last_queued) - t->sched_info.last_queued =sched_clock (); + t->sched_info.last_queued =task_rq (t)->clock; }/*
@@ -195,7 +195,8 @@ static inline void sched_info_queued(str*/ static inline void sched_info_depart (struct task_struct *t) { -unsigned long long delta =sched_clock () - t->sched_info.last_arrival; +unsigned long long delta =task_rq (t)->clock - + t->sched_info.last_arrival; t->sched_info.cpu_time += delta;rq_sched_info_depart (task_rq (t), delta);
@@ -231,5 +232,5 @@ sched_info_switch(struct task_struct *pr#else #define sched_info_queued (t) do { } while (0)#define sched_info_switch (t, next) do { } while (0) -#endif /* CONFIG_SCHEDSTATS */ +#endif /* CONFIG_SCHEDSTATS || CONFIG_TASK_DELAY_ACCT */
kernel/sysctl.c +17 -6
@@ -226,20 +226,23 @@ static struct ctl_table root_table[] = {#ifdef CONFIG_SCHED_DEBUG static unsigned long min_sched_granularity_ns =100000 ;/* 100 usecs */ -static unsigned long max_sched_granularity_ns =1000000000 ;/* 1 second */ +static unsigned long max_sched_granularity_ns =NSEC_PER_SEC ;/* 1 second */ static unsigned long min_wakeup_granularity_ns;/* 0 usecs */ -static unsigned long max_wakeup_granularity_ns =1000000000 ;/* 1 second */ +static unsigned long max_wakeup_granularity_ns =NSEC_PER_SEC ;/* 1 second */ #endif static struct ctl_table kern_table[] = { #ifdef CONFIG_SCHED_DEBUG { .ctl_name = CTL_UNNUMBERED, - .procname = "sched_nr_latency", - .data = &sysctl_sched_nr_latency, + .procname = "sched_min_granularity_ns", + .data = &sysctl_sched_min_granularity, .maxlen = sizeof(unsigned int), .mode = 0644, - .proc_handler = &proc_dointvec, + .proc_handler = &sched_nr_latency_handler, + .strategy = &sysctl_intvec, + .extra1 = &min_sched_granularity_ns, + .extra2 = &max_sched_granularity_ns, }, { .ctl_name = CTL_UNNUMBERED,
@@ -247,7 +250,7 @@ static struct ctl_table kern_table[] = { .data = &sysctl_sched_latency, .maxlen = sizeof(unsigned int), .mode = 0644, - .proc_handler = &proc_dointvec_minmax, + .proc_handler = &sched_nr_latency_handler, .strategy = &sysctl_intvec, .extra1 = &min_sched_granularity_ns, .extra2 = &max_sched_granularity_ns,
@@ -298,6 +301,14 @@ static struct ctl_table kern_table[] = { .mode = 0644, .proc_handler = &proc_dointvec, }, + { + .ctl_name = CTL_UNNUMBERED, + .procname = "sched_nr_migrate", + .data = &sysctl_sched_nr_migrate, + .maxlen = sizeof(unsigned int), + .mode = 644, + .proc_handler = &proc_dointvec, + },#endif { .ctl_name = CTL_UNNUMBERED,
kernel/timer.c +14 -7
@@ -817,6 +817,19 @@ unsigned long next_timer_interrupt(void)#endif +#ifndef CONFIG_VIRT_CPU_ACCOUNTING +void account_process_tick (struct task_struct *p,int user_tick) +{ +if (user_tick) { +account_user_time (p,jiffies_to_cputime (1 )); +account_user_time_scaled (p,jiffies_to_cputime (1 )); + }else { +account_system_time (p,HARDIRQ_OFFSET ,jiffies_to_cputime (1 )); +account_system_time_scaled (p,jiffies_to_cputime (1 )); + } +} +#endif +/* * Called from the timer interrupt handler to charge one tick to the current * process. user_tick is 1 if the tick is user time, 0 for system.
@@ -827,13 +840,7 @@ void update_process_times(int user_tick)int cpu =smp_processor_id ();/* Note: this timer irq context must be accounted for as well. */ -if (user_tick) { -account_user_time (p,jiffies_to_cputime (1 )); -account_user_time_scaled (p,jiffies_to_cputime (1 )); - }else { -account_system_time (p,HARDIRQ_OFFSET ,jiffies_to_cputime (1 )); -account_system_time_scaled (p,jiffies_to_cputime (1 )); - } +account_process_tick (p, user_tick);run_local_timers ();if (rcu_pending (cpu))rcu_check_callbacks (cpu, user_tick);