scheduler changes for v3.7
Subsystems (10)
Ingo Molnar
maintainerLinus,
Please pull the latest sched-core-for-linus git tree from:
git://git.kernel.org/pub/scm/linux/kernel/git/tip/tip.git sched-core-for-linus
HEAD: fdf9c356502ae02238efcdf90cefd7b473a63fd4 cputime: Make finegrained irqtime accounting generally available
Continued quest to clean up and enhance the cputime code by Frederic Weisbecker, in preparation for future tickless kernel features. Other than that, smallish changes.
Thanks,
Ingo
------------------>
Alex Shi (3):
sched: recover SD_WAKE_AFFINE in select_task_rq_fair and code clean up
tile: Remove SD_PREFER_LOCAL leftover
sched/nohz: Clean up select_nohz_load_balancer()
Borislav Petkov (1):
sched: Fix __sched_period comment
Frederic Weisbecker (10):
cputime: Generalize CONFIG_VIRT_CPU_ACCOUNTING
sched: Move cputime code to its own file
cputime: Consolidate vtime handling on context switch
s390: Remove leftover account_tick_vtime() header
cputime: Use a proper subsystem naming for vtime related APIs
vtime: Consolidate system/idle context detection
ia64: Consolidate user vtime accounting
ia64: Reuse system and user vtime accounting functions on task switch
cputime: Gather time/stats accounting config options into a single menu
cputime: Make finegrained irqtime accounting generally available
Michael Wang (2):
sched: using dst_rq instead of this_rq during load balance
sched: Remove useless code in yield_to()
Namhyung Kim (3):
sched: Remove AFFINE_WAKEUPS feature flag sched/debug: Limit sd->*_idx range on sysctl
sched: Add time unit suffix to sched sysctl knobs
Pekka Enberg (1):
sched: Document schedule() entry points
Peter Zijlstra (2):
sched: Remove __ARCH_WANT_INTERRUPTS_ON_CTXSW
sched: Fix load avg vs. cpu-hotplug
Vincent Guittot (2):
sched: Fix nohz_idle_balance()
sched: cpu_power: enable ARCH_POWER
Documentation/scheduler/sched-arch.txt | 10 - arch/Kconfig | 9 + arch/ia64/Kconfig | 12 +- arch/ia64/include/asm/switch_to.h | 8 - arch/ia64/kernel/time.c | 66 ++-- arch/powerpc/include/asm/time.h | 6 - arch/powerpc/kernel/process.c | 3 - arch/powerpc/kernel/time.c | 55 ++- arch/powerpc/platforms/Kconfig.cputype | 16 +- arch/s390/Kconfig | 5 +- arch/s390/include/asm/cputime.h | 3 + arch/s390/include/asm/switch_to.h | 4 - arch/s390/kernel/vtime.c | 8 +- arch/tile/include/asm/topology.h | 1 - arch/x86/Kconfig | 12 +- include/linux/hardirq.h | 8 +- include/linux/kernel_stat.h | 8 + include/linux/kvm_host.h | 4 +- include/linux/sched.h | 10 +- include/linux/topology.h | 2 - init/Kconfig | 157 +++++--- kernel/fork.c | 4 - kernel/sched/Makefile | 2 +- kernel/sched/core.c | 677 +++------------------------------ kernel/sched/cputime.c | 530 ++++++++++++++++++++++++++ kernel/sched/fair.c | 81 ++-- kernel/sched/features.h | 10 +- kernel/sched/rt.c | 5 - kernel/sched/sched.h | 69 +++- kernel/softirq.c | 6 +- kernel/sysctl.c | 6 +- kernel/time/tick-sched.c | 3 +- 32 files changed, 894 insertions(+), 906 deletions(-) create mode 100644 kernel/sched/cputime.c
Documentation/scheduler/sched-arch.txt +0 -10
@@ -17,16 +17,6 @@ you must `#define __ARCH_WANT_UNLOCKED_CTXSW` in a header file Unlocked context switches introduce only a very minor performance penalty to the core scheduler implementation in the CONFIG_SMP case. -2. Interrupt status -By default, the switch_to arch function is called with interrupts -disabled. Interrupts may be enabled over the call if it is likely to -introduce a significant interrupt latency by adding the line -`#define __ARCH_WANT_INTERRUPTS_ON_CTXSW` in the same place as for -unlocked context switches. This define also implies -`__ARCH_WANT_UNLOCKED_CTXSW`. See arch/arm/include/asm/system.h for an -example. - - CPU idle ======== Your cpu_idle routines need to obey the following rules:
arch/Kconfig +9 -0
@@ -281,4 +281,13 @@ config SECCOMP_FILTER See Documentation/prctl/seccomp_filter.txt for details. +config HAVE_VIRT_CPU_ACCOUNTING + bool + +config HAVE_IRQ_TIME_ACCOUNTING + bool + help + Archs need to ensure they use a high enough resolution clock to + support irq time accounting and then call enable_sched_clock_irqtime(). + source "kernel/gcov/Kconfig"
arch/ia64/Kconfig +1 -11
@@ -25,6 +25,7 @@ config IA64 select HAVE_GENERIC_HARDIRQS select HAVE_MEMBLOCK select HAVE_MEMBLOCK_NODE_MAP + select HAVE_VIRT_CPU_ACCOUNTING select ARCH_DISCARD_MEMBLOCK select GENERIC_IRQ_PROBE select GENERIC_PENDING_IRQ if SMP
@@ -340,17 +341,6 @@ config FORCE_MAX_ZONEORDER default "17" if HUGETLB_PAGE default "11" -config VIRT_CPU_ACCOUNTING - bool "Deterministic task and CPU time accounting" - default n - help - Select this option to enable more accurate task and CPU time - accounting. This is done by reading a CPU counter on each - kernel entry and exit and on transitions within the kernel - between system, softirq and hardirq state, so there is a - small performance impact. - If in doubt, say N here. - config SMP bool "Symmetric multi-processing support" select USE_GENERIC_SMP_HELPERS
arch/ia64/include/asm/switch_to.h +0 -8
@@ -30,13 +30,6 @@ extern struct task_struct *ia64_switch_to (void *next_task);extern void ia64_save_extra (struct task_struct *task);extern void ia64_load_extra (struct task_struct *task); -#ifdef CONFIG_VIRT_CPU_ACCOUNTING -extern void ia64_account_on_switch (struct task_struct *prev,struct task_struct *next); -# define IA64_ACCOUNT_ON_SWITCH (p,n) ia64_account_on_switch(p,n) -#else -# define IA64_ACCOUNT_ON_SWITCH (p,n) -#endif -#ifdef CONFIG_PERFMON DECLARE_PER_CPU(unsigned long, pfm_syst_info);# define PERFMON_IS_SYSWIDE () (__get_cpu_var(pfm_syst_info) & 0x1)
@@ -49,7 +42,6 @@ extern void ia64_account_on_switch (struct task_struct *prev, struct task_struct || PERFMON_IS_SYSWIDE())#define __switch_to (prev,next,last) do { \ - IA64_ACCOUNT_ON_SWITCH(prev, next); \ if (IA64_HAS_EXTRA_STATE(prev)) \ ia64_save_extra(prev); \ if (IA64_HAS_EXTRA_STATE(next)) \
arch/ia64/kernel/time.c +33 -33
108 lines not on this page · raw message
arch/powerpc/include/asm/time.h +0 -6
@@ -197,12 +197,6 @@ struct cpu_usage { DECLARE_PER_CPU(struct cpu_usage, cpu_usage_array); -#if defined(CONFIG_VIRT_CPU_ACCOUNTING ) -#define account_process_vtime (tsk) account_process_tick(tsk, 0) -#else -#define account_process_vtime (tsk) do { } while (0) -#endif -extern void secondary_cpu_time_init (void );DECLARE_PER_CPU (u64, decrementers_next_tb);
arch/powerpc/kernel/process.c +0 -3
@@ -514,9 +514,6 @@ struct task_struct *__switch_to(struct task_struct *prev,local_irq_save (flags); -account_system_vtime (current); -account_process_vtime (current); -/* * We can't take a PMU exception inside _switch() since there is a * window where the kernel stack SLB and the kernel stack are out
arch/powerpc/kernel/time.c +35 -20
96 lines not on this page · raw message
arch/powerpc/platforms/Kconfig.cputype +1 -15
@@ -1,6 +1,7 @@ config PPC64 bool "64-bit kernel" default n + select HAVE_VIRT_CPU_ACCOUNTING help This option selects whether a 32-bit or a 64-bit kernel will be built.
@@ -337,21 +338,6 @@ config PPC_MM_SLICES default y if (!PPC_FSL_BOOK3E && PPC64 && HUGETLB_PAGE) || (PPC_STD_MMU_64 && PPC_64K_PAGES) default n -config VIRT_CPU_ACCOUNTING - bool "Deterministic task and CPU time accounting" - depends on PPC64 - default y - help - Select this option to enable more accurate task and CPU time - accounting. This is done by reading a CPU counter on each - kernel entry and exit and on transitions within the kernel - between system, softirq and hardirq state, so there is a - small performance impact. This also enables accounting of - stolen time on logically-partitioned systems running on - IBM POWER5-based machines. - - If in doubt, say Y here. - config PPC_HAVE_PMU_SUPPORT bool
arch/s390/Kconfig +2 -3
@@ -49,9 +49,6 @@ config GENERIC_LOCKBREAK config PGSTE def_bool y if KVM -config VIRT_CPU_ACCOUNTING - def_bool y - config ARCH_SUPPORTS_DEBUG_PAGEALLOC def_bool y
@@ -89,6 +86,8 @@ config S390 select HAVE_MEMBLOCK select HAVE_MEMBLOCK_NODE_MAP select HAVE_CMPXCHG_LOCAL + select HAVE_VIRT_CPU_ACCOUNTING + select VIRT_CPU_ACCOUNTING select ARCH_DISCARD_MEMBLOCK select BUILDTIME_EXTABLE_SORT select ARCH_INLINE_SPIN_TRYLOCK
arch/s390/include/asm/cputime.h +3 -0
@@ -12,6 +12,9 @@#include <linux/spinlock.h> #include <asm/div64.h> + +#define __ARCH_HAS_VTIME_ACCOUNT +/* We want to use full resolution of the CPU timer: 2**-12 micro-seconds. */ typedef unsigned long long __nocast cputime_t ;
arch/s390/include/asm/switch_to.h +0 -4
@@ -89,12 +89,8 @@ static inline void restore_access_regs(unsigned int *acrs) prev =__switch_to (prev,next); \ } while (0) -extern void account_vtime(struct task_struct *, struct task_struct *); -extern void account_tick_vtime (struct task_struct *); -#define finish_arch_switch (prev) do { \ set_fs(current->thread.mm_segment); \ - account_vtime(prev, current); \ } while (0)#endif /* __ASM_SWITCH_TO_H */
arch/s390/kernel/vtime.c +4 -4
@@ -99,7 +99,7 @@ static int do_account_vtime(struct task_struct *tsk, int hardirq_offset)return virt_timer_forward (user + system); } -void account_vtime (struct task_struct *prev,struct task_struct *next) +void vtime_task_switch (struct task_struct *prev) {struct thread_info *ti;
@@ -107,7 +107,7 @@ void account_vtime(struct task_struct *prev, struct task_struct *next) ti =task_thread_info (prev); ti->user_timer = S390_lowcore.user_timer; ti->system_timer = S390_lowcore.system_timer; - ti =task_thread_info (next); + ti =task_thread_info (current); S390_lowcore.user_timer = ti->user_timer; S390_lowcore.system_timer = ti->system_timer; }
@@ -122,7 +122,7 @@ void account_process_tick(struct task_struct *tsk, int user_tick)* Update process times based on virtual cpu times stored by entry.S * to the lowcore fields user_timer, system_timer & steal_clock. */ -void account_system_vtime (struct task_struct *tsk) +void vtime_account (struct task_struct *tsk) {struct thread_info *ti =task_thread_info (tsk);u64 timer, system;
@@ -138,7 +138,7 @@ void account_system_vtime(struct task_struct *tsk)virt_timer_forward (system); } -EXPORT_SYMBOL_GPL (account_system_vtime); +EXPORT_SYMBOL_GPL (vtime_account);void __kprobes vtime_stop_cpu (void ) {
arch/tile/include/asm/topology.h +0 -1
@@ -69,7 +69,6 @@ static inline const struct cpumask *cpumask_of_node(int node) | 1*SD_BALANCE_FORK \ |0 *SD_BALANCE_WAKE \ |0 *SD_WAKE_AFFINE \ - |0 *SD_PREFER_LOCAL \ |0 *SD_SHARE_CPUPOWER \ |0 *SD_SHARE_PKG_RESOURCES \ |0 *SD_SERIALIZE \
arch/x86/Kconfig +1 -11
@@ -97,6 +97,7 @@ config X86 select KTIME_SCALAR if X86_32 select GENERIC_STRNCPY_FROM_USER select GENERIC_STRNLEN_USER + select HAVE_IRQ_TIME_ACCOUNTING config INSTRUCTION_DECODER def_bool (KPROBES || PERF_EVENTS || UPROBES)
@@ -796,17 +797,6 @@ config SCHED_MC making when dealing with multi-core CPU chips at a cost of slightly increased overhead in some places. If unsure say N here. -config IRQ_TIME_ACCOUNTING - bool "Fine granularity task level IRQ time accounting" - default n - ---help--- - Select this option to enable fine granularity task irq time - accounting. This is done by reading a timestamp on each - transitions between softirq and hardirq state, so there can be a - small performance impact. - - If in doubt, say N here. - source "kernel/Kconfig.preempt" config X86_UP_APIC
include/linux/hardirq.h +4 -4
@@ -132,11 +132,11 @@ extern void synchronize_irq(unsigned int irq);struct task_struct ;#if !defined(CONFIG_VIRT_CPU_ACCOUNTING ) && !defined(CONFIG_IRQ_TIME_ACCOUNTING ) -static inline void account_system_vtime (struct task_struct *tsk) +static inline void vtime_account (struct task_struct *tsk) { }#else -extern void account_system_vtime (struct task_struct *tsk); +extern void vtime_account (struct task_struct *tsk);#endif #if defined(CONFIG_TINY_RCU ) || defined(CONFIG_TINY_PREEMPT_RCU )
@@ -162,7 +162,7 @@ extern void rcu_nmi_exit(void);*/ #define __irq_enter () \ do { \ - account_system_vtime(current); \ + vtime_account(current); \ add_preempt_count(HARDIRQ_OFFSET); \ trace_hardirq_enter(); \ } while (0)
@@ -178,7 +178,7 @@ extern void irq_enter(void);#define __irq_exit () \ do { \ trace_hardirq_exit(); \ - account_system_vtime(current); \ + vtime_account(current); \ sub_preempt_count(HARDIRQ_OFFSET); \ } while (0)
include/linux/kernel_stat.h +8 -0
@@ -130,4 +130,12 @@ extern void account_process_tick(struct task_struct *, int user);extern void account_steal_ticks (unsigned long ticks);extern void account_idle_ticks (unsigned long ticks); +#ifdef CONFIG_VIRT_CPU_ACCOUNTING +extern void vtime_task_switch (struct task_struct *prev); +extern void vtime_account_system (struct task_struct *tsk); +extern void vtime_account_idle (struct task_struct *tsk); +#else +static inline void vtime_task_switch (struct task_struct *prev) { } +#endif +#endif /* _LINUX_KERNEL_STAT_H */
include/linux/kvm_host.h +2 -2
@@ -685,7 +685,7 @@ static inline int kvm_deassign_device(struct kvm *kvm,static inline void kvm_guest_enter (void ) {BUG_ON (preemptible ()); -account_system_vtime (current); +vtime_account (current); current->flags |=PF_VCPU ;/* KVM does not hold any references to rcu protected data when it * switches CPU into a guest mode. In fact switching to a guest mode
@@ -699,7 +699,7 @@ static inline void kvm_guest_enter(void)static inline void kvm_guest_exit (void ) { -account_system_vtime (current); +vtime_account (current); current->flags &= ~PF_VCPU ; }
include/linux/sched.h +2 -8
@@ -273,11 +273,11 @@ extern void init_idle_bootup_task(struct task_struct *idle);extern int runqueue_is_locked (int cpu);#if defined(CONFIG_SMP ) && defined(CONFIG_NO_HZ ) -extern void select_nohz_load_balancer (int stop_tick); +extern void nohz_balance_enter_idle (int cpu);extern void set_cpu_sd_state_idle (void );extern int get_nohz_timer_target (void );#else -static inline void select_nohz_load_balancer (int stop_tick) { } +static inline void nohz_balance_enter_idle (int cpu) { }static inline void set_cpu_sd_state_idle (void ) { }#endif
@@ -678,11 +678,6 @@ struct signal_struct {* (notably. ptrace) */ }; -/* Context switch must be unlocked if interrupts are to be enabled */ -#ifdef __ARCH_WANT_INTERRUPTS_ON_CTXSW -# define __ARCH_WANT_UNLOCKED_CTXSW -#endif -/* * Bits in flags field of signal_struct. */
@@ -860,7 +855,6 @@ enum cpu_idle_type {#define SD_BALANCE_FORK 0x0008/* Balance on fork, clone */ #define SD_BALANCE_WAKE 0x0010/* Balance on wakeup */ #define SD_WAKE_AFFINE 0x0020/* Wake task to waking CPU */ -#define SD_PREFER_LOCAL 0x0040/* Prefer to keep tasks local to this domain */ #define SD_SHARE_CPUPOWER 0x0080/* Domain members share cpu power */ #define SD_SHARE_PKG_RESOURCES 0x0200/* Domain members share cpu pkg resources */ #define SD_SERIALIZE 0x0400/* Only a single load balancing instance */
include/linux/topology.h +0 -2
@@ -129,7 +129,6 @@ int arch_update_cpu_topology(void); | 1*SD_BALANCE_FORK \ |0 *SD_BALANCE_WAKE \ |1 *SD_WAKE_AFFINE \ - |0 *SD_PREFER_LOCAL \ |0 *SD_SHARE_CPUPOWER \ |1 *SD_SHARE_PKG_RESOURCES \ |0 *SD_SERIALIZE \
@@ -160,7 +159,6 @@ int arch_update_cpu_topology(void); | 1*SD_BALANCE_FORK \ |0 *SD_BALANCE_WAKE \ |1 *SD_WAKE_AFFINE \ - |0 *SD_PREFER_LOCAL \ |0 *SD_SHARE_CPUPOWER \ |0 *SD_SHARE_PKG_RESOURCES \ |0 *SD_SERIALIZE \
init/Kconfig +101 -56
175 lines not on this page · raw message
kernel/fork.c +0 -4
@@ -1280,11 +1280,7 @@ static struct task_struct *copy_process(unsigned long clone_flags,#endif #ifdef CONFIG_TRACE_IRQFLAGS p->irq_events =0 ; -#ifdef __ARCH_WANT_INTERRUPTS_ON_CTXSW - p->hardirqs_enabled =1 ; -#else p->hardirqs_enabled =0 ; -#endif p->hardirq_enable_ip =0 ; p->hardirq_enable_event =0 ; p->hardirq_disable_ip = _THIS_IP_;
kernel/sched/Makefile +1 -1
@@ -11,7 +11,7 @@ ifneq ($(CONFIG_SCHED_OMIT_FRAME_POINTER),y) CFLAGS_core.o := $(PROFILING) -fno-omit-frame-pointer endif -obj-y += core.o clock.o idle_task.o fair.o rt.o stop_task.o +obj-y += core.o clock.o cputime.o idle_task.o fair.o rt.o stop_task.o obj-$(CONFIG_SMP) += cpupri.o obj-$(CONFIG_SCHED_AUTOGROUP) += auto_group.o obj-$(CONFIG_SCHEDSTATS) += stats.o
kernel/sched/core.c +62 -615
780 lines not on this page · raw message
kernel/sched/cputime.c new file +530 -0
530 lines not on this page · raw message
kernel/sched/fair.c +24 -57
178 lines not on this page · raw message
kernel/sched/features.h +1 -9
@@ -12,14 +12,6 @@ SCHED_FEAT(GENTLE_FAIR_SLEEPERS, true)SCHED_FEAT (START_DEBIT , true)/* -* Based on load and program behaviour, see if it makes sense to place -* a newly woken task on the same cpu as the task that woke it -- -* improve cache locality. Typically used with SYNC wakeups as -* generated by pipes and the like, see also SYNC_WAKEUPS. - */ -SCHED_FEAT (AFFINE_WAKEUPS , true) - -/* * Prefer to schedule the task we woke last (assuming it failed * wakeup-preemption), since its likely going to consume data we * touched, increases cache locality.
@@ -42,7 +34,7 @@ SCHED_FEAT(CACHE_HOT_BUDDY, true)/* * Use arch dependent cpu power functions */ -SCHED_FEAT(ARCH_POWER, false) +SCHED_FEAT(ARCH_POWER, true) SCHED_FEAT(HRTICK, false)SCHED_FEAT (DOUBLE_TICK , false)
kernel/sched/rt.c +0 -5
@@ -1632,11 +1632,6 @@ static int push_rt_task(struct rq *rq)if (!next_task)return 0 ; -#ifdef __ARCH_WANT_INTERRUPTS_ON_CTXSW -if (unlikely (task_running (rq, next_task))) -return 0 ; -#endif - retry:if (unlikely (next_task == rq->curr)) {WARN_ON (1 );
kernel/sched/sched.h +63 -6
98 lines not on this page · raw message
kernel/softirq.c +3 -3
@@ -220,7 +220,7 @@ asmlinkage void __do_softirq(void) current->flags &= ~PF_MEMALLOC ; pending =local_softirq_pending (); -account_system_vtime (current); +vtime_account (current);__local_bh_disable ((unsigned long )__builtin_return_address (0 ),SOFTIRQ_OFFSET );
@@ -271,7 +271,7 @@ restart:lockdep_softirq_exit (); -account_system_vtime (current); +vtime_account (current);__local_bh_enable (SOFTIRQ_OFFSET );tsk_restore_flags (current, old_flags,PF_MEMALLOC ); }
@@ -340,7 +340,7 @@ static inline void invoke_softirq(void)*/ void irq_exit (void ) { -account_system_vtime (current); +vtime_account (current);trace_hardirq_exit ();sub_preempt_count (IRQ_EXIT_OFFSET );if (!in_interrupt () &&local_softirq_pending ())
kernel/sysctl.c +3 -3
@@ -307,7 +307,7 @@ static struct ctl_table kern_table[] = { .extra2 = &max_sched_tunable_scaling, }, { - .procname = "sched_migration_cost", + .procname = "sched_migration_cost_ns", .data = &sysctl_sched_migration_cost, .maxlen = sizeof(unsigned int), .mode = 0644,
@@ -321,14 +321,14 @@ static struct ctl_table kern_table[] = { .proc_handler = proc_dointvec, }, { - .procname = "sched_time_avg", + .procname = "sched_time_avg_ms", .data = &sysctl_sched_time_avg, .maxlen = sizeof(unsigned int), .mode = 0644, .proc_handler = proc_dointvec, }, { - .procname = "sched_shares_window", + .procname = "sched_shares_window_ns", .data = &sysctl_sched_shares_window, .maxlen = sizeof(unsigned int), .mode = 0644,
kernel/time/tick-sched.c +1 -2
@@ -372,7 +372,7 @@ static ktime_t tick_nohz_stop_sched_tick(struct tick_sched *ts,* the scheduler tick in nohz_restart_sched_tick. */ if (!ts->tick_stopped) { -select_nohz_load_balancer (1 ); +nohz_balance_enter_idle (cpu);calc_load_enter_idle (); ts->last_tick =hrtimer_get_expires (&ts->sched_timer);
@@ -569,7 +569,6 @@ static void tick_nohz_restart(struct tick_sched *ts, ktime_t now)static void tick_nohz_restart_sched_tick (struct tick_sched *ts,ktime_t now) {/* Update jiffies first */ -select_nohz_load_balancer (0 );tick_do_update_jiffies64 (now);update_cpu_load_nohz ();