sched/numa: Implement numa balancer
Subsystems (4)
Commit-ID: b4cdf91668c27a5a6a5a3ed4234756c042dd8288 Gitweb: http://git.kernel.org/tip/b4cdf91668c27a5a6a5a3ed4234756c042dd8288 Author: Peter Zijlstra <[email protected]> AuthorDate: Sat, 3 Mar 2012 17:01:06 +0100 Committer: Ingo Molnar <[email protected]> CommitDate: Sat, 19 May 2012 12:52:20 +0200
sched/numa: Implement numa balancer
Implement a NUMA process balancer that migrates processes across nodes (it changes their home-node). This implies full memory migration.
Adds node wide cpu load tracking in two measures, tasks that should have ran on this node and tasks that run away from their node (the former includes the latter). We use the latter measure as indication that the node is overloaded and use the former to compute cpu imbalance.
Adds node wide memory load tracking in two measures, page rate of page allocations that miss the preferred node (NUMA_FOREIGN) and an absolute measure of pages used on the node (MR_ANON_PAGES + NR_ACTIVE_FILE). We use the first as indication that the node is overloaded with memory and use the second to compute imbalance.
For process mem load measure we use RSS (MM_ANONPAGES), this is comparable to our absolute memory load (both are in pages). For process cpu load we use the sum of load over the process thread group.
Using all this information we build two main functions:
- select_task_node(); this is ran on fork and exec and finds a suitable node for the 'new' process. This is a typical least loaded node scan, controlled through the NUMA_SELECT feature flag.
- numa_balance(); active pull-based node load-balancer, tries to balance node cpu usage against node mem usage, controlled through the NUMA_BALANCE feature flag.
Signed-off-by Peter Zijlstra <[email protected]>
Cc Suresh Siddha <[email protected]>
Cc Paul Turner <[email protected]>
Cc Dan Smith <[email protected]>
Cc Bharata B Rao <[email protected]>
Cc Lee Schermerhorn <[email protected]>
Cc Christoph Lameter <[email protected]>
Cc Rik van Riel <[email protected]>
Cc Andrea Arcangeli <[email protected]>
Cc Andrew Morton <[email protected]>
Cc Linus Torvalds <[email protected]>
Signed-off-by Ingo Molnar <[email protected]>
--- include/linux/mm_types.h | 8 + include/linux/sched.h | 13 + init/Kconfig | 2 +- kernel/fork.c | 2 + kernel/sched/Makefile | 1 + kernel/sched/core.c | 1 + kernel/sched/fair.c | 6 +- kernel/sched/features.h | 4 + kernel/sched/numa.c | 741 ++++++++++++++++++++++++++++++++++++++++++++++ kernel/sched/sched.h | 16 + mm/init-mm.c | 10 + 11 files changed, 799 insertions(+), 5 deletions(-)
include/linux/mm_types.h +8 -0
@@ -285,6 +285,13 @@ struct mm_rss_stat {atomic_long_t count[NR_MM_COUNTERS ]; }; +struct numa_entity { +#ifdef CONFIG_NUMA +int node;/* home node */ +struct list_head numa_entry;/* balance list */ +#endif +}; +struct mm_struct {struct vm_area_struct * mmap;/* list of VMAs */ struct rb_root mm_rb;
@@ -388,6 +395,7 @@ struct mm_struct {#ifdef CONFIG_CPUMASK_OFFSTACK struct cpumask cpumask_allocation;#endif +struct numa_entity numa; };static inline void mm_init_cpumask (struct mm_struct *mm)
include/linux/sched.h +13 -0
@@ -1234,6 +1234,11 @@ struct task_struct {struct sched_entity se;struct sched_rt_entity rt; +#ifdef CONFIG_NUMA +unsigned long numa_contrib; +int numa_remote; +#endif +#ifdef CONFIG_PREEMPT_NOTIFIERS /* list of struct preempt_notifier: */ struct hlist_head preempt_notifiers;
@@ -2782,6 +2787,14 @@ static inline unsigned long rlimit_max(unsigned int limit)return task_rlimit_max (current, limit); } +#ifdef CONFIG_NUMA +void mm_init_numa (struct mm_struct *mm); +void exit_numa (struct mm_struct *mm); +#else +static inline void mm_init_numa (struct mm_struct *mm) { } +static inline void exit_numa (struct mm_struct *mm) { } +#endif +#endif /* __KERNEL__ */ #endif
init/Kconfig +1 -1
@@ -866,7 +866,7 @@ config SCHED_AUTOGROUP upon task session. config MM_OWNER - bool + def_bool NUMA config SYSFS_DEPRECATED bool "Enable deprecated sysfs features to support old userspace tools"
kernel/fork.c +2 -0
@@ -527,6 +527,7 @@ static struct mm_struct *mm_init(struct mm_struct *mm, struct task_struct *p) mm->cached_hole_size = ~0UL ;mm_init_aio (mm);mm_init_owner (mm, p); +mm_init_numa (mm);if (likely (!mm_alloc_pgd (mm))) { mm->def_flags =0 ;
@@ -595,6 +596,7 @@ void mmput(struct mm_struct *mm)might_sleep ();if (atomic_dec_and_test (&mm->mm_users)) { +exit_numa (mm);exit_aio (mm);ksm_exit (mm);khugepaged_exit (mm);/* must run before exit_mmap */
kernel/sched/Makefile +1 -0
@@ -16,3 +16,4 @@ obj-$(CONFIG_SMP) += cpupri.o obj-$(CONFIG_SCHED_AUTOGROUP) += auto_group.o obj-$(CONFIG_SCHEDSTATS) += stats.o obj-$(CONFIG_SCHED_DEBUG) += debug.o +obj-$(CONFIG_NUMA) += numa.o
kernel/sched/core.c +1 -0
@@ -7132,6 +7132,7 @@ void __init sched_init(void)idle_thread_set_boot_cpu ();#endif init_sched_fair_class (); +init_sched_numa (); scheduler_running =1 ; }
kernel/sched/fair.c +2 -4
@@ -3279,8 +3279,6 @@ static int move_one_task(struct lb_env *env)return 0 ; } -static unsigned long task_h_load (struct task_struct *p); -static const unsigned int sched_nr_migrate_break =32 ;/*
@@ -3454,7 +3452,7 @@ static void update_h_load(long cpu)rcu_read_unlock (); } -static unsigned long task_h_load (struct task_struct *p) +unsigned long task_h_load (struct task_struct *p) {struct cfs_rq *cfs_rq =task_cfs_rq (p);unsigned long load;
@@ -3473,7 +3471,7 @@ static inline void update_h_load(long cpu) { } -static unsigned long task_h_load (struct task_struct *p) +unsigned long task_h_load (struct task_struct *p) {return p->se.load.weight; }
kernel/sched/features.h +4 -0
@@ -75,5 +75,9 @@ SCHED_FEAT(NUMA_HOT, true) SCHED_FEAT(NUMA_BIAS, true) SCHED_FEAT(NUMA_PULL, true) SCHED_FEAT(NUMA_PULL_BIAS, true) +SCHED_FEAT(NUMA_BALANCE, true) +SCHED_FEAT(NUMA_SELECT, true) +SCHED_FEAT(NUMA_SLOW, false) +SCHED_FEAT(NUMA_BALANCE_FILTER, false)#endif
kernel/sched/numa.c new file +741 -0
741 lines not on this page · raw message
kernel/sched/sched.h +16 -0
@@ -1179,9 +1179,25 @@ enum rq_nohz_flag_bits {#define nohz_flags (cpu) (&cpu_rq(cpu)->nohz_flags)#endif +unsigned long task_h_load (struct task_struct *p); + +#ifdef CONFIG_NUMA + +void sched_setnode (struct task_struct *p,int node); +void select_task_node (struct task_struct *p,struct mm_struct *mm,int sd_flags); +bool account_numa_enqueue (struct task_struct *p); +void account_numa_dequeue (struct task_struct *p); +void init_sched_numa (void ); + +#else /* CONFIG_NUMA */ +/* * Macro to avoid argument evaluation */ #define select_task_node (p, mm, sd_flags) do { } while (0)static inline bool account_numa_enqueue (struct task_struct *p) {return false; }static inline void account_numa_dequeue (struct task_struct *p) { } +static inline void init_sched_numa (void ) { } + +#endif /* CONFIG_NUMA */ +
mm/init-mm.c +10 -0
@@ -13,6 +13,15 @@#define INIT_MM_CONTEXT (name)#endif +#ifdef CONFIG_NUMA +# define INIT_MM_NUMA (mm) \ + .numa = { \ + .node = -1, \ + }, +#else +# define INIT_MM_NUMA (mm) +#endif +struct mm_struct init_mm = { .mm_rb =RB_ROOT , .pgd = swapper_pg_dir,
@@ -22,4 +31,5 @@ struct mm_struct init_mm = { .page_table_lock = __SPIN_LOCK_UNLOCKED(init_mm.page_table_lock), .mmlist = LIST_HEAD_INIT(init_mm.mmlist), INIT_MM_CONTEXT(init_mm) + INIT_MM_NUMA(init_mm) };