sched: Implement home-node awareness
Commit-ID: a722b115ba8461858e26ba9cf899e98a694e5c72 Gitweb: http://git.kernel.org/tip/a722b115ba8461858e26ba9cf899e98a694e5c72 Author: Peter Zijlstra <[email protected]> AuthorDate: Sat, 3 Mar 2012 16:56:25 +0100 Committer: Ingo Molnar <[email protected]> CommitDate: Sat, 19 May 2012 12:52:20 +0200
sched: Implement home-node awareness
Implement home node preference in the load-balancer.
This is done in four pieces:
- task_numa_hot(); make it harder to migrate tasks away from their home-node, controlled using the NUMA_HOT feature flag.
- select_task_rq_fair(); prefer placing the task in their home-node, controlled using the NUMA_BIAS feature flag.
- load_balance(); during the regular pull load-balance pass, try pulling tasks that are on the wrong node first with a preference of moving them nearer to their home-node through task_numa_hot(), controlled through the NUMA_PULL feature flag.
- load_balance(); when the balancer finds no imbalance, introduce some such that it still prefers to move tasks towards their home-node, using active load-balance if needed, controlled through the NUMA_PULL_BIAS feature flag.
In order to easily find off-node tasks, split the per-cpu task list into two parts.
Signed-off-by Peter Zijlstra <[email protected]>
Cc Suresh Siddha <[email protected]>
Cc Paul Turner <[email protected]>
Cc Dan Smith <[email protected]>
Cc Bharata B Rao <[email protected]>
Cc Lee Schermerhorn <[email protected]>
Cc Christoph Lameter <[email protected]>
Cc Rik van Riel <[email protected]>
Cc Andrea Arcangeli <[email protected]>
Cc Andrew Morton <[email protected]>
Cc Linus Torvalds <[email protected]>
Signed-off-by Ingo Molnar <[email protected]>
--- include/linux/sched.h | 1 + kernel/sched/core.c | 21 ++++- kernel/sched/debug.c | 3 + kernel/sched/fair.c | 236 ++++++++++++++++++++++++++++++++++++++++++++--- kernel/sched/features.h | 8 ++ kernel/sched/sched.h | 18 ++++ 6 files changed, 270 insertions(+), 17 deletions(-)
include/linux/sched.h +1 -0
@@ -860,6 +860,7 @@ enum cpu_idle_type {#define SD_ASYM_PACKING 0x0800/* Place busy groups earlier in the domain */ #define SD_PREFER_SIBLING 0x1000/* Prefer to place tasks in a sibling domain */ #define SD_OVERLAP 0x2000/* sched_domains of this level overlap */ +#define SD_NUMA 0x4000/* cross-node balancing */ extern int __weak arch_sd_sibiling_asym_packing (void );
kernel/sched/core.c +19 -2
@@ -5859,7 +5859,9 @@ static void destroy_sched_domains(struct sched_domain *sd, int cpu) DEFINE_PER_CPU(struct sched_domain *, sd_llc);DEFINE_PER_CPU (int, sd_llc_id); -static void update_top_cache_domain (int cpu) +DEFINE_PER_CPU(struct sched_domain *, sd_node); + +static void update_domain_cache (int cpu) {struct sched_domain *sd;int id = cpu;
@@ -5870,6 +5872,15 @@ static void update_top_cache_domain(int cpu)rcu_assign_pointer (per_cpu (sd_llc, cpu), sd);per_cpu (sd_llc_id, cpu) = id; + +for_each_domain (cpu, sd) { +if (cpumask_equal (sched_domain_span (sd), +cpumask_of_node (cpu_to_node (cpu)))) + goto got_node; + } + sd =NULL ; +got_node: +rcu_assign_pointer (per_cpu (sd_node, cpu), sd); }/*
@@ -5912,7 +5923,7 @@ cpu_attach_domain(struct sched_domain *sd, struct root_domain *rd, int cpu)rcu_assign_pointer (rq->sd, sd);destroy_sched_domains (tmp, cpu); -update_top_cache_domain (cpu); +update_domain_cache (cpu); }/* cpus with isolated domains */
@@ -6359,6 +6370,7 @@ sd_numa_init(struct sched_domain_topology_level *tl, int cpu) | 0*SD_SHARE_PKG_RESOURCES |1 *SD_SERIALIZE |0 *SD_PREFER_SIBLING + |1 *SD_NUMA |sd_local_flags (level) , .last_balance = jiffies,
@@ -7066,6 +7078,11 @@ void __init sched_init(void) rq->avg_idle =2 *sysctl_sched_migration_cost;INIT_LIST_HEAD (&rq->cfs_tasks); +#ifdef CONFIG_NUMA +INIT_LIST_HEAD (&rq->offnode_tasks); + rq->offnode_running =0 ; + rq->offnode_weight =0 ; +#endif rq_attach_root (rq, &def_root_domain);#ifdef CONFIG_NO_HZ
kernel/sched/debug.c +3 -0
@@ -132,6 +132,9 @@ print_task(struct seq_file *m, struct rq *rq, struct task_struct *p)SEQ_printf (m,"%15Ld %15Ld %15Ld.%06ld %15Ld.%06ld %15Ld.%06ld" ,0LL ,0LL ,0LL ,0L ,0LL ,0L ,0LL ,0L );#endif +#ifdef CONFIG_NUMA +SEQ_printf (m," %d/%d" , p->node,cpu_to_node (task_cpu (p))); +#endif #ifdef CONFIG_CGROUP_SCHED SEQ_printf (m," %s" ,task_group_path (task_group (p)));#endif
kernel/sched/fair.c +221 -15
395 lines not on this page · raw message
kernel/sched/features.h +8 -0
@@ -69,3 +69,11 @@ SCHED_FEAT(TTWU_QUEUE, true) SCHED_FEAT(FORCE_SD_OVERLAP, false) SCHED_FEAT(RT_RUNTIME_SHARE, true) SCHED_FEAT(LB_MIN, false) + +#ifdef CONFIG_NUMA +SCHED_FEAT(NUMA_HOT, true) +SCHED_FEAT(NUMA_BIAS, true) +SCHED_FEAT(NUMA_PULL, true) +SCHED_FEAT(NUMA_PULL_BIAS, true) +#endif +
kernel/sched/sched.h +18 -0
@@ -414,6 +414,12 @@ struct rq {struct list_head cfs_tasks; +#ifdef CONFIG_NUMA +unsigned long offnode_running; +unsigned long offnode_weight; +struct list_head offnode_tasks; +#endif +u64 rt_avg;u64 age_stamp;u64 idle_stamp;
@@ -465,6 +471,15 @@ struct rq {#endif }; +static inline struct list_head *offnode_tasks (struct rq *rq) +{ +#ifdef CONFIG_NUMA +return &rq->offnode_tasks; +#else +return NULL ; +#endif +} +static inline int cpu_of (struct rq *rq) {#ifdef CONFIG_SMP
@@ -525,6 +540,7 @@ static inline struct sched_domain *highest_flag_domain(int cpu, int flag) DECLARE_PER_CPU(struct sched_domain *, sd_llc);DECLARE_PER_CPU (int, sd_llc_id); +DECLARE_PER_CPU(struct sched_domain *, sd_node);#endif /* CONFIG_SMP */
@@ -1167,3 +1183,5 @@ enum rq_nohz_flag_bits {* Macro to avoid argument evaluation */ #define select_task_node (p, mm, sd_flags) do { } while (0) +static inline bool account_numa_enqueue (struct task_struct *p) {return false; } +static inline void account_numa_dequeue (struct task_struct *p) { }