sched: Implement home-node awareness
Commit-ID: fc2fbef0374693278a767a3d31ef18293e68f6be Gitweb: http://git.kernel.org/tip/fc2fbef0374693278a767a3d31ef18293e68f6be Author: Peter Zijlstra <[email protected]> AuthorDate: Sat, 3 Mar 2012 16:56:25 +0100 Committer: Ingo Molnar <[email protected]> CommitDate: Thu, 27 Sep 2012 17:04:44 +0200
sched: Implement home-node awareness
Implement home node preference in the scheduler's load-balancer.
This is done in four pieces:
- task_numa_hot(); make it harder to migrate tasks away from their home-node, controlled using the NUMA_HOT feature flag.
- select_task_rq_fair(); prefer placing the task in their home-node, controlled using the NUMA_BIAS feature flag.
- load_balance(); during the regular pull load-balance pass, try pulling tasks that are on the wrong node first with a preference of moving them nearer to their home-node through task_numa_hot(), controlled through the NUMA_PULL feature flag.
- load_balance(); when the balancer finds no imbalance, introduce some such that it still prefers to move tasks towards their home-node, using active load-balance if needed, controlled through the NUMA_PULL_BIAS feature flag.
In order to easily find off-node tasks, split the per-cpu task list into two parts.
--- include/linux/sched.h | 1 + kernel/sched/core.c | 21 ++++- kernel/sched/debug.c | 3 + kernel/sched/fair.c | 248 +++++++++++++++++++++++++++++++++++++++++++++-- kernel/sched/features.h | 8 ++ kernel/sched/sched.h | 16 +++ 6 files changed, 286 insertions(+), 11 deletions(-)
include/linux/sched.h +1 -0
@@ -861,6 +861,7 @@ enum cpu_idle_type {#define SD_ASYM_PACKING 0x0800/* Place busy groups earlier in the domain */ #define SD_PREFER_SIBLING 0x1000/* Prefer to place tasks in a sibling domain */ #define SD_OVERLAP 0x2000/* sched_domains of this level overlap */ +#define SD_NUMA 0x4000/* cross-node balancing */ extern int __weak arch_sd_sibiling_asym_packing (void );
kernel/sched/core.c +19 -2
@@ -5472,7 +5472,9 @@ static void destroy_sched_domains(struct sched_domain *sd, int cpu) DEFINE_PER_CPU(struct sched_domain *, sd_llc);DEFINE_PER_CPU (int, sd_llc_id); -static void update_top_cache_domain (int cpu) +DEFINE_PER_CPU(struct sched_domain *, sd_node); + +static void update_domain_cache (int cpu) {struct sched_domain *sd;int id = cpu;
@@ -5515,6 +5517,15 @@ static void update_top_cache_domain(int cpu)rcu_assign_pointer (per_cpu (sd_llc, cpu), sd);per_cpu (sd_llc_id, cpu) = id; + +for_each_domain (cpu, sd) { +if (cpumask_equal (sched_domain_span (sd), +cpumask_of_node (cpu_to_node (cpu)))) + goto got_node; + } + sd =NULL ; +got_node: +rcu_assign_pointer (per_cpu (sd_node, cpu), sd); }/*
@@ -5557,7 +5568,7 @@ cpu_attach_domain(struct sched_domain *sd, struct root_domain *rd, int cpu)rcu_assign_pointer (rq->sd, sd);destroy_sched_domains (tmp, cpu); -update_top_cache_domain (cpu); +update_domain_cache (cpu); }/* cpus with isolated domains */
@@ -6060,6 +6071,7 @@ sd_numa_init(struct sched_domain_topology_level *tl, int cpu) | 0*SD_SHARE_PKG_RESOURCES |1 *SD_SERIALIZE |0 *SD_PREFER_SIBLING + |1 *SD_NUMA |sd_local_flags (level) , .last_balance = jiffies,
@@ -6852,6 +6864,11 @@ void __init sched_init(void) rq->avg_idle =2 *sysctl_sched_migration_cost;INIT_LIST_HEAD (&rq->cfs_tasks); +#ifdef CONFIG_SCHED_NUMA +INIT_LIST_HEAD (&rq->offnode_tasks); + rq->offnode_running =0 ; + rq->offnode_weight =0 ; +#endif rq_attach_root (rq, &def_root_domain);#ifdef CONFIG_NO_HZ
kernel/sched/debug.c +3 -0
@@ -132,6 +132,9 @@ print_task(struct seq_file *m, struct rq *rq, struct task_struct *p)SEQ_printf (m,"%15Ld %15Ld %15Ld.%06ld %15Ld.%06ld %15Ld.%06ld" ,0LL ,0LL ,0LL ,0L ,0LL ,0L ,0LL ,0L );#endif +#ifdef CONFIG_SCHED_NUMA +SEQ_printf (m," %d/%d" , p->node,cpu_to_node (task_cpu (p))); +#endif #ifdef CONFIG_CGROUP_SCHED SEQ_printf (m," %s" ,task_group_path (task_group (p)));#endif
kernel/sched/fair.c +239 -9
400 lines not on this page · raw message
kernel/sched/features.h +8 -0
@@ -61,3 +61,11 @@ SCHED_FEAT(TTWU_QUEUE, true) SCHED_FEAT(FORCE_SD_OVERLAP, false) SCHED_FEAT(RT_RUNTIME_SHARE, true) SCHED_FEAT(LB_MIN, false) + +#ifdef CONFIG_SCHED_NUMA +SCHED_FEAT(NUMA_HOT, true) +SCHED_FEAT(NUMA_BIAS, true) +SCHED_FEAT(NUMA_PULL, true) +SCHED_FEAT(NUMA_PULL_BIAS, true) +#endif +
kernel/sched/sched.h +16 -0
@@ -418,6 +418,12 @@ struct rq {struct list_head cfs_tasks; +#ifdef CONFIG_SCHED_NUMA +unsigned long offnode_running; +unsigned long offnode_weight; +struct list_head offnode_tasks; +#endif +u64 rt_avg;u64 age_stamp;u64 idle_stamp;
@@ -469,6 +475,15 @@ struct rq {#endif }; +static inline struct list_head *offnode_tasks (struct rq *rq) +{ +#ifdef CONFIG_SCHED_NUMA +return &rq->offnode_tasks; +#else +return NULL ; +#endif +} +static inline int cpu_of (struct rq *rq) {#ifdef CONFIG_SMP
@@ -529,6 +544,7 @@ static inline struct sched_domain *highest_flag_domain(int cpu, int flag) DECLARE_PER_CPU(struct sched_domain *, sd_llc);DECLARE_PER_CPU (int, sd_llc_id); +DECLARE_PER_CPU(struct sched_domain *, sd_node);extern int group_balance_cpu (struct sched_group *sg);