sched/numa: Implement NUMA home-node selection code
Subsystems (3)
Commit-ID: 4ae834f767c59b42cdf2a5077a18388525ba78b2 Gitweb: http://git.kernel.org/tip/4ae834f767c59b42cdf2a5077a18388525ba78b2 Author: Peter Zijlstra <[email protected]> AuthorDate: Mon, 16 Jul 2012 11:22:09 +0200 Committer: Ingo Molnar <[email protected]> CommitDate: Thu, 27 Sep 2012 17:04:47 +0200
sched/numa: Implement NUMA home-node selection code
Now that we have infrastructure in place to migrate pages back to their home-node, and migrate memory towards the home-node, we need to set the home-node.
Instead of creating a seconday control loop, fully rely on the existing load-balancer to do the right thing. The home-node selection logic will simply pick the node the task has been found to run on for two consequtive samples (see task_tick_numa).
This means NUMA placement is directly related to regular placement. The home-node logic in the load-balancer tries to keep a task on the home-node wheras the fairness and work-conserving constraints will try and move it away.
The balance between these two 'forces' is what will result in the NUMA placement.
--- include/linux/init_task.h | 3 +- include/linux/mm_types.h | 3 + include/linux/sched.h | 16 +++-- kernel/sched/core.c | 16 ++++- kernel/sched/fair.c | 174 +++++++++++++++++++++++++++++++++++++++++++-- kernel/sched/features.h | 1 + kernel/sched/sched.h | 33 ++++++--- kernel/sysctl.c | 13 +++- 8 files changed, 233 insertions(+), 26 deletions(-)
include/linux/init_task.h +2 -1
@@ -145,7 +145,8 @@ extern struct task_group root_task_group;#ifdef CONFIG_SCHED_NUMA # define INIT_TASK_NUMA (tsk) \ - .node = -1, + .node = -1, \ + .node_last = -1,#else # define INIT_TASK_NUMA (tsk)#endif
include/linux/mm_types.h +3 -0
@@ -404,6 +404,9 @@ struct mm_struct {#ifdef CONFIG_CPUMASK_OFFSTACK struct cpumask cpumask_allocation;#endif +#ifdef CONFIG_SCHED_NUMA +unsigned long numa_next_scan; +#endif struct uprobes_state uprobes_state; };
include/linux/sched.h +10 -6
@@ -1520,8 +1520,12 @@ struct task_struct {short pref_node_fork;#endif #ifdef CONFIG_SCHED_NUMA -int node; -#endif +int node;/* task home node */ +int node_last;/* home node filter */ +u64 node_stamp;/* migration stamp */ +unsigned long numa_contrib; +#endif /* CONFIG_SCHED_NUMA */ +struct rcu_head rcu;/*
@@ -2037,22 +2041,22 @@ extern unsigned int sysctl_sched_nr_migrate;extern unsigned int sysctl_sched_time_avg;extern unsigned int sysctl_timer_migration;extern unsigned int sysctl_sched_shares_window; +extern unsigned int sysctl_sched_numa_task_period;int sched_proc_update_handler (struct ctl_table *table,int write,void __user *buffer,size_t *length,loff_t *ppos); -#endif -#ifdef CONFIG_SCHED_DEBUG +static inline unsigned int get_sysctl_timer_migration (void ) {return sysctl_timer_migration; } -#else +#else /* CONFIG_SCHED_DEBUG */ static inline unsigned int get_sysctl_timer_migration (void ) {return 1 ; } -#endif +#endif /* CONFIG_SCHED_DEBUG */ extern unsigned int sysctl_sched_rt_period;extern int sysctl_sched_rt_runtime;
kernel/sched/core.c +13 -3
@@ -1533,6 +1533,15 @@ static void __sched_fork(struct task_struct *p)#ifdef CONFIG_PREEMPT_NOTIFIERS INIT_HLIST_HEAD (&p->preempt_notifiers);#endif + +#ifdef CONFIG_SCHED_NUMA +if (p->mm &&atomic_read (&p->mm->mm_users) ==1 ) + p->mm->numa_next_scan = jiffies; + + p->node =-1 ; + p->node_last =-1 ; + p->node_stamp =0ULL ; +#endif /* CONFIG_NUMA */ }/*
@@ -5996,9 +6005,9 @@ static struct sched_domain_topology_level *sched_domain_topology = default_topol* Requeues a task ensuring its on the right load-balance list so * that it might get migrated to its new home. * -* Note that we cannot actively migrate ourselves since our callers -* can be from atomic context. We rely on the regular load-balance -* mechanisms to move us around -- its all preference anyway. +* Since home-node is pure preference there's no hard migrate to force +* us anywhere, this also allows us to call this from atomic context if +* required. */ void sched_setnode (struct task_struct *p,int node) {
@@ -6016,6 +6025,7 @@ void sched_setnode(struct task_struct *p, int node) p->sched_class->put_prev_task (rq, p); p->node = node; + p->node_last = node;if (running) p->sched_class->set_curr_task (rq);
kernel/sched/fair.c +168 -6
211 lines not on this page · raw message
kernel/sched/features.h +1 -0
@@ -63,6 +63,7 @@ SCHED_FEAT(RT_RUNTIME_SHARE, true) SCHED_FEAT(LB_MIN, false)#ifdef CONFIG_SCHED_NUMA +SCHED_FEAT(NUMA, true) SCHED_FEAT(NUMA_HOT, true) SCHED_FEAT(NUMA_BIAS, true)SCHED_FEAT (NUMA_PULL , true)
kernel/sched/sched.h +24 -9
@@ -475,15 +475,6 @@ struct rq {#endif }; -static inline struct list_head *offnode_tasks (struct rq *rq) -{ -#ifdef CONFIG_SCHED_NUMA -return &rq->offnode_tasks; -#else -return NULL ; -#endif -} -static inline int cpu_of (struct rq *rq) {#ifdef CONFIG_SMP
@@ -501,6 +492,30 @@ DECLARE_PER_CPU(struct rq, runqueues);#define cpu_curr (cpu) (cpu_rq(cpu)->curr)#define raw_rq () (&__raw_get_cpu_var(runqueues)) +#ifdef CONFIG_SCHED_NUMA +static inline bool offnode_task (struct task_struct *t) +{ +return t->node !=-1 && t->node !=cpu_to_node (task_cpu (t)); +} + +static inline struct list_head *offnode_tasks (struct rq *rq) +{ +return &rq->offnode_tasks; +} + +void sched_setnode (struct task_struct *p,int node); +#else /* CONFIG_SCHED_NUMA */ +static inline bool offnode_task (struct task_struct *t) +{ +return false; +} + +static inline struct list_head *offnode_tasks (struct rq *rq) +{ +return NULL ; +} +#endif /* CONFIG_SCHED_NUMA */ +#ifdef CONFIG_SMP #define rcu_dereference_check_sched_domain (p) \
kernel/sysctl.c +12 -1
@@ -297,6 +297,7 @@ static struct ctl_table kern_table[] = { .extra1 = &min_wakeup_granularity_ns, .extra2 = &max_wakeup_granularity_ns, }, +#ifdef CONFIG_SMP { .procname = "sched_tunable_scaling", .data = &sysctl_sched_tunable_scaling,
@@ -343,7 +344,17 @@ static struct ctl_table kern_table[] = { .extra1 = &zero, .extra2 = &one, }, -#endif +#endif /* CONFIG_SMP */ +#ifdef CONFIG_SCHED_NUMA + { + .procname = "sched_numa_task_period_ms", + .data = &sysctl_sched_numa_task_period, + .maxlen = sizeof(unsigned int), + .mode = 0644, + .proc_handler = proc_dointvec, + }, +#endif /* CONFIG_SCHED_NUMA */ +#endif /* CONFIG_SCHED_DEBUG */ { .procname = "sched_rt_period_us", .data = &sysctl_sched_rt_period,