sched, numa, mm: Add fault driven placement and migration policy
Subsystems (5)
Commit-ID: f7a688de2312e39cb456df3844f3996cc8c8ba5d Gitweb: http://git.kernel.org/tip/f7a688de2312e39cb456df3844f3996cc8c8ba5d Author: Peter Zijlstra <[email protected]> AuthorDate: Tue, 9 Oct 2012 13:46:22 +0200 Committer: Ingo Molnar <[email protected]> CommitDate: Sun, 28 Oct 2012 17:31:15 +0100
sched, numa, mm: Add fault driven placement and migration policy
As per the problem/design document Documentation/scheduler/numa-problem.txt implement 3ac & 4.
( A pure 3a was found too unstable, I did briefly try 3bc
but found no significant improvement. )
Implement a per-task memory placement scheme relying on a regular PROT_NONE 'migration' fault to scan the memory space of the procress and uses a two stage migration scheme to reduce the invluence of unlikely usage relations.
It relies on the assumption that the compute part is tied to a paticular task and builds a task<->page relation set to model the compute<->data relation.
In the previous patch we made memory migrate towards where the task is running, here we select the node on which most memory is located as the preferred node to run on.
This creates a feed-back control loop between trying to schedule a task on a node and migrating memory towards the node the task is scheduled on.
Suggested-by Andrea Arcangeli <[email protected]>
Suggested-by Rik van Riel <[email protected]>
Fixes-by David Rientjes <[email protected]>
Cc Linus Torvalds <[email protected]>
Cc Andrew Morton <[email protected]>
Signed-off-by Peter Zijlstra <[email protected]>
Signed-off-by Ingo Molnar <[email protected]>
--- include/linux/mm_types.h | 4 + include/linux/sched.h | 35 +++++++-- kernel/sched/core.c | 16 ++++ kernel/sched/fair.c | 187 ++++++++++++++++++++++++++++++++++++++++++++++ kernel/sched/features.h | 1 + kernel/sched/sched.h | 31 ++++++-- kernel/sysctl.c | 31 +++++++- mm/huge_memory.c | 7 +- mm/memory.c | 4 +- 9 files changed, 294 insertions(+), 22 deletions(-)
include/linux/mm_types.h +4 -0
@@ -403,6 +403,10 @@ struct mm_struct {#ifdef CONFIG_CPUMASK_OFFSTACK struct cpumask cpumask_allocation;#endif +#ifdef CONFIG_SCHED_NUMA +unsigned long numa_next_scan; +int numa_scan_seq; +#endif struct uprobes_state uprobes_state; };
include/linux/sched.h +27 -8
@@ -1481,9 +1481,16 @@ struct task_struct {short pref_node_fork;#endif #ifdef CONFIG_SCHED_NUMA -int node; +int node;/* task home node */ +int numa_scan_seq; +int numa_migrate_seq; +unsigned int numa_scan_period; +u64 node_stamp;/* migration stamp */ unsigned long numa_contrib; -#endif +unsigned long *numa_faults; +struct callback_head numa_work; +#endif /* CONFIG_SCHED_NUMA */ +struct rcu_head rcu;/*
@@ -1558,15 +1565,24 @@ struct task_struct {/* Future-safe accessor for struct task_struct's cpus_allowed. */ #define tsk_cpus_allowed (tsk) (&(tsk)->cpus_allowed) +#ifdef CONFIG_SCHED_NUMA static inline int tsk_home_node (struct task_struct *p) { -#ifdef CONFIG_SCHED_NUMA return p->node; +} + +extern void task_numa_fault (int node,int pages);#else +static inline int tsk_home_node (struct task_struct *p) +{return -1 ; -#endif } +static inline void task_numa_fault (int node,int pages) +{ +} +#endif /* CONFIG_SCHED_NUMA */ +/* * Priority of a process goes from 0..MAX_PRIO-1, valid RT * priority is 0..MAX_RT_PRIO-1, and SCHED_NORMAL/SCHED_BATCH
@@ -2004,6 +2020,10 @@ enum sched_tunable_scaling { };extern enum sched_tunable_scaling sysctl_sched_tunable_scaling; +extern unsigned int sysctl_sched_numa_scan_period_min; +extern unsigned int sysctl_sched_numa_scan_period_max; +extern unsigned int sysctl_sched_numa_settle_count; +#ifdef CONFIG_SCHED_DEBUG extern unsigned int sysctl_sched_migration_cost;extern unsigned int sysctl_sched_nr_migrate;
@@ -2014,18 +2034,17 @@ extern unsigned int sysctl_sched_shares_window;int sched_proc_update_handler (struct ctl_table *table,int write,void __user *buffer,size_t *length,loff_t *ppos); -#endif -#ifdef CONFIG_SCHED_DEBUG +static inline unsigned int get_sysctl_timer_migration (void ) {return sysctl_timer_migration; } -#else +#else /* CONFIG_SCHED_DEBUG */ static inline unsigned int get_sysctl_timer_migration (void ) {return 1 ; } -#endif +#endif /* CONFIG_SCHED_DEBUG */ extern unsigned int sysctl_sched_rt_period;extern int sysctl_sched_rt_runtime;
kernel/sched/core.c +16 -0
@@ -1533,6 +1533,21 @@ static void __sched_fork(struct task_struct *p)#ifdef CONFIG_PREEMPT_NOTIFIERS INIT_HLIST_HEAD (&p->preempt_notifiers);#endif + +#ifdef CONFIG_SCHED_NUMA +if (p->mm &&atomic_read (&p->mm->mm_users) ==1 ) { + p->mm->numa_next_scan = jiffies; + p->mm->numa_scan_seq =0 ; + } + + p->node =-1 ; + p->node_stamp =0ULL ; + p->numa_scan_seq = p->mm ? p->mm->numa_scan_seq :0 ; + p->numa_migrate_seq = p->mm ? p->mm->numa_scan_seq -1 :0 ; + p->numa_faults =NULL ; + p->numa_scan_period = sysctl_sched_numa_scan_period_min; + p->numa_work.next = &p->numa_work; +#endif /* CONFIG_SCHED_NUMA */ }/*
@@ -1774,6 +1789,7 @@ static void finish_task_switch(struct rq *rq, struct task_struct *prev)if (mm)mmdrop (mm);if (unlikely (prev_state ==TASK_DEAD )) { +task_numa_free (prev);/* * Remove function-return probe instances associated with this * task and put them back on the free list.
kernel/sched/fair.c +187 -0
217 lines not on this page · raw message
kernel/sched/features.h +1 -0
@@ -69,5 +69,6 @@ SCHED_FEAT(NUMA_TTWU_BIAS, false) SCHED_FEAT(NUMA_TTWU_TO, false) SCHED_FEAT(NUMA_PULL, true) SCHED_FEAT(NUMA_PULL_BIAS, true) +SCHED_FEAT(NUMA_SETTLE, true)#endif
kernel/sched/sched.h +22 -9
@@ -3,6 +3,7 @@#include <linux/mutex.h> #include <linux/spinlock.h> #include <linux/stop_machine.h> +#include <linux/slab.h> #include "cpupri.h"
@@ -476,15 +477,6 @@ struct rq {#endif }; -static inline struct list_head *offnode_tasks (struct rq *rq) -{ -#ifdef CONFIG_SCHED_NUMA -return &rq->offnode_tasks; -#else -return NULL ; -#endif -} -static inline int cpu_of (struct rq *rq) {#ifdef CONFIG_SMP
@@ -502,6 +494,27 @@ DECLARE_PER_CPU(struct rq, runqueues);#define cpu_curr (cpu) (cpu_rq(cpu)->curr)#define raw_rq () (&__raw_get_cpu_var(runqueues)) +#ifdef CONFIG_SCHED_NUMA +static inline struct list_head *offnode_tasks (struct rq *rq) +{ +return &rq->offnode_tasks; +} + +static inline void task_numa_free (struct task_struct *p) +{ +kfree (p->numa_faults); +} +#else /* CONFIG_SCHED_NUMA */ +static inline struct list_head *offnode_tasks (struct rq *rq) +{ +return NULL ; +} + +static inline void task_numa_free (struct task_struct *p) +{ +} +#endif /* CONFIG_SCHED_NUMA */ +#ifdef CONFIG_SMP #define rcu_dereference_check_sched_domain (p) \
kernel/sysctl.c +29 -2
@@ -256,9 +256,11 @@ static int min_sched_granularity_ns = 100000; /* 100 usecs */static int max_sched_granularity_ns =NSEC_PER_SEC ;/* 1 second */ static int min_wakeup_granularity_ns;/* 0 usecs */ static int max_wakeup_granularity_ns =NSEC_PER_SEC ;/* 1 second */ +#ifdef CONFIG_SMP static int min_sched_tunable_scaling =SCHED_TUNABLESCALING_NONE ;static int max_sched_tunable_scaling =SCHED_TUNABLESCALING_END -1 ; -#endif +#endif /* CONFIG_SMP */ +#endif /* CONFIG_SCHED_DEBUG */ #ifdef CONFIG_COMPACTION static int min_extfrag_threshold;
@@ -301,6 +303,7 @@ static struct ctl_table kern_table[] = { .extra1 = &min_wakeup_granularity_ns, .extra2 = &max_wakeup_granularity_ns, }, +#ifdef CONFIG_SMP { .procname = "sched_tunable_scaling", .data = &sysctl_sched_tunable_scaling,
@@ -347,7 +350,31 @@ static struct ctl_table kern_table[] = { .extra1 = &zero, .extra2 = &one, }, -#endif +#endif /* CONFIG_SMP */ +#ifdef CONFIG_SCHED_NUMA + { + .procname = "sched_numa_scan_period_min_ms", + .data = &sysctl_sched_numa_scan_period_min, + .maxlen = sizeof(unsigned int), + .mode = 0644, + .proc_handler = proc_dointvec, + }, + { + .procname = "sched_numa_scan_period_max_ms", + .data = &sysctl_sched_numa_scan_period_max, + .maxlen = sizeof(unsigned int), + .mode = 0644, + .proc_handler = proc_dointvec, + }, + { + .procname = "sched_numa_settle_count", + .data = &sysctl_sched_numa_settle_count, + .maxlen = sizeof(unsigned int), + .mode = 0644, + .proc_handler = proc_dointvec, + }, +#endif /* CONFIG_SCHED_NUMA */ +#endif /* CONFIG_SCHED_DEBUG */ { .procname = "sched_rt_period_us", .data = &sysctl_sched_rt_period,
mm/huge_memory.c +5 -2
@@ -774,9 +774,10 @@ fixup: unlock:spin_unlock (&mm->page_table_lock); -if (page) +if (page) { +task_numa_fault (page_to_nid (page),HPAGE_PMD_NR );put_page (page); - + }return ; migrate:
@@ -845,6 +846,8 @@ migrate:put_page (page);/* Drop the rmap reference */ +task_numa_fault (node,HPAGE_PMD_NR ); +if (lru)put_page (page);/* drop the LRU isolation reference */
mm/memory.c +3 -1
@@ -3512,8 +3512,10 @@ out_pte_upgrade_unlock: out_unlock:pte_unmap_unlock (ptep, ptl); out: -if (page) +if (page) { +task_numa_fault (page_nid,1 );put_page (page); + }return 0 ;