Skip to contentSkip to conversation

sched/numa: Implement NUMA home-node selection code

tip:sched/numa tip-bot for Peter Zijlstra 1 participant 1 message Sep 28, 2012

Subsystems (3)
lore
Share & tools
lore
#0 lore raw
Commit-ID:  4ae834f767c59b42cdf2a5077a18388525ba78b2
Gitweb:     http://git.kernel.org/tip/4ae834f767c59b42cdf2a5077a18388525ba78b2
Author:     Peter Zijlstra <[email protected]>
AuthorDate: Mon, 16 Jul 2012 11:22:09 +0200
Committer:  Ingo Molnar <[email protected]>
CommitDate: Thu, 27 Sep 2012 17:04:47 +0200

sched/numa: Implement NUMA home-node selection code

Now that we have infrastructure in place to migrate pages back to their home-node, and migrate memory towards the home-node, we need to set the home-node.

Instead of creating a seconday control loop, fully rely on the existing load-balancer to do the right thing. The home-node selection logic will simply pick the node the task has been found to run on for two consequtive samples (see task_tick_numa).

This means NUMA placement is directly related to regular placement. The home-node logic in the load-balancer tries to keep a task on the home-node wheras the fairness and work-conserving constraints will try and move it away.

The balance between these two 'forces' is what will result in the NUMA placement.

---
 include/linux/init_task.h |    3 +-
 include/linux/mm_types.h  |    3 +
 include/linux/sched.h     |   16 +++--
 kernel/sched/core.c       |   16 ++++-
 kernel/sched/fair.c       |  174 +++++++++++++++++++++++++++++++++++++++++++--
 kernel/sched/features.h   |    1 +
 kernel/sched/sched.h      |   33 ++++++---
 kernel/sysctl.c           |   13 +++-
 8 files changed, 233 insertions(+), 26 deletions(-)
include/linux/init_task.h +2 -1
index 18906c1..36aca4e 100644
@@ -145,7 +145,8 @@ extern struct task_group root_task_group;
 
 #ifdef CONFIG_SCHED_NUMA
 # define INIT_TASK_NUMA(tsk)						\
-	.node = -1,
+	.node = -1,							\
+	.node_last = -1,
 #else
 # define INIT_TASK_NUMA(tsk)
 #endif
include/linux/mm_types.h +3 -0
index bf78672..f407966 100644
@@ -404,6 +404,9 @@ struct mm_struct {
 #ifdef CONFIG_CPUMASK_OFFSTACK
 	struct cpumask cpumask_allocation;
 #endif
+#ifdef CONFIG_SCHED_NUMA
+	unsigned long numa_next_scan;
+#endif
 	struct uprobes_state uprobes_state;
 };
 
include/linux/sched.h +10 -6
index 8755ef1..63dde76 100644
@@ -1520,8 +1520,12 @@ struct task_struct {
 	short pref_node_fork;
 #endif
 #ifdef CONFIG_SCHED_NUMA
-	int node;
-#endif
+	int node;			/* task home node   */
+	int node_last;			/* home node filter */
+	u64 node_stamp;			/* migration stamp  */
+	unsigned long numa_contrib;
+#endif /* CONFIG_SCHED_NUMA */
+
 	struct rcu_head rcu;
 
 	/*
@@ -2037,22 +2041,22 @@ extern unsigned int sysctl_sched_nr_migrate;
 extern unsigned int sysctl_sched_time_avg;
 extern unsigned int sysctl_timer_migration;
 extern unsigned int sysctl_sched_shares_window;
+extern unsigned int sysctl_sched_numa_task_period;
 
 int sched_proc_update_handler(struct ctl_table *table, int write,
 		void __user *buffer, size_t *length,
 		loff_t *ppos);
-#endif
-#ifdef CONFIG_SCHED_DEBUG
+
 static inline unsigned int get_sysctl_timer_migration(void)
 {
 	return sysctl_timer_migration;
 }
-#else
+#else /* CONFIG_SCHED_DEBUG */
 static inline unsigned int get_sysctl_timer_migration(void)
 {
 	return 1;
 }
-#endif
+#endif /* CONFIG_SCHED_DEBUG */
 extern unsigned int sysctl_sched_rt_period;
 extern int sysctl_sched_rt_runtime;
 
kernel/sched/core.c +13 -3
index f7e7432..fda809d 100644
@@ -1533,6 +1533,15 @@ static void __sched_fork(struct task_struct *p)
 #ifdef CONFIG_PREEMPT_NOTIFIERS
 	INIT_HLIST_HEAD(&p->preempt_notifiers);
 #endif
+
+#ifdef CONFIG_SCHED_NUMA
+	if (p->mm && atomic_read(&p->mm->mm_users) == 1)
+		p->mm->numa_next_scan = jiffies;
+
+	p->node = -1;
+	p->node_last = -1;
+	p->node_stamp = 0ULL;
+#endif /* CONFIG_NUMA */
 }
 
 /*
@@ -5996,9 +6005,9 @@ static struct sched_domain_topology_level *sched_domain_topology = default_topol
  * Requeues a task ensuring its on the right load-balance list so
  * that it might get migrated to its new home.
  *
- * Note that we cannot actively migrate ourselves since our callers
- * can be from atomic context. We rely on the regular load-balance
- * mechanisms to move us around -- its all preference anyway.
+ * Since home-node is pure preference there's no hard migrate to force
+ * us anywhere, this also allows us to call this from atomic context if
+ * required.
  */
 void sched_setnode(struct task_struct *p, int node)
 {
@@ -6016,6 +6025,7 @@ void sched_setnode(struct task_struct *p, int node)
 		p->sched_class->put_prev_task(rq, p);
 
 	p->node = node;
+	p->node_last = node;
 
 	if (running)
 		p->sched_class->set_curr_task(rq);
kernel/sched/fair.c +168 -6

211 lines not on this page · raw message

kernel/sched/features.h +1 -0
index a5cc07b..fa6a0ac 100644
@@ -63,6 +63,7 @@ SCHED_FEAT(RT_RUNTIME_SHARE, true)
 SCHED_FEAT(LB_MIN, false)
 
 #ifdef CONFIG_SCHED_NUMA
+SCHED_FEAT(NUMA,           true)
 SCHED_FEAT(NUMA_HOT,       true)
 SCHED_FEAT(NUMA_BIAS,      true)
 SCHED_FEAT(NUMA_PULL,      true)
kernel/sched/sched.h +24 -9
index 74c37e6..bc5fc3e 100644
@@ -475,15 +475,6 @@ struct rq {
 #endif
 };
 
-static inline struct list_head *offnode_tasks(struct rq *rq)
-{
-#ifdef CONFIG_SCHED_NUMA
-	return &rq->offnode_tasks;
-#else
-	return NULL;
-#endif
-}
-
 static inline int cpu_of(struct rq *rq)
 {
 #ifdef CONFIG_SMP
@@ -501,6 +492,30 @@ DECLARE_PER_CPU(struct rq, runqueues);
 #define cpu_curr(cpu)		(cpu_rq(cpu)->curr)
 #define raw_rq()		(&__raw_get_cpu_var(runqueues))
 
+#ifdef CONFIG_SCHED_NUMA
+static inline bool offnode_task(struct task_struct *t)
+{
+	return t->node != -1 && t->node != cpu_to_node(task_cpu(t));
+}
+
+static inline struct list_head *offnode_tasks(struct rq *rq)
+{
+	return &rq->offnode_tasks;
+}
+
+void sched_setnode(struct task_struct *p, int node);
+#else /* CONFIG_SCHED_NUMA */
+static inline bool offnode_task(struct task_struct *t)
+{
+	return false;
+}
+
+static inline struct list_head *offnode_tasks(struct rq *rq)
+{
+	return NULL;
+}
+#endif /* CONFIG_SCHED_NUMA */
+
 #ifdef CONFIG_SMP
 
 #define rcu_dereference_check_sched_domain(p) \
kernel/sysctl.c +12 -1
index 81c7b1a..c08b95e 100644
@@ -297,6 +297,7 @@ static struct ctl_table kern_table[] = {
 		.extra1		= &min_wakeup_granularity_ns,
 		.extra2		= &max_wakeup_granularity_ns,
 	},
+#ifdef CONFIG_SMP
 	{
 		.procname	= "sched_tunable_scaling",
 		.data		= &sysctl_sched_tunable_scaling,
@@ -343,7 +344,17 @@ static struct ctl_table kern_table[] = {
 		.extra1		= &zero,
 		.extra2		= &one,
 	},
-#endif
+#endif /* CONFIG_SMP */
+#ifdef CONFIG_SCHED_NUMA
+	{
+		.procname	= "sched_numa_task_period_ms",
+		.data		= &sysctl_sched_numa_task_period,
+		.maxlen		= sizeof(unsigned int),
+		.mode		= 0644,
+		.proc_handler	= proc_dointvec,
+	},
+#endif /* CONFIG_SCHED_NUMA */
+#endif /* CONFIG_SCHED_DEBUG */
 	{
 		.procname	= "sched_rt_period_us",
 		.data		= &sysctl_sched_rt_period,