sched,topology: Update sched topology atomically
PATCH 1/1 sched,topology: Update sched topology atomically
When Peter Zijlstra reviewed commit 0fb3978b0aac ("sched/numa: Fix NUMA topology for systems with CPU-less nodes") [1], he pointed out that sched_domains_numa_distance and sched_domains_numa_masks are made separate RCU variables. That could go side-ways if there were a function using both, although there isn't for now.
So we update sched_domains_numa_distance and sched_domains_numa_masks and some other related sched topology parameters atomically to address the potential issues.
[1] https://lkml.kernel.org/r/[email protected]
Changelog:
v3:
- Fixed a bug to use sched_domain_topology_default as default and white space issue per comments from Valentin Schneider, Thanks!
v2:
- Addressed comments from Valentin Schneider, Thanks!
--- kernel/sched/fair.c | 33 ++--- kernel/sched/sched.h | 3 +- kernel/sched/topology.c | 261 ++++++++++++++++++++++------------------ 3 files changed, 161 insertions(+), 136 deletions(-)
kernel/sched/fair.c +17 -16
@@ -1294,17 +1294,17 @@ static unsigned long score_nearby_nodes(struct task_struct *p, int nid, int lim_dist, bool task) {unsigned long score =0 ; -int node, max_dist; +int node, max_dist, type; + +sched_fetch_numa_topology (&type, &max_dist);/* * All nodes are directly connected, and the same distance * from each other. No need for fancy placement algorithms. */ -if (sched_numa_topology_type ==NUMA_DIRECT ) +if (type ==NUMA_DIRECT )return 0 ; -/* sched_max_numa_distance may be changed in parallel. */ - max_dist =READ_ONCE (sched_max_numa_distance);/* * This code is called for each node, introducing N^2 complexity, * which should be ok given the number of nodes rarely exceeds 8.
@@ -1327,7 +1327,7 @@ static unsigned long score_nearby_nodes(struct task_struct *p, int nid,* "hoplimit", only nodes closer by than "hoplimit" are part * of each group. Skip other nodes. */ -if (sched_numa_topology_type ==NUMA_BACKPLANE && dist >= lim_dist) +if (type ==NUMA_BACKPLANE && dist >= lim_dist)continue ;/* Add up the faults from nearby nodes. */
@@ -1344,7 +1344,7 @@ static unsigned long score_nearby_nodes(struct task_struct *p, int nid,* The further away a node is, the less the faults count. * This seems to result in good task placement. */ -if (sched_numa_topology_type ==NUMA_GLUELESS_MESH ) { +if (type ==NUMA_GLUELESS_MESH ) { faults *= (max_dist - dist); faults /= (max_dist -LOCAL_DISTANCE ); }
@@ -1971,7 +1971,9 @@ static int task_numa_migrate(struct task_struct *p)long taskimp, groupimp;struct numa_group *ng;struct rq *best_rq; -int nid, ret, dist; +int nid, ret, dist, type; + +sched_fetch_numa_topology (&type,NULL );/* * Pick the lowest SD_NUMA domain, as that would have the smallest
@@ -2026,8 +2028,7 @@ static int task_numa_migrate(struct task_struct *p)continue ; dist =node_distance (env.src_nid, env.dst_nid); -if (sched_numa_topology_type ==NUMA_BACKPLANE && - dist != env.dist) { +if (type ==NUMA_BACKPLANE && dist != env.dist) { taskweight =task_weight (p, env.src_nid, dist); groupweight =group_weight (p, env.src_nid, dist); }
@@ -2262,10 +2263,12 @@ static u64 numa_get_avg_runtime(struct task_struct *p, u64 *period)static int preferred_group_nid (struct task_struct *p,int nid) {nodemask_t nodes; -int dist; +int dist, max_dist, type; + +sched_fetch_numa_topology (&type, &max_dist);/* Direct connections between all NUMA nodes. */ -if (sched_numa_topology_type ==NUMA_DIRECT ) +if (type ==NUMA_DIRECT )return nid;/*
@@ -2273,14 +2276,12 @@ static int preferred_group_nid(struct task_struct *p, int nid)* scores nodes according to the number of NUMA hinting faults on * both the node itself, and on nearby nodes. */ -if (sched_numa_topology_type ==NUMA_GLUELESS_MESH ) { +if (type ==NUMA_GLUELESS_MESH ) {unsigned long score, max_score =0 ;int node, max_node = nid; - dist = sched_max_numa_distance; -for_each_node_state (node,N_CPU ) { - score =group_weight (p, node, dist); + score =group_weight (p, node, max_dist);if (score > max_score) { max_score = score; max_node = node;
@@ -2299,7 +2300,7 @@ static int preferred_group_nid(struct task_struct *p, int nid)* keep the complexity of the search down. */ nodes = node_states[N_CPU ]; -for (dist = sched_max_numa_distance; dist >LOCAL_DISTANCE ; dist--) { +for (dist = max_dist; dist >LOCAL_DISTANCE ; dist--) {unsigned long max_faults =0 ;nodemask_t max_group =NODE_MASK_NONE ;int a, b;
kernel/sched/sched.h +1 -2
@@ -1638,9 +1638,8 @@ enum numa_topology_type {NUMA_GLUELESS_MESH , NUMA_BACKPLANE, }; -extern enum numa_topology_type sched_numa_topology_type; -extern int sched_max_numa_distance;extern bool find_numa_distance (int distance); +extern void sched_fetch_numa_topology (int *type,int *max_distance);extern void sched_init_numa (int offline_node);extern void sched_update_numa (int cpu,bool online);extern void sched_domains_numa_masks_set (unsigned int cpu);
kernel/sched/topology.c +143 -118
463 lines not on this page · raw message
Signature · 1 line
2.30.2