Skip to contentSkip to conversation

sched,topology: Update sched topology atomically

PATCH In review Huang Ying 2 participants 4 messages Apr 6, 2022 – Apr 11, 2022 Scheduler

lore
Share & tools
lore

Huang Ying

author
#0 lore raw

PATCH 1/1 sched,topology: Update sched topology atomically

When Peter Zijlstra reviewed commit 0fb3978b0aac ("sched/numa: Fix NUMA topology for systems with CPU-less nodes") [1], he pointed out that sched_domains_numa_distance and sched_domains_numa_masks are made separate RCU variables. That could go side-ways if there were a function using both, although there isn't for now.

So we update sched_domains_numa_distance and sched_domains_numa_masks and some other related sched topology parameters atomically to address the potential issues.

[1] https://lkml.kernel.org/r/[email protected]

Cc Valentin Schneider <[email protected]>
Cc Ingo Molnar <[email protected]>
Cc Mel Gorman <[email protected]>
Cc Rik van Riel <[email protected]>
Cc Srikar Dronamraju <[email protected]>
---
 kernel/sched/fair.c     |  47 ++++++---
 kernel/sched/sched.h    |  14 ++-
 kernel/sched/topology.c | 223 +++++++++++++++++++---------------------
 3 files changed, 151 insertions(+), 133 deletions(-)
kernel/sched/fair.c +31 -16
index d4bd299d67ab..546e7347fafc 100644
@@ -1294,17 +1294,22 @@ static unsigned long score_nearby_nodes(struct task_struct *p, int nid,
 					int lim_dist, bool task)
 {
 	unsigned long score = 0;
-	int node, max_dist;
+	int node, max_dist, type;
+	struct sched_numa_topology *topology;
+
+	rcu_read_lock();
+	topology = rcu_dereference(sched_numa_topology);
+	type = topology->type;
+	max_dist = topology->max_distance;
+	rcu_read_unlock();
 
 	/*
 	 * All nodes are directly connected, and the same distance
 	 * from each other. No need for fancy placement algorithms.
 	 */
-	if (sched_numa_topology_type == NUMA_DIRECT)
+	if (type == NUMA_DIRECT)
 		return 0;
 
-	/* sched_max_numa_distance may be changed in parallel. */
-	max_dist = READ_ONCE(sched_max_numa_distance);
 	/*
 	 * This code is called for each node, introducing N^2 complexity,
 	 * which should be ok given the number of nodes rarely exceeds 8.
@@ -1327,7 +1332,7 @@ static unsigned long score_nearby_nodes(struct task_struct *p, int nid,
 		 * "hoplimit", only nodes closer by than "hoplimit" are part
 		 * of each group. Skip other nodes.
 		 */
-		if (sched_numa_topology_type == NUMA_BACKPLANE && dist >= lim_dist)
+		if (type == NUMA_BACKPLANE && dist >= lim_dist)
 			continue;
 
 		/* Add up the faults from nearby nodes. */
@@ -1344,7 +1349,7 @@ static unsigned long score_nearby_nodes(struct task_struct *p, int nid,
 		 * The further away a node is, the less the faults count.
 		 * This seems to result in good task placement.
 		 */
-		if (sched_numa_topology_type == NUMA_GLUELESS_MESH) {
+		if (type == NUMA_GLUELESS_MESH) {
 			faults *= (max_dist - dist);
 			faults /= (max_dist - LOCAL_DISTANCE);
 		}
@@ -1971,7 +1976,13 @@ static int task_numa_migrate(struct task_struct *p)
 	long taskimp, groupimp;
 	struct numa_group *ng;
 	struct rq *best_rq;
-	int nid, ret, dist;
+	int nid, ret, dist, type;
+	struct sched_numa_topology *topology;
+
+	rcu_read_lock();
+	topology = rcu_dereference(sched_numa_topology);
+	type = topology->type;
+	rcu_read_unlock();
 
 	/*
 	 * Pick the lowest SD_NUMA domain, as that would have the smallest
@@ -2026,8 +2037,7 @@ static int task_numa_migrate(struct task_struct *p)
 				continue;
 
 			dist = node_distance(env.src_nid, env.dst_nid);
-			if (sched_numa_topology_type == NUMA_BACKPLANE &&
-						dist != env.dist) {
+			if (type == NUMA_BACKPLANE && dist != env.dist) {
 				taskweight = task_weight(p, env.src_nid, dist);
 				groupweight = group_weight(p, env.src_nid, dist);
 			}
@@ -2262,10 +2272,17 @@ static u64 numa_get_avg_runtime(struct task_struct *p, u64 *period)
 static int preferred_group_nid(struct task_struct *p, int nid)
 {
 	nodemask_t nodes;
-	int dist;
+	int dist, max_dist, type;
+	struct sched_numa_topology *topology;
+
+	rcu_read_lock();
+	topology = rcu_dereference(sched_numa_topology);
+	type = topology->type;
+	max_dist = topology->max_distance;
+	rcu_read_unlock();
 
 	/* Direct connections between all NUMA nodes. */
-	if (sched_numa_topology_type == NUMA_DIRECT)
+	if (type == NUMA_DIRECT)
 		return nid;
 
 	/*
@@ -2273,14 +2290,12 @@ static int preferred_group_nid(struct task_struct *p, int nid)
 	 * scores nodes according to the number of NUMA hinting faults on
 	 * both the node itself, and on nearby nodes.
 	 */
-	if (sched_numa_topology_type == NUMA_GLUELESS_MESH) {
+	if (type == NUMA_GLUELESS_MESH) {
 		unsigned long score, max_score = 0;
 		int node, max_node = nid;
 
-		dist = sched_max_numa_distance;
-
 		for_each_node_state(node, N_CPU) {
-			score = group_weight(p, node, dist);
+			score = group_weight(p, node, max_dist);
 			if (score > max_score) {
 				max_score = score;
 				max_node = node;
@@ -2299,7 +2314,7 @@ static int preferred_group_nid(struct task_struct *p, int nid)
 	 * keep the complexity of the search down.
 	 */
 	nodes = node_states[N_CPU];
-	for (dist = sched_max_numa_distance; dist > LOCAL_DISTANCE; dist--) {
+	for (dist = max_dist; dist > LOCAL_DISTANCE; dist--) {
 		unsigned long max_faults = 0;
 		nodemask_t max_group = NODE_MASK_NONE;
 		int a, b;
kernel/sched/sched.h +12 -2
index 58263f90c559..21a7aea01286 100644
@@ -1644,8 +1644,18 @@ enum numa_topology_type {
 	NUMA_GLUELESS_MESH,
 	NUMA_BACKPLANE,
 };
-extern enum numa_topology_type sched_numa_topology_type;
-extern int sched_max_numa_distance;
+
+struct sched_numa_topology {
+	enum numa_topology_type	type;
+	int			nr_levels;
+	int			max_distance;
+	int			*distances;
+	struct cpumask		***masks;
+	struct rcu_head		rcu;
+};
+
+extern struct sched_numa_topology *sched_numa_topology;
+
 extern bool find_numa_distance(int distance);
 extern void sched_init_numa(int offline_node);
 extern void sched_update_numa(int cpu, bool online);
kernel/sched/topology.c +108 -115

412 lines not on this page · raw message

Signature · 1 line
2.30.2
#1 lore raw

Hi,

Quoted 6 lines · Huang Ying

When Peter Zijlstra reviewed commit 0fb3978b0aac ("sched/numa: Fix NUMA topology for systems with CPU-less nodes") [1], he pointed out that sched_domains_numa_distance and sched_domains_numa_masks are made separate RCU variables. That could go side-ways if there were a function using both, although there isn't for now.

Quoted 6 lines

So we update sched_domains_numa_distance and sched_domains_numa_masks and some other related sched topology parameters atomically to address the potential issues.

[1] https://lkml.kernel.org/r/[email protected]

I had to re-acquaint myself with [1], but AFAICT this looks mostly OK. Have some nits/comments below, I haven't actually tested this yet as I'm still setting stuff up over here...

Quoted 21 lines
Cc Valentin Schneider <[email protected]>
Cc Ingo Molnar <[email protected]>
Cc Mel Gorman <[email protected]>
Cc Rik van Riel <[email protected]>
Cc Srikar Dronamraju <[email protected]>
---
 kernel/sched/fair.c     |  47 ++++++---
 kernel/sched/sched.h    |  14 ++-
 kernel/sched/topology.c | 223 +++++++++++++++++++---------------------
 3 files changed, 151 insertions(+), 133 deletions(-)
kernel/sched/fair.c +0 -0
index d4bd299d67ab..546e7347fafc 100644
@@ -1294,17 +1294,22 @@ static unsigned long score_nearby_nodes(struct task_struct *p, int nid,
                                      int lim_dist, bool task)
 {
      unsigned long score = 0;

kernel/sched/fair.c static unsigned long score_nearby_nodes(struct task_struct *p, int nid, · line 1305

Quoted 10 lines
kernel/sched/fair.c +8 -1
-	int node, max_dist;
+	int node, max_dist, type;
+	struct sched_numa_topology *topology;
+
+	rcu_read_lock();
+	topology = rcu_dereference(sched_numa_topology);
+	type = topology->type;
+	max_dist = topology->max_distance;
+	rcu_read_unlock();
 

One thing worth pointing out, the NUMA topology type *wasn't* RCU protected previously, which I think we've been wanting since

0083242c9375 ("sched/topology: Skip updating masks for non-online nodes")

(before then the topology was init'd once-and-for-all).

Another thing; this pattern is now repeated a handful of times in fair.c, what about bundling it up? As a bonus it lets us keep the definition of struct sched_numa_topology internal to topology.c:

void sched_fetch_numa_topology(int *type int *max_distance)

{
        struct sched_numa_topology __rcu *topology;

        rcu_read_lock();
        topology = rcu_dereference(sched_numa_topology);

        if (type)
                *type = topology->type;

        if (max_distance)
                *max_distance = topology->max_distance;


        rcu_read_unlock();
}

kernel/sched/topology.c static void claim_allocations(int cpu, struct sched_domain *sd) · line 1486

Quoted 10 lines
kernel/sched/topology.c +1 -1
index 810750e62118..19523b23034f 100644
@@ -1483,14 +1483,10 @@ static void claim_allocations(int cpu, struct sched_domain *sd)
 }
 
 #ifdef CONFIG_NUMA
-enum numa_topology_type sched_numa_topology_type;
+struct sched_numa_topology	default_numa_topology;

Shouldn't this be static?

kernel/sched/topology.c static void claim_allocations(int cpu, struct sched_domain *sd) · line 1487

Quoted 1 line
kernel/sched/topology.c +1 -0
+struct sched_numa_topology	*sched_numa_topology = &default_numa_topology;

This should also be __rcu (sparse should complain about this)

Quoted 14 lines
kernel/sched/topology.c +3 -5
-static int			sched_domains_numa_levels;
 static int			sched_domains_curr_level;
-
-int				sched_max_numa_distance;
-static int			*sched_domains_numa_distance;
-static struct cpumask		***sched_domains_numa_masks;
 #endif
 
 /*
 
+static void sched_numa_topology_switch(struct sched_numa_topology *topology,
+				       struct sched_domain_topology_level *tl)
+{

kernel/sched/topology.c static void claim_allocations(int cpu, struct sched_domain *sd)

Quoted 10 lines
kernel/sched/topology.c +10 -0
+	struct sched_numa_topology *old_topology = sched_numa_topology;
+	struct sched_domain_topology_level *old_tl = sched_domain_topology;
+
+	rcu_assign_pointer(sched_numa_topology, topology);
+	sched_domain_topology = tl;
+
+	if (old_topology == &default_numa_topology)
+		return;
+
+	init_rcu_head(&old_topology->rcu);

I'm not familiar with this and I can find only a handful of users, should we be doing that for struct sched_domain?

Quoted 16 lines
kernel/sched/topology.c +5 -3
+	call_rcu(&old_topology->rcu, sched_numa_topology_free_rcu);
+	kfree(old_tl);
+}
 
 #define NR_DISTANCE_VALUES (1 << DISTANCE_BITS)
 
 void sched_init_numa(int offline_node)
 {
      struct sched_domain_topology_level *tl;
-	unsigned long *distance_map;
+	unsigned long *distance_map = NULL;
      int nr_levels = 0;
      int i, j;
-	int *distances;
-	struct cpumask ***masks;
+	struct sched_numa_topology *topology;

kernel/sched/topology.c static void init_numa_topology_type(int offline_node)

Quoted 10 lines
kernel/sched/topology.c +6 -0
+
+	topology = kzalloc(sizeof(*topology), GFP_KERNEL);
+	if (!topology)
+		goto err;
 
      /*
       * O(nr_nodes^2) deduplicating selection sort -- in order to find the
 
+	return;
+err:

I was expecting the "usual" multi-label setup, but this seems to be working too - though not in the most obvious way ;)

Quoted 21 lines
kernel/sched/topology.c +5 -2
+	bitmap_free(distance_map);
+	sched_numa_topology_free(topology);
+	sched_numa_topology_switch(&default_numa_topology,
+				   sched_domain_topology_default);
 }
 
 /*
@@ -1981,7 +1977,6 @@ void sched_update_numa(int cpu, bool online)
      if (cpumask_weight(cpumask_of_node(node)) != 1)
              return;
 
-	sched_reset_numa();
      sched_init_numa(online ? NUMA_NO_NODE : node);
 }
 
@@ -1990,14 +1985,15 @@ void sched_domains_numa_masks_set(unsigned int cpu)
      int node = cpu_to_node(cpu);
      int i, j;
 
-	for (i = 0; i < sched_domains_numa_levels; i++) {
+	for (i = 0; i < sched_numa_topology->nr_levels; i++) {

kernel/sched/topology.c void sched_domains_numa_masks_set(unsigned int cpu) · line 1996

Quoted 10 lines
kernel/sched/topology.c +3 -2
              for (j = 0; j < nr_node_ids; j++) {
                      if (!node_state(j, N_CPU))
                              continue;
 
                      /* Set ourselves in the remote node's masks */
-			if (node_distance(j, node) <= sched_domains_numa_distance[i])
-				cpumask_set_cpu(cpu, sched_domains_numa_masks[i][j]);
+			if (node_distance(j, node) <=
+			    sched_numa_topology->distances[i])
+				cpumask_set_cpu(cpu, sched_numa_topology->masks[i][j]);

I'm guessing the assumption here is that this is done in sched_cpu_{de,}activate(), so we have the hotplug lock and so the masks are stable... Should we sprinkle in a lockdep_assert_cpus_held() for good measure?

Having a second look at this, I think the same applies to your change to sd_numa_mask() - those end up used in the domain rebuild which itself might be deferred cpuset_hotplug_work, but that should still hold the hotplug lock...

Quoted 23 lines
kernel/sched/topology.c +4 -4
              }
      }
 }
@@ -2006,10 +2002,10 @@ void sched_domains_numa_masks_clear(unsigned int cpu)
 {
      int i, j;
 
-	for (i = 0; i < sched_domains_numa_levels; i++) {
+	for (i = 0; i < sched_numa_topology->nr_levels; i++) {
              for (j = 0; j < nr_node_ids; j++) {
-			if (sched_domains_numa_masks[i][j])
-				cpumask_clear_cpu(cpu, sched_domains_numa_masks[i][j]);
+			if (sched_numa_topology->masks[i][j])
+				cpumask_clear_cpu(cpu, sched_numa_topology->masks[i][j]);
              }
      }
 }
@@ -2025,22 +2021,19 @@ void sched_domains_numa_masks_clear(unsigned int cpu)
 int sched_numa_find_closest(const struct cpumask *cpus, int cpu)
 {
      int i, j = cpu_to_node(cpu), found = nr_cpu_ids;
-	struct cpumask ***masks;
+	struct sched_numa_topology *topology;

kernel/sched/topology.c void sched_domains_numa_masks_clear(unsigned int cpu) · line 2029

Quoted 10 lines
kernel/sched/topology.c +3 -5
 
      rcu_read_lock();
-	masks = rcu_dereference(sched_domains_numa_masks);
-	if (!masks)
-		goto unlock;
-	for (i = 0; i < sched_domains_numa_levels; i++) {
-		if (!masks[i][j])
+	topology = rcu_dereference(sched_numa_topology);
+	for (i = 0; i < topology->nr_levels; i++) {
+		if (!topology->masks[i][j])

If I got default_numa_topology right and it's meant to be static, sched_numa_topology->masks can still be NULL, but here this would be gated by nr_levels being 0... Did I get that right?

Quoted 14 lines
kernel/sched/topology.c +1 -3
                      break;
-		cpu = cpumask_any_and(cpus, masks[i][j]);
+		cpu = cpumask_any_and(cpus, topology->masks[i][j]);
              if (cpu < nr_cpu_ids) {
                      found = cpu;
                      break;
              }
      }
-unlock:
      rcu_read_unlock();
 
      return found;
--

2.30.2

#2 lore raw
Quoted 8 lines · Valentin Schneider

kernel/sched/topology.c void sched_domains_numa_masks_clear(unsigned int cpu) · line 2029

Quoted 3 lines
kernel/sched/topology.c +3 -0
+	topology = rcu_dereference(sched_numa_topology);
+	for (i = 0; i < topology->nr_levels; i++) {
+		if (!topology->masks[i][j])

If I got default_numa_topology right and it's meant to be static, sched_numa_topology->masks can still be NULL, but here this would be gated by nr_levels being 0... Did I get that right?

Being a file-global variable, that's already init'd-as-0, (I'll use my longish break as an excuse for forgetting that one), but the rest still applies.

Quoted 14 lines
Quoted 14 lines
                      break;
kernel/sched/topology.c +1 -3
-		cpu = cpumask_any_and(cpus, masks[i][j]);
+		cpu = cpumask_any_and(cpus, topology->masks[i][j]);
              if (cpu < nr_cpu_ids) {
                      found = cpu;
                      break;
              }
      }
-unlock:
      rcu_read_unlock();
 
      return found;
--

2.30.2

#3 lore raw
In reply to Valentin Schneider

Hi, Valentin,

Thanks a lot for review!

Quoted 76 lines · Valentin Schneider

Hi,

Quoted 6 lines · Huang Ying

When Peter Zijlstra reviewed commit 0fb3978b0aac ("sched/numa: Fix NUMA topology for systems with CPU-less nodes") [1], he pointed out that sched_domains_numa_distance and sched_domains_numa_masks are made separate RCU variables.  That could go side-ways if there were a function using both, although there isn't for now.

Quoted 7 lines

So we update sched_domains_numa_distance and sched_domains_numa_masks and some other related sched topology parameters atomically to address the potential issues.

[1] https://lkml.kernel.org/r/20220214121553.582248-1-
[email protected]

I had to re-acquaint myself with [1], but AFAICT this looks mostly OK. Have some nits/comments below, I haven't actually tested this yet as I'm still setting stuff up over here...

Quoted 22 lines
Cc Valentin Schneider <[email protected]>
Cc Ingo Molnar <[email protected]>
Cc Mel Gorman <[email protected]>
Cc Rik van Riel <[email protected]>
Cc Srikar Dronamraju <[email protected]>

---
 kernel/sched/fair.c     |  47 ++++++--- kernel/sched/sched.h    |  14 ++-
 kernel/sched/topology.c | 223 +++++++++++++++++++---------------------

 3 files changed, 151 insertions(+), 133 deletions(-)
kernel/sched/fair.c +0 -0
index d4bd299d67ab..546e7347fafc 100644
@@ -1294,17 +1294,22 @@ static unsigned long score_nearby_nodes(struct
 task_struct *p, int nid,
                                       int lim_dist, bool task)
  {
       unsigned long score = 0;

kernel/sched/fair.c static unsigned long score_nearby_nodes(struct · line 1306

Quoted 10 lines
kernel/sched/fair.c +8 -1
-       int node, max_dist;
+       int node, max_dist, type;
+       struct sched_numa_topology *topology;
+
+       rcu_read_lock();
+       topology = rcu_dereference(sched_numa_topology);
+       type = topology->type;
+       max_dist = topology->max_distance;
+       rcu_read_unlock();
 

One thing worth pointing out, the NUMA topology type *wasn't* RCU protected previously, which I think we've been wanting since

  0083242c9375 ("sched/topology: Skip updating masks for non-online nodes")

(before then the topology was init'd once-and-for-all).

Another thing; this pattern is now repeated a handful of times in fair.c, what about bundling it up? As a bonus it lets us keep the definition of struct sched_numa_topology internal to topology.c:

void sched_fetch_numa_topology(int *type int *max_distance) {
        struct sched_numa_topology __rcu *topology;

        rcu_read_lock();
        topology = rcu_dereference(sched_numa_topology);

        if (type)
                *type = topology->type;

Quoted 7 lines

        if (max_distance)
                *max_distance = topology->max_distance;

        rcu_read_unlock();
}

This looks good. Will use it in the next version. Thanks!

Quoted 8 lines
Quoted 8 lines
kernel/sched/topology.c +0 -0
index 810750e62118..19523b23034f 100644
@@ -1483,14 +1483,10 @@ static void claim_allocations(int cpu, struct
 sched_domain *sd)
  }
 
Quoted 5 lines

kernel/sched/topology.c static void claim_allocations(int cpu, struct sched_domain *sd) · line 1486

Quoted 3 lines
kernel/sched/topology.c +1 -1
  #ifdef CONFIG_NUMA
-enum numa_topology_type sched_numa_topology_type;
+struct sched_numa_topology     default_numa_topology;

Shouldn't this be static?

Yes. Will change it in the next version.

Quoted 4 lines

kernel/sched/topology.c static void claim_allocations(int cpu, struct · line 1489

Quoted 2 lines
kernel/sched/topology.c +1 -0
+struct sched_numa_topology     *sched_numa_topology =
 &default_numa_topology;

This should also be __rcu (sparse should complain about this)

Yes. Will change it in the next version.

Quoted 22 lines
Quoted 22 lines
kernel/sched/topology.c +0 -5
-static int                     sched_domains_numa_levels;
  static int                     sched_domains_curr_level;
-
-int                            sched_max_numa_distance;
-static int                     *sched_domains_numa_distance;
-static struct cpumask          ***sched_domains_numa_masks;
  #endif
 

 /*

+static void sched_numa_topology_switch(struct sched_numa_topology *topology,
+                                      struct sched_domain_topology_level *tl)
+{
+       struct sched_numa_topology *old_topology = sched_numa_topology; +       struct sched_domain_topology_level *old_tl = sched_domain_topology;
+
+       rcu_assign_pointer(sched_numa_topology, topology);
+       sched_domain_topology = tl;

Quoted 8 lines

kernel/sched/topology.c static void init_numa_topology_type(int offline_node) · line 1816

Quoted 5 lines
kernel/sched/topology.c +5 -0
+
+       if (old_topology == &default_numa_topology)
+               return;
+
+       init_rcu_head(&old_topology->rcu);

I'm not familiar with this and I can find only a handful of users, should we be doing that for struct sched_domain?

Per my understanding, this is for debug and we should call it before call_rcu(). Copying Paul for confirmation.

Quoted 26 lines
Quoted 25 lines

+       call_rcu(&old_topology->rcu, sched_numa_topology_free_rcu); +       kfree(old_tl);
+}

 #define NR_DISTANCE_VALUES (1 << DISTANCE_BITS)

 void sched_init_numa(int offline_node)
 {
      struct sched_domain_topology_level *tl;
-       unsigned long *distance_map;
+       unsigned long *distance_map = NULL;
      int nr_levels = 0;
      int i, j;
-       int *distances;
-       struct cpumask ***masks;
+       struct sched_numa_topology *topology;
+
+       topology = kzalloc(sizeof(*topology), GFP_KERNEL); +       if (!topology)
+               goto err;

      /*
       * O(nr_nodes^2) deduplicating selection sort -- in order to find the

Quoted 5 lines
Quoted 2 lines

+       return;
+err:

I was expecting the "usual" multi-label setup, but this seems to be working too - though not in the most obvious way ;)

OK. Will try to convert it to multi-label.

Quoted 42 lines
Quoted 35 lines

+       bitmap_free(distance_map);
+       sched_numa_topology_free(topology);
+       sched_numa_topology_switch(&default_numa_topology, +                                  sched_domain_topology_default); }

 /*

kernel/sched/topology.c +4 -4
@@ -1981,7 +1977,6 @@ void sched_update_numa(int cpu, bool online)
       if (cpumask_weight(cpumask_of_node(node)) != 1)
               return;
 
-       sched_reset_numa();
       sched_init_numa(online ? NUMA_NO_NODE : node);
  }
 
@@ -1990,14 +1985,15 @@ void sched_domains_numa_masks_set(unsigned int
 cpu)
       int node = cpu_to_node(cpu);
       int i, j;
 
-       for (i = 0; i < sched_domains_numa_levels; i++) {
+       for (i = 0; i < sched_numa_topology->nr_levels; i++) {
               for (j = 0; j < nr_node_ids; j++) {
                       if (!node_state(j, N_CPU))
                               continue;
 
                       /* Set ourselves in the remote node's masks */
-                       if (node_distance(j, node) <=
 sched_domains_numa_distance[i])
-                               cpumask_set_cpu(cpu,
 sched_domains_numa_masks[i][j]);
+                       if (node_distance(j, node) <=
+                           sched_numa_topology->distances[i])
+                               cpumask_set_cpu(cpu, sched_numa_topology-
Quoted 1 line
Quoted 1 line

masks[i][j]);

will be 0
I'm guessing the assumption here is that this is done in sched_cpu_{de,}activate(), so we have the hotplug lock and so the masks are stable... Should we sprinkle in a lockdep_assert_cpus_held() for good measure?

Quoted 4 lines

Having a second look at this, I think the same applies to your change to sd_numa_mask() - those end up used in the domain rebuild which itself might be deferred cpuset_hotplug_work, but that should still hold the hotplug lock...

Yes. hotplug lock needs to be held in these places. Will add lockdep_assert_cpus_held() in necessary places.

Quoted 28 lines
Quoted 28 lines

              }
      }
 }

kernel/sched/topology.c +3 -3
@@ -2006,10 +2002,10 @@ void sched_domains_numa_masks_clear(unsigned int
 cpu)
  {
       int i, j;
 
-       for (i = 0; i < sched_domains_numa_levels; i++) {
+       for (i = 0; i < sched_numa_topology->nr_levels; i++) {
               for (j = 0; j < nr_node_ids; j++) {
-                       if (sched_domains_numa_masks[i][j])
-                               cpumask_clear_cpu(cpu,
 sched_domains_numa_masks[i][j]);
+                       if (sched_numa_topology->masks[i][j])
+                               cpumask_clear_cpu(cpu,
 sched_numa_topology->masks[i][j]);

              }
      }
 }

kernel/sched/topology.c +1 -1
@@ -2025,22 +2021,19 @@ void sched_domains_numa_masks_clear(unsigned int
 cpu)
  int sched_numa_find_closest(const struct cpumask *cpus, int cpu)
  {
       int i, j = cpu_to_node(cpu), found = nr_cpu_ids;
-       struct cpumask ***masks;
+       struct sched_numa_topology *topology;
 
Quoted 13 lines

kernel/sched/topology.c void sched_domains_numa_masks_clear(unsigned int cpu) · line 2029

Quoted 9 lines
kernel/sched/topology.c +3 -5
       rcu_read_lock();
-       masks = rcu_dereference(sched_domains_numa_masks);
-       if (!masks)
-               goto unlock;
-       for (i = 0; i < sched_domains_numa_levels; i++) {
-               if (!masks[i][j])
+       topology = rcu_dereference(sched_numa_topology);
+       for (i = 0; i < topology->nr_levels; i++) {
+               if (!topology->masks[i][j])

If I got default_numa_topology right and it's meant to be static, sched_numa_topology->masks can still be NULL, but here this would be gated by nr_levels being 0... Did I get that right?

Yes. nr_levels in default_numa_topology is 0.

Quoted 12 lines
Quoted 12 lines
kernel/sched/topology.c +1 -2
                       break;
-               cpu = cpumask_any_and(cpus, masks[i][j]);
+               cpu = cpumask_any_and(cpus, topology->masks[i][j]);
               if (cpu < nr_cpu_ids) {
                       found = cpu;
                       break;
               }
       }
-unlock:
       rcu_read_unlock();
 
Quoted 4 lines
Quoted 3 lines

      return found;
--
2.30.2

Best Regards,
Huang, Ying