Skip to contentSkip to conversation

non-contiguous NUMA nodes handling in numactl and libnuma

PATCH In review Amit K. Arora 2 participants 6 messages Mar 2, 2009 – Mar 9, 2009

lore
Share & tools
lore
#0 lore raw

cover non-contiguous NUMA nodes handling in numactl and libnuma

PROBLEM: If the NUMA nodes in a system are not contiguous, numactl shows incorrect output. Here is an example from one such system:

------------------------------------------------------

# ls /sys/devices/system/node
has_cpu  has_normal_memory  node0  node1  node4  node5  online  possible
# numactl --hardware
available: 6 nodes (0-5)
node 0 cpus: 0 1 2 3
node 0 size: 15232 MB
node 0 free: 2265 MB
node 1 cpus: 4 5 6 7
node 1 size: 16256 MB
node 1 free: 3108 MB
libnuma: Warning: /sys not mounted or invalid. Assuming one node: No
such file or directory
node 2 cpus:
node 2 size: <not available>
node 2 free: <not available>
node 3 cpus:
node 3 size: <not available>
node 3 free: <not available>
node 4 cpus: 8 9 10 11
node 4 size: 16000 MB
node 4 free: 9292 MB
node 5 cpus: 12 13 14 15
node 5 size: 16128 MB
node 5 free: 9479 MB
node distances:
node   0   1   2   3   4   5 
  0:  10  20  20  20  20  10 
  1:  20  10  20  20   0   0 
  2:   0   0   0   0   0   0 
  3:   0   0   0   0   0   0 
  4:   0   0   0  1112145  1024  2607344 
  5:  1024  2607344  1024  2607344   0   0 
------------------------------------------------------

As we see above, there are three problems with above output:
o it is showing wrong number of available nodes Actually there are 4 nodes available, but it says "6".
o it tries to access information for non-exsistent nodes from /sys Thus displaying a wrong warning message too
o the NUMA distances it shows above are clearly incorrect.

Following two patches fixes these problems:
1) numactl-handle-NUMA-dist-properly.patch
2) numactl-ignore-nonavailable-nodes.patch

--
Regards,
Amit Arora

#1 lore raw

PATCH 1/2 handle NUMA distances properly for non-contiguous nodes

Patch: numactl-handle-NUMA-dist-properly.patch

This patch fixes the problem of displaying incorrect NUMA distances. It increases the table size to accomodate node distance information for nodes from 0 to nodemax. parse_numbers() makes sure not to add distances for nodes which are not existing.

distance.c +12 -18
diff -Nuarp numactl-2.0.2.ORG/distance.c numactl-2.0.2/distance.c
@@ -26,25 +26,16 @@
 static int distance_numnodes;
 static int *distance_table; 
 
-static int count_numbers(char *s)
-{
-	int n = 0;
-	for (;;) {
-		char *end;  
-		strtoul(s, &end, 0);
-		if (s == end)
-			return n;
-		s = end;
-		n++; 
-	}
-}
-
 static void parse_numbers(char *s, int *iptr, int n)
 {
-	int i;
+	int i, d, j;
 	char *end;
-	for (i = 0; i < n; i++) { 
-		*iptr++ = strtoul(s, &end, 0);
+	for (i = 0, j = 0; i < n; i++, j++) { 
+		d = strtoul(s, &end, 0);
+		/* Skip unavailable nodes */
+		while (j<n &&  !numa_bitmask_isbitset(numa_all_nodes_ptr, j))
+			j++;
+		*(iptr+j) = d;
 		if (s == end)
 			break;
 		s = end; 
@@ -68,7 +59,10 @@ static int read_distance_table(void)
 		if (!dfh) { 
 			if (errno == ENOENT && nd > 0)
 				err = 0;
-			break;
+			if (!err && nd<numa_num_configured_nodes())
+				continue;
+			else
+				break;
 		}
 		len = getdelim(&line, &linelen, '\n', dfh);
 		fclose(dfh);
@@ -76,7 +70,7 @@ static int read_distance_table(void)
 			break;
 
 		if (!table) { 
-			numnodes = count_numbers(line); 
+			numnodes = numa_num_configured_nodes();
 			table = calloc(numnodes * numnodes, sizeof(int)); 
 			if (!table) {
 				errno = ENOMEM; 

--
Regards,
Amit Arora

#2 lore raw

PATCH 2/2 ignore unavailable NUMA nodes

In reply to Amit K. Arora

Patch: numactl-ignore-nonavailable-nodes.patch

This patch fixes the problem of displaying wrong number of "available" nodes. It also ignores non-existent nodes and thus avoids the wrong warning message also.

libnuma.c +8 -0
diff -Nuarp numactl-2.0.3-rc2.ORG/libnuma.c numactl-2.0.3-rc2/libnuma.c
@@ -54,6 +54,8 @@ struct bitmask *numa_all_cpus_ptr = NULL
 static unsigned long *node_cpu_mask_v1[NUMA_NUM_NODES];
 struct bitmask **node_cpu_mask_v2;
 
+char *nodes_allowed_list = NULL;
+
 WEAK void numa_error(char *where);
 
 #ifdef __thread
@@ -452,6 +454,12 @@ set_thread_constraints(void)
 			maxprocnode =
 				read_mask(buffer + 15, numa_all_nodes_ptr);
 		}
+		if (strncmp(buffer,"Mems_allowed_list:",18) == 0) {
+			nodes_allowed_list = malloc(strlen(buffer)-18);
+			strncpy(nodes_allowed_list, buffer + 19,
+				strlen(buffer) - 19);
+			nodes_allowed_list[strlen(nodes_allowed_list)-1] = '\0';
+		}
 	}
 	fclose(f);
 	free(buffer);
numactl.c +13 -2
diff -Nuarp numactl-2.0.3-rc2.ORG/numactl.c numactl-2.0.3-rc2/numactl.c
@@ -214,13 +214,24 @@ void print_node_cpus(int node)
 
 void hardware(void)
 { 
-	int i;
+	int i, numconfigurednodes=0;
 	int maxnode = numa_num_configured_nodes()-1;
-	printf("available: %d nodes (0-%d)\n", 1+maxnode, maxnode); 	
+
+	for (i = 0; i<=maxnode; i++)
+		if (numa_bitmask_isbitset(numa_all_nodes_ptr, i))
+			numconfigurednodes++;
+	if (nodes_allowed_list)
+		printf("available: %d nodes (%s)\n", numconfigurednodes, nodes_allowed_list);
+	else
+		printf("available: %d nodes (0-%d)\n", maxnode+1, maxnode); 	
+		
 	for (i = 0; i <= maxnode; i++) { 
 		char buf[64];
 		long long fr;
 		unsigned long long sz = numa_node_size64(i, &fr); 
+		if (!numa_bitmask_isbitset(numa_all_nodes_ptr, i))
+			continue;
+
 		printf("node %d cpus:", i);
 		print_node_cpus(i);
 		printf("node %d size: %s\n", i, fmt_mem(sz, buf));
numa.h +2 -0
diff -Nuarp numactl-2.0.3-rc2.ORG/numa.h numactl-2.0.3-rc2/numa.h
@@ -161,6 +161,8 @@ extern struct bitmask *numa_no_nodes_ptr
 /* Source compatibility */
 extern nodemask_t numa_no_nodes;
 
+extern char *nodes_allowed_list;
+
 /* Only run and allocate memory from a specific set of nodes. */
 void numa_bind(struct bitmask *nodes);
 
versions.ldscript +1 -0
diff -Nuarp numactl-2.0.3-rc2.ORG/versions.ldscript numactl-2.0.3-rc2/versions.ldscript
@@ -143,6 +143,7 @@ libnuma_1.2 {
     numa_tonode_memory;
     numa_tonodemask_memory;
     numa_warn;
+    nodes_allowed_list;
   local:
     *;
 } libnuma_1.1;

--
Regards,
Amit Arora

#3 lore raw

Results after applying the patch

In reply to Amit K. Arora
Quoted 24 lines · Amit K. Arora
# ls /sys/devices/system/node
has_cpu  has_normal_memory  node0  node1  node4  node5  online  possible
# numactl --hardware
available: 6 nodes (0-5)
node 0 cpus: 0 1 2 3
node 0 size: 15232 MB
node 0 free: 2265 MB
node 1 cpus: 4 5 6 7
node 1 size: 16256 MB
node 1 free: 3108 MB
libnuma: Warning: /sys not mounted or invalid. Assuming one node: No
such file or directory
node 2 cpus:
node 2 size: <not available>
node 2 free: <not available>
node 3 cpus:
node 3 size: <not available>
node 3 free: <not available>
node 4 cpus: 8 9 10 11
node 4 size: 16000 MB
node 4 free: 9292 MB
node 5 cpus: 12 13 14 15
node 5 size: 16128 MB
node 5 free: 9479 MB
Quoted 8 lines
node distances:
node   0   1   2   3   4   5 
  0:  10  20  20  20  20  10 
  1:  20  10  20  20   0   0 
  2:   0   0   0   0   0   0 
  3:   0   0   0   0   0   0 
  4:   0   0   0  1112145  1024  2607344 
  5:  1024  2607344  1024  2607344   0   0 

After applying the two patches, we see following result on the same system:

# export LD_LIBRARY_PATH=`pwd`
# ./numactl --hardware
available: 4 nodes (0-1,4-5)
node 0 cpus: 0 1 2 3
node 0 size: 15232 MB
node 0 free: 2273 MB
node 1 cpus: 4 5 6 7
node 1 size: 16256 MB
node 1 free: 3108 MB
node 4 cpus: 8 9 10 11
node 4 size: 16000 MB
node 4 free: 9292 MB
node 5 cpus: 12 13 14 15
node 5 size: 16128 MB
node 5 free: 9479 MB
node distances:
node   0   1   2   3   4   5 
  0:  10  20   0   0  20  20 
  1:  20  10   0   0  20  20 
  2:   0   0   0   0   0   0 
  3:   0   0   0   0   0   0 
  4:  20  20   0   0  10  20 
  5:  20  20   0   0  20  10 

--
Regards,
Amit Arora

#4 lore raw
In reply to Amit K. Arora

Hi Amit,

Thanks much for the patches.

I tested them this morning. The standard tests pass on ia64 (8 nodes).

The patches are included in the numactl-2.0.3-rc2.tar.gz tarball at ftp://oss.sgi.com/www/projects/libnuma/download/

Any other review and testing is welcome.

-Cliff

Quoted 58 lines · Amit K. Arora

PROBLEM: If the NUMA nodes in a system are not contiguous, numactl shows incorrect output. Here is an example from one such system:

------------------------------------------------------

# ls /sys/devices/system/node
has_cpu  has_normal_memory  node0  node1  node4  node5  online  possible
# numactl --hardware
available: 6 nodes (0-5)
node 0 cpus: 0 1 2 3
node 0 size: 15232 MB
node 0 free: 2265 MB
node 1 cpus: 4 5 6 7
node 1 size: 16256 MB
node 1 free: 3108 MB
libnuma: Warning: /sys not mounted or invalid. Assuming one node: No
such file or directory
node 2 cpus:
node 2 size: <not available>
node 2 free: <not available>
node 3 cpus:
node 3 size: <not available>
node 3 free: <not available>
node 4 cpus: 8 9 10 11
node 4 size: 16000 MB
node 4 free: 9292 MB
node 5 cpus: 12 13 14 15
node 5 size: 16128 MB
node 5 free: 9479 MB
node distances:
node   0   1   2   3   4   5 
  0:  10  20  20  20  20  10 
  1:  20  10  20  20   0   0 
  2:   0   0   0   0   0   0 
  3:   0   0   0   0   0   0 
  4:   0   0   0  1112145  1024  2607344 
  5:  1024  2607344  1024  2607344   0   0 
------------------------------------------------------

As we see above, there are three problems with above output:
o it is showing wrong number of available nodes Actually there are 4 nodes available, but it says "6".
o it tries to access information for non-exsistent nodes from /sys Thus displaying a wrong warning message too
o the NUMA distances it shows above are clearly incorrect.

Following two patches fixes these problems:
1) numactl-handle-NUMA-dist-properly.patch
2) numactl-ignore-nonavailable-nodes.patch

--
Regards,
Amit Arora
--
To unsubscribe from this list: send the line "unsubscribe linux-numa" in the body of a message to [email protected]
More majordomo info at http://vger.kernel.org/majordomo-info.html

Signature · 4 lines
Cliff Wickman
Silicon Graphics, Inc.
[email protected]
(651) 683-3824
#5 lore raw

PATCH 1/1 distance table for non-contiguous NUMA nodes

Hi Cliff,

I am suggesting this new patch, which applies on top of the previous two (which I think you have already added to 2.0.3-rc2). This patch avoids showing those NUMA nodes which are unavailable when showing the NUMA distances too.

So, currently (with previous two patches applied) the NUMA distance table for non-contiguous NODES looks something like this (assuming nodes 2 and 3 are not existing) :

node   0   1   2   3   4   5
  0:  10  20   0   0  20  20
  1:  20  10   0   0  20  20
  2:   0   0   0   0   0   0
  3:   0   0   0   0   0   0
  4:  20  20   0   0  10  20
  5:  20  20   0   0  20  10

Please note that for non-existing NODES 2 and 3, a NUMA distance of "0" is being shown, which doesn't make sense.

Hence I propose this patch which will skip the unavailable nodes (in this case NODE 2 and 3), while printing the NUMA distances and the output on the same machine will look something like:

node 0 1 4 5

  0:  10  20  20  20 
  1:  20  10  20  20 
  4:  20  20  10  20 
  5:  20  20  20  10 

Please find the patch below. Thanks!

numactl.c +7 -3
diff -Nuarp numactl-2.0.3-rc2.ORG/numactl.c numactl-2.0.3-rc2/numactl.c
@@ -187,12 +187,16 @@ static void print_distances(int maxnode)
 	printf("node distances:\n"); 
 	printf("node ");
 	for (i = 0; i <= maxnode; i++) 
-		printf("% 3d ", i);
+		if (numa_bitmask_isbitset(numa_all_nodes_ptr, i))
+			printf("% 3d ", i);
 	printf("\n");
 	for (i = 0; i <= maxnode; i++) {
+		if (!numa_bitmask_isbitset(numa_all_nodes_ptr, i))
+			continue;
 		printf("% 3d: ", i);
-		for (k = 0; k <= maxnode; k++) 
-			printf("% 3d ", numa_distance(i,k)); 
+		for (k = 0; k <= maxnode; k++)
+			if (numa_bitmask_isbitset(numa_all_nodes_ptr, k))
+				printf("% 3d ", numa_distance(i,k)); 
 		printf("\n");
 	}			
 }

--
Regards,
Amit Arora