non-contiguous NUMA nodes handling in numactl and libnuma
cover non-contiguous NUMA nodes handling in numactl and libnuma
PROBLEM: If the NUMA nodes in a system are not contiguous, numactl shows incorrect output. Here is an example from one such system:
------------------------------------------------------
# ls /sys/devices/system/node has_cpu has_normal_memory node0 node1 node4 node5 online possible # numactl --hardware available: 6 nodes (0-5) node 0 cpus: 0 1 2 3 node 0 size: 15232 MB node 0 free: 2265 MB node 1 cpus: 4 5 6 7 node 1 size: 16256 MB node 1 free: 3108 MB libnuma: Warning: /sys not mounted or invalid. Assuming one node: No such file or directory node 2 cpus: node 2 size: <not available> node 2 free: <not available> node 3 cpus: node 3 size: <not available> node 3 free: <not available> node 4 cpus: 8 9 10 11 node 4 size: 16000 MB node 4 free: 9292 MB node 5 cpus: 12 13 14 15 node 5 size: 16128 MB node 5 free: 9479 MB node distances: node 0 1 2 3 4 5 0: 10 20 20 20 20 10 1: 20 10 20 20 0 0 2: 0 0 0 0 0 0 3: 0 0 0 0 0 0 4: 0 0 0 1112145 1024 2607344 5: 1024 2607344 1024 2607344 0 0 ------------------------------------------------------
As we see above, there are three problems with above output:
o it is showing wrong number of available nodes Actually there are 4 nodes available, but it says "6".
o it tries to access information for non-exsistent nodes from /sys Thus displaying a wrong warning message too
o the NUMA distances it shows above are clearly incorrect.
Following two patches fixes these problems:
1) numactl-handle-NUMA-dist-properly.patch
2) numactl-ignore-nonavailable-nodes.patch
--
Regards,
Amit Arora
PATCH 1/2 handle NUMA distances properly for non-contiguous nodes
Patch: numactl-handle-NUMA-dist-properly.patch
This patch fixes the problem of displaying incorrect NUMA distances. It increases the table size to accomodate node distance information for nodes from 0 to nodemax. parse_numbers() makes sure not to add distances for nodes which are not existing.
distance.c +12 -18
@@ -26,25 +26,16 @@static int distance_numnodes;static int *distance_table; -static int count_numbers (char *s) -{ -int n =0 ; -for (;;) { -char *end; -strtoul (s, &end,0 ); -if (s == end) -return n; - s = end; - n++; - } -} -static void parse_numbers (char *s,int *iptr,int n) { -int i; +int i, d, j;char *end; -for (i =0 ; i < n; i++) { - *iptr++ =strtoul (s, &end,0 ); +for (i =0 , j =0 ; i < n; i++, j++) { + d =strtoul (s, &end,0 ); +/* Skip unavailable nodes */ +while (j<n && !numa_bitmask_isbitset (numa_all_nodes_ptr, j)) + j++; + *(iptr+j) = d;if (s == end)break ; s = end;
@@ -68,7 +59,10 @@ static int read_distance_table(void)if (!dfh) {if (errno ==ENOENT && nd >0 ) err =0 ; -break ; +if (!err && nd<numa_num_configured_nodes ()) +continue ; +else +break ; } len =getdelim (&line, &linelen,'\n' , dfh);fclose (dfh);
@@ -76,7 +70,7 @@ static int read_distance_table(void)break ;if (!table) { - numnodes =count_numbers (line); + numnodes =numa_num_configured_nodes (); table =calloc (numnodes * numnodes,sizeof (int ));if (!table) { errno =ENOMEM ;
--
Regards,
Amit Arora
PATCH 2/2 ignore unavailable NUMA nodes
In reply to Amit K. AroraPatch: numactl-ignore-nonavailable-nodes.patch
This patch fixes the problem of displaying wrong number of "available" nodes. It also ignores non-existent nodes and thus avoids the wrong warning message also.
libnuma.c +8 -0
@@ -54,6 +54,8 @@ struct bitmask *numa_all_cpus_ptr = NULLstatic unsigned long *node_cpu_mask_v1[NUMA_NUM_NODES ];struct bitmask **node_cpu_mask_v2; +char *nodes_allowed_list =NULL ; + WEAK void numa_error(char *where);#ifdef __thread
@@ -452,6 +454,12 @@ set_thread_constraints(void) maxprocnode =read_mask (buffer +15 , numa_all_nodes_ptr); } +if (strncmp (buffer,"Mems_allowed_list:" ,18 ) ==0 ) { + nodes_allowed_list =malloc (strlen (buffer)-18 ); +strncpy (nodes_allowed_list, buffer +19 , +strlen (buffer) -19 ); + nodes_allowed_list[strlen (nodes_allowed_list)-1 ] ='\0' ; + } }fclose (f);free (buffer);
numactl.c +13 -2
@@ -214,13 +214,24 @@ void print_node_cpus(int node)void hardware (void ) { -int i; +int i, numconfigurednodes=0 ;int maxnode =numa_num_configured_nodes ()-1 ; -printf ("available: %d nodes (0-%d)\n" ,1 +maxnode, maxnode); + +for (i =0 ; i<=maxnode; i++) +if (numa_bitmask_isbitset (numa_all_nodes_ptr, i)) + numconfigurednodes++; +if (nodes_allowed_list) +printf ("available: %d nodes (%s)\n" , numconfigurednodes, nodes_allowed_list); +else +printf ("available: %d nodes (0-%d)\n" , maxnode+1 , maxnode); +for (i =0 ; i <= maxnode; i++) {char buf[64 ];long long fr;unsigned long long sz =numa_node_size64 (i, &fr); +if (!numa_bitmask_isbitset (numa_all_nodes_ptr, i)) +continue ; +printf ("node %d cpus:" , i);print_node_cpus (i);printf ("node %d size: %s\n" , i,fmt_mem (sz, buf));
numa.h +2 -0
@@ -161,6 +161,8 @@ extern struct bitmask *numa_no_nodes_ptr/* Source compatibility */ extern nodemask_t numa_no_nodes; +extern char *nodes_allowed_list; +/* Only run and allocate memory from a specific set of nodes. */ void numa_bind (struct bitmask *nodes);
versions.ldscript +1 -0
@@ -143,6 +143,7 @@ libnuma_1.2 { numa_tonode_memory; numa_tonodemask_memory; numa_warn; + nodes_allowed_list; local: *; } libnuma_1.1;
--
Regards,
Amit Arora
Results after applying the patch
In reply to Amit K. AroraQuoted 24 lines · Amit K. Arora
# ls /sys/devices/system/node has_cpu has_normal_memory node0 node1 node4 node5 online possible # numactl --hardware available: 6 nodes (0-5) node 0 cpus: 0 1 2 3 node 0 size: 15232 MB node 0 free: 2265 MB node 1 cpus: 4 5 6 7 node 1 size: 16256 MB node 1 free: 3108 MB libnuma: Warning: /sys not mounted or invalid. Assuming one node: No such file or directory node 2 cpus: node 2 size: <not available> node 2 free: <not available> node 3 cpus: node 3 size: <not available> node 3 free: <not available> node 4 cpus: 8 9 10 11 node 4 size: 16000 MB node 4 free: 9292 MB node 5 cpus: 12 13 14 15 node 5 size: 16128 MB node 5 free: 9479 MB
Quoted 8 lines
node distances: node 0 1 2 3 4 5 0: 10 20 20 20 20 10 1: 20 10 20 20 0 0 2: 0 0 0 0 0 0 3: 0 0 0 0 0 0 4: 0 0 0 1112145 1024 2607344 5: 1024 2607344 1024 2607344 0 0
After applying the two patches, we see following result on the same system:
# export LD_LIBRARY_PATH=`pwd` # ./numactl --hardware available: 4 nodes (0-1,4-5) node 0 cpus: 0 1 2 3 node 0 size: 15232 MB node 0 free: 2273 MB node 1 cpus: 4 5 6 7 node 1 size: 16256 MB node 1 free: 3108 MB node 4 cpus: 8 9 10 11 node 4 size: 16000 MB node 4 free: 9292 MB node 5 cpus: 12 13 14 15 node 5 size: 16128 MB node 5 free: 9479 MB node distances: node 0 1 2 3 4 5 0: 10 20 0 0 20 20 1: 20 10 0 0 20 20 2: 0 0 0 0 0 0 3: 0 0 0 0 0 0 4: 20 20 0 0 10 20 5: 20 20 0 0 20 10
--
Regards,
Amit Arora
Hi Amit,
Thanks much for the patches.
I tested them this morning. The standard tests pass on ia64 (8 nodes).
The patches are included in the numactl-2.0.3-rc2.tar.gz tarball at ftp://oss.sgi.com/www/projects/libnuma/download/
Any other review and testing is welcome.
-Cliff
Quoted 58 lines · Amit K. Arora
PROBLEM: If the NUMA nodes in a system are not contiguous, numactl shows incorrect output. Here is an example from one such system:
------------------------------------------------------
# ls /sys/devices/system/node has_cpu has_normal_memory node0 node1 node4 node5 online possible # numactl --hardware available: 6 nodes (0-5) node 0 cpus: 0 1 2 3 node 0 size: 15232 MB node 0 free: 2265 MB node 1 cpus: 4 5 6 7 node 1 size: 16256 MB node 1 free: 3108 MB libnuma: Warning: /sys not mounted or invalid. Assuming one node: No such file or directory node 2 cpus: node 2 size: <not available> node 2 free: <not available> node 3 cpus: node 3 size: <not available> node 3 free: <not available> node 4 cpus: 8 9 10 11 node 4 size: 16000 MB node 4 free: 9292 MB node 5 cpus: 12 13 14 15 node 5 size: 16128 MB node 5 free: 9479 MB node distances: node 0 1 2 3 4 5 0: 10 20 20 20 20 10 1: 20 10 20 20 0 0 2: 0 0 0 0 0 0 3: 0 0 0 0 0 0 4: 0 0 0 1112145 1024 2607344 5: 1024 2607344 1024 2607344 0 0 ------------------------------------------------------As we see above, there are three problems with above output:
o it is showing wrong number of available nodes Actually there are 4 nodes available, but it says "6".
o it tries to access information for non-exsistent nodes from /sys Thus displaying a wrong warning message too
o the NUMA distances it shows above are clearly incorrect.Following two patches fixes these problems:
1) numactl-handle-NUMA-dist-properly.patch
2) numactl-ignore-nonavailable-nodes.patch--
Regards,
Amit Arora
--
To unsubscribe from this list: send the line "unsubscribe linux-numa" in the body of a message to [email protected]
More majordomo info at http://vger.kernel.org/majordomo-info.html
Signature · 4 lines
Cliff Wickman Silicon Graphics, Inc. [email protected] (651) 683-3824
PATCH 1/1 distance table for non-contiguous NUMA nodes
Hi Cliff,
I am suggesting this new patch, which applies on top of the previous two (which I think you have already added to 2.0.3-rc2). This patch avoids showing those NUMA nodes which are unavailable when showing the NUMA distances too.
So, currently (with previous two patches applied) the NUMA distance table for non-contiguous NODES looks something like this (assuming nodes 2 and 3 are not existing) :
node 0 1 2 3 4 5 0: 10 20 0 0 20 20 1: 20 10 0 0 20 20 2: 0 0 0 0 0 0 3: 0 0 0 0 0 0 4: 20 20 0 0 10 20 5: 20 20 0 0 20 10
Please note that for non-existing NODES 2 and 3, a NUMA distance of "0" is being shown, which doesn't make sense.
Hence I propose this patch which will skip the unavailable nodes (in this case NODE 2 and 3), while printing the NUMA distances and the output on the same machine will look something like:
node 0 1 4 5
0: 10 20 20 20 1: 20 10 20 20 4: 20 20 10 20 5: 20 20 20 10
Please find the patch below. Thanks!
numactl.c +7 -3
@@ -187,12 +187,16 @@ static void print_distances(int maxnode)printf ("node distances:\n" );printf ("node " );for (i =0 ; i <= maxnode; i++) -printf ("% 3d " , i); +if (numa_bitmask_isbitset (numa_all_nodes_ptr, i)) +printf ("% 3d " , i);printf ("\n" );for (i =0 ; i <= maxnode; i++) { +if (!numa_bitmask_isbitset (numa_all_nodes_ptr, i)) +continue ;printf ("% 3d: " , i); -for (k =0 ; k <= maxnode; k++) -printf ("% 3d " ,numa_distance (i,k)); +for (k =0 ; k <= maxnode; k++) +if (numa_bitmask_isbitset (numa_all_nodes_ptr, k)) +printf ("% 3d " ,numa_distance (i,k));printf ("\n" ); } }
--
Regards,
Amit Arora